Merge branch 'master' into devel/sphinx

1645 files changed, 74417 insertions, 21731 deletions

diff --git a/ChangeLog b/ChangeLog
index f468393..4b8570b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,41 @@
+2021-11-12  Alan Modra  <amodra@gmail.com>
+
+	* Makefile.def (configure-opcodes): Depend on configure-bfd.
+	* Makefile.in: Regenerate.
+
+2021-11-03  Maciej W. Rozycki  <macro@embecosm.com>
+
+	* MAINTAINERS: Clarify the policy WRT the Write After Approval
+	list.
+
+2021-11-02  Andrew Burgess  <andrew.burgess@embecosm.com>
+
+	* MAINTAINERS (Reviewers, arc): Remove my entry.
+	(Write After Approval): Add an entry for myself.
+
+2021-10-28  Richard Purdie  <richard.purdie@linuxfoundation.org>
+
+	* Makefile.tpl: Add CPP_FOR_BUILD and CPPFLAGS_FOR_BUILD support
+	* Makefile.in: Regenerate.
+	* configure: Regenerate.
+	* configure.ac: Add CPP_FOR_BUILD and CPPFLAGS_FOR_BUILD support
+
+2021-10-26  Jeff Law  <jeffreyalaw@gmail.com>
+
+	* MAINTAINERS: Fix up Maciej's entries.
+
+2021-10-26  Maciej W. Rozycki  <macro@embecosm.com>
+
+	* MAINTAINERS (CPU Port Maintainers): Add myself as a VAX port
+	maintainer.
+
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Makefile.def: Handle install-dvi target.
+	* Makefile.tpl: Likewise.
+	* Makefile.in: Regenerate.
+
 2021-10-19  Clément Chigot  <clement.chigot@atos.net>
 
 	* MAINTAINERS: Add myself for write after approval.
diff --git a/MAINTAINERS b/MAINTAINERS
index b22f930..1b70fce 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -15,6 +15,9 @@ To report problems in GCC, please visit:
 
   http://gcc.gnu.org/bugs/
 
+Note: when adding someone to a more specific section please remove any
+corresponding entry from the Write After Approval list.
+
 Maintainers
 ===========
 
@@ -101,7 +104,7 @@ pru port		Dimitar Dimitrov	<dimitar@dinux.eu>
 riscv port		Kito Cheng		<kito.cheng@gmail.com>
 riscv port		Palmer Dabbelt		<palmer@dabbelt.com>
 riscv port		Andrew Waterman		<andrew@sifive.com>
-riscv port		Jim Wilson		<jimw@sifive.com>
+riscv port		Jim Wilson		<jim.wilson.gcc@gmail.com>
 rs6000/powerpc port	David Edelsohn		<dje.gcc@gmail.com>
 rs6000/powerpc port	Segher Boessenkool	<segher@kernel.crashing.org>
 rs6000 vector extns	Aldy Hernandez		<aldyh@redhat.com>
@@ -117,6 +120,7 @@ tilegx port		Walter Lee		<walt@tilera.com>
 tilepro port		Walter Lee		<walt@tilera.com>
 v850 port		Nick Clifton		<nickc@redhat.com>
 vax port		Matt Thomas		<matt@3am-software.com>
+vax port		Maciej W. Rozycki	<macro@embecosm.com>
 visium port		Eric Botcazou		<ebotcazou@libertysurf.fr>
 x86-64 port		Jan Hubicka		<hubicka@ucw.cz>
 xstormy16 port		Nick Clifton		<nickc@redhat.com>
@@ -256,7 +260,6 @@ check in changes outside of the parts of the compiler they maintain.
 
 			Reviewers
 
-arc port		Andrew Burgess		<andrew.burgess@embecosm.com>
 arc port		Claudiu Zissulescu	<claziss@synopsys.com>
 callgraph		Martin Liska		<mliska@suse.cz>
 callgraph		Martin Jambor		<mjambor@suse.cz>
@@ -343,6 +346,7 @@ Robert Bowdidge					<bowdidge@apple.com>
 Joel Brobecker					<brobecker@gnat.com>
 Dave Brolley					<brolley@redhat.com>
 Christian Bruel					<christian.bruel@st.com>
+Andrew Burgess					<aburgess@redhat.com>
 Kevin Buettner					<kevinb@redhat.com>
 Adam Butcher					<adam@jessamine.co.uk>
 Andrew Cagney					<cagney@gnu.org>
@@ -590,7 +594,6 @@ Erven Rohou					<erven.rohou@inria.fr>
 Ira Rosen					<irar@il.ibm.com>
 Yvan Roux					<yvan.roux@linaro.org>
 Eugene Rozenfeld				<erozen@microsoft.com>
-Maciej W. Rozycki				<macro@orcam.me.uk>
 Silvius Rus					<rus@google.com>
 Matthew Sachs					<msachs@apple.com>
 Ankur Saini					<arsenic@sourceware.org>
diff --git a/Makefile.def b/Makefile.def
index d70020d..a504192 100644
--- a/Makefile.def
+++ b/Makefile.def
@@ -4,7 +4,7 @@ AutoGen definitions Makefile.tpl;
 // Makefile.in is generated from Makefile.tpl by 'autogen Makefile.def'.
 // This file was originally written by Nathanael Nerode.
 //
-//   Copyright 2002-2019 Free Software Foundation
+//   Copyright 2002-2021 Free Software Foundation
 //
 // This file is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -42,7 +42,8 @@ host_modules= { module= dejagnu; };
 host_modules= { module= etc; };
 host_modules= { module= fastjar; no_check_cross= true; };
 host_modules= { module= fixincludes; bootstrap=true;
-		missing= TAGS; };
+		missing= TAGS;
+		missing= install-dvi; };
 host_modules= { module= flex; no_check_cross= true; };
 host_modules= { module= gas; bootstrap=true; };
 host_modules= { module= gcc; bootstrap=true; 
@@ -87,9 +88,11 @@ host_modules= { module= libcody; bootstrap=true;
 		missing= pdf;
 		missing= html;
 		missing= info;
+		missing= dvi;
 		missing= install-pdf;
 		missing= install-html;
 		missing= install-info;
+		missing= install-dvi;
 		missing=TAGS; };
 host_modules= { module= libdecnumber; bootstrap=true;
 		missing=TAGS; };
@@ -164,12 +167,14 @@ target_modules = { module= liboffloadmic;
 target_modules = { module= libssp; lib_path=.libs; };
 target_modules = { module= newlib; };
 target_modules = { module= libgcc; bootstrap=true; no_check=true;
-		  missing=TAGS; };
+		  missing=TAGS;
+		  missing=install-dvi; };
 target_modules = { module= libbacktrace; };
 target_modules = { module= libquadmath; };
 target_modules = { module= libgfortran; };
 target_modules = { module= libobjc;
-		  missing=TAGS; };
+		  missing=TAGS;
+		  missing=install-dvi; };
 target_modules = { module= libgo; };
 target_modules = { module= libphobos;
 		   lib_path=src/.libs; };
@@ -205,6 +210,9 @@ recursive_targets = { make_target= TAGS;
 recursive_targets = { make_target= install-info; 
                       depend=configure;
                       depend=info; };
+recursive_targets = { make_target= install-dvi;
+                      depend=configure;
+                      depend=dvi; };
 recursive_targets = { make_target= install-pdf; 
                       depend=configure;
                       depend=pdf; };
@@ -485,6 +493,7 @@ dependencies = { module=install-strip-ld; on=install-strip-bfd; };
 dependencies = { module=install-strip-ld; on=install-strip-libctf; };
 
 // libopcodes depends on libbfd
+dependencies = { module=configure-opcodes; on=configure-bfd; hard=true; };
 dependencies = { module=install-opcodes; on=install-bfd; };
 dependencies = { module=install-strip-opcodes; on=install-strip-bfd; };
 
diff --git a/Makefile.in b/Makefile.in
index 27e8a68..860cf8f 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -154,6 +154,8 @@ BUILD_EXPORTS = \
 	CC="$(CC_FOR_BUILD)"; export CC; \
 	CFLAGS="$(CFLAGS_FOR_BUILD)"; export CFLAGS; \
 	CONFIG_SHELL="$(SHELL)"; export CONFIG_SHELL; \
+	CPP="$(CPP_FOR_BUILD)"; export CPP; \
+	CPPFLAGS="$(CPPFLAGS_FOR_BUILD)"; export CPPFLAGS; \
 	CXX="$(CXX_FOR_BUILD)"; export CXX; \
 	CXXFLAGS="$(CXXFLAGS_FOR_BUILD)"; export CXXFLAGS; \
 	GFORTRAN="$(GFORTRAN_FOR_BUILD)"; export GFORTRAN; \
@@ -202,6 +204,8 @@ HOST_EXPORTS = \
 	AR="$(AR)"; export AR; \
 	AS="$(AS)"; export AS; \
 	CC_FOR_BUILD="$(CC_FOR_BUILD)"; export CC_FOR_BUILD; \
+	CPP_FOR_BUILD="$(CPP_FOR_BUILD)"; export CPP_FOR_BUILD; \
+	CPPFLAGS_FOR_BUILD="$(CPPFLAGS_FOR_BUILD)"; export CPPFLAGS_FOR_BUILD; \
 	CXX_FOR_BUILD="$(CXX_FOR_BUILD)"; export CXX_FOR_BUILD; \
 	DLLTOOL="$(DLLTOOL)"; export DLLTOOL; \
 	DSYMUTIL="$(DSYMUTIL)"; export DSYMUTIL; \
@@ -360,6 +364,8 @@ AR_FOR_BUILD = @AR_FOR_BUILD@
 AS_FOR_BUILD = @AS_FOR_BUILD@
 CC_FOR_BUILD = @CC_FOR_BUILD@
 CFLAGS_FOR_BUILD = @CFLAGS_FOR_BUILD@
+CPP_FOR_BUILD = @CPP_FOR_BUILD@
+CPPFLAGS_FOR_BUILD = @CPPFLAGS_FOR_BUILD@
 CXXFLAGS_FOR_BUILD = @CXXFLAGS_FOR_BUILD@
 CXX_FOR_BUILD = @CXX_FOR_BUILD@
 DLLTOOL_FOR_BUILD = @DLLTOOL_FOR_BUILD@
@@ -1802,6 +1808,96 @@ install-info-target: maybe-install-info-target-libgomp
 install-info-target: maybe-install-info-target-libitm
 install-info-target: maybe-install-info-target-libatomic
 
+.PHONY: do-install-dvi
+do-install-dvi:
+	@: $(MAKE); $(unstage)
+	@r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(MAKE) $(RECURSE_FLAGS_TO_PASS) install-dvi-host \
+	  install-dvi-target
+
+
+.PHONY: install-dvi-host
+
+install-dvi-host: maybe-install-dvi-bfd
+install-dvi-host: maybe-install-dvi-opcodes
+install-dvi-host: maybe-install-dvi-binutils
+install-dvi-host: maybe-install-dvi-bison
+install-dvi-host: maybe-install-dvi-cgen
+install-dvi-host: maybe-install-dvi-dejagnu
+install-dvi-host: maybe-install-dvi-etc
+install-dvi-host: maybe-install-dvi-fastjar
+install-dvi-host: maybe-install-dvi-fixincludes
+install-dvi-host: maybe-install-dvi-flex
+install-dvi-host: maybe-install-dvi-gas
+install-dvi-host: maybe-install-dvi-gcc
+install-dvi-host: maybe-install-dvi-gmp
+install-dvi-host: maybe-install-dvi-mpfr
+install-dvi-host: maybe-install-dvi-mpc
+install-dvi-host: maybe-install-dvi-isl
+install-dvi-host: maybe-install-dvi-libelf
+install-dvi-host: maybe-install-dvi-gold
+install-dvi-host: maybe-install-dvi-gprof
+install-dvi-host: maybe-install-dvi-intl
+install-dvi-host: maybe-install-dvi-tcl
+install-dvi-host: maybe-install-dvi-itcl
+install-dvi-host: maybe-install-dvi-ld
+install-dvi-host: maybe-install-dvi-libbacktrace
+install-dvi-host: maybe-install-dvi-libcpp
+install-dvi-host: maybe-install-dvi-libcody
+install-dvi-host: maybe-install-dvi-libdecnumber
+install-dvi-host: maybe-install-dvi-libgui
+install-dvi-host: maybe-install-dvi-libiberty
+install-dvi-host: maybe-install-dvi-libiberty-linker-plugin
+install-dvi-host: maybe-install-dvi-libiconv
+install-dvi-host: maybe-install-dvi-m4
+install-dvi-host: maybe-install-dvi-readline
+install-dvi-host: maybe-install-dvi-sid
+install-dvi-host: maybe-install-dvi-sim
+install-dvi-host: maybe-install-dvi-texinfo
+install-dvi-host: maybe-install-dvi-zlib
+install-dvi-host: maybe-install-dvi-gnulib
+install-dvi-host: maybe-install-dvi-gdbsupport
+install-dvi-host: maybe-install-dvi-gdbserver
+install-dvi-host: maybe-install-dvi-gdb
+install-dvi-host: maybe-install-dvi-expect
+install-dvi-host: maybe-install-dvi-guile
+install-dvi-host: maybe-install-dvi-tk
+install-dvi-host: maybe-install-dvi-libtermcap
+install-dvi-host: maybe-install-dvi-utils
+install-dvi-host: maybe-install-dvi-c++tools
+install-dvi-host: maybe-install-dvi-gnattools
+install-dvi-host: maybe-install-dvi-lto-plugin
+install-dvi-host: maybe-install-dvi-libcc1
+install-dvi-host: maybe-install-dvi-gotools
+install-dvi-host: maybe-install-dvi-libctf
+
+.PHONY: install-dvi-target
+
+install-dvi-target: maybe-install-dvi-target-libstdc++-v3
+install-dvi-target: maybe-install-dvi-target-libsanitizer
+install-dvi-target: maybe-install-dvi-target-libvtv
+install-dvi-target: maybe-install-dvi-target-liboffloadmic
+install-dvi-target: maybe-install-dvi-target-libssp
+install-dvi-target: maybe-install-dvi-target-newlib
+install-dvi-target: maybe-install-dvi-target-libgcc
+install-dvi-target: maybe-install-dvi-target-libbacktrace
+install-dvi-target: maybe-install-dvi-target-libquadmath
+install-dvi-target: maybe-install-dvi-target-libgfortran
+install-dvi-target: maybe-install-dvi-target-libobjc
+install-dvi-target: maybe-install-dvi-target-libgo
+install-dvi-target: maybe-install-dvi-target-libphobos
+install-dvi-target: maybe-install-dvi-target-libtermcap
+install-dvi-target: maybe-install-dvi-target-winsup
+install-dvi-target: maybe-install-dvi-target-libgloss
+install-dvi-target: maybe-install-dvi-target-libffi
+install-dvi-target: maybe-install-dvi-target-zlib
+install-dvi-target: maybe-install-dvi-target-rda
+install-dvi-target: maybe-install-dvi-target-libada
+install-dvi-target: maybe-install-dvi-target-libgomp
+install-dvi-target: maybe-install-dvi-target-libitm
+install-dvi-target: maybe-install-dvi-target-libatomic
+
 .PHONY: do-install-pdf
 do-install-pdf:
 	@: $(MAKE); $(unstage)
@@ -2436,7 +2532,7 @@ maintainer-clean-target: maybe-maintainer-clean-target-libatomic
 # Here are the targets which correspond to the do-X targets.
 
 .PHONY: info installcheck dvi pdf html
-.PHONY: install-info install-pdf install-html
+.PHONY: install-info install-dvi install-pdf install-html
 .PHONY: clean distclean mostlyclean maintainer-clean realclean
 .PHONY: local-clean local-distclean local-maintainer-clean
 info: do-info
@@ -2455,6 +2551,8 @@ install-info: do-install-info dir.info
 	  $(INSTALL_DATA) dir.info $(DESTDIR)$(infodir)/dir.info; \
 	else true; fi
 
+install-dvi: do-install-dvi
+
 install-pdf: do-install-pdf
 
 install-html: do-install-html
@@ -4267,6 +4365,32 @@ install-info-bfd: \
 
 @endif bfd
 
+.PHONY: maybe-install-dvi-bfd install-dvi-bfd
+maybe-install-dvi-bfd:
+@if bfd
+maybe-install-dvi-bfd: install-dvi-bfd
+
+install-dvi-bfd: \
+    configure-bfd \
+    dvi-bfd 
+	@[ -f ./bfd/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in bfd"; \
+	(cd $(HOST_SUBDIR)/bfd && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif bfd
+
 .PHONY: maybe-install-pdf-bfd install-pdf-bfd
 maybe-install-pdf-bfd:
 @if bfd
@@ -5381,6 +5505,32 @@ install-info-opcodes: \
 
 @endif opcodes
 
+.PHONY: maybe-install-dvi-opcodes install-dvi-opcodes
+maybe-install-dvi-opcodes:
+@if opcodes
+maybe-install-dvi-opcodes: install-dvi-opcodes
+
+install-dvi-opcodes: \
+    configure-opcodes \
+    dvi-opcodes 
+	@[ -f ./opcodes/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in opcodes"; \
+	(cd $(HOST_SUBDIR)/opcodes && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif opcodes
+
 .PHONY: maybe-install-pdf-opcodes install-pdf-opcodes
 maybe-install-pdf-opcodes:
 @if opcodes
@@ -6495,6 +6645,32 @@ install-info-binutils: \
 
 @endif binutils
 
+.PHONY: maybe-install-dvi-binutils install-dvi-binutils
+maybe-install-dvi-binutils:
+@if binutils
+maybe-install-dvi-binutils: install-dvi-binutils
+
+install-dvi-binutils: \
+    configure-binutils \
+    dvi-binutils 
+	@[ -f ./binutils/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in binutils"; \
+	(cd $(HOST_SUBDIR)/binutils && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif binutils
+
 .PHONY: maybe-install-pdf-binutils install-pdf-binutils
 maybe-install-pdf-binutils:
 @if binutils
@@ -6932,6 +7108,33 @@ install-info-bison: \
 
 @endif bison
 
+.PHONY: maybe-install-dvi-bison install-dvi-bison
+maybe-install-dvi-bison:
+@if bison
+maybe-install-dvi-bison: install-dvi-bison
+
+install-dvi-bison: \
+    configure-bison \
+    dvi-bison 
+	@: $(MAKE); $(unstage)
+	@[ -f ./bison/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in bison"; \
+	(cd $(HOST_SUBDIR)/bison && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif bison
+
 .PHONY: maybe-install-pdf-bison install-pdf-bison
 maybe-install-pdf-bison:
 @if bison
@@ -7373,6 +7576,33 @@ install-info-cgen: \
 
 @endif cgen
 
+.PHONY: maybe-install-dvi-cgen install-dvi-cgen
+maybe-install-dvi-cgen:
+@if cgen
+maybe-install-dvi-cgen: install-dvi-cgen
+
+install-dvi-cgen: \
+    configure-cgen \
+    dvi-cgen 
+	@: $(MAKE); $(unstage)
+	@[ -f ./cgen/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in cgen"; \
+	(cd $(HOST_SUBDIR)/cgen && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif cgen
+
 .PHONY: maybe-install-pdf-cgen install-pdf-cgen
 maybe-install-pdf-cgen:
 @if cgen
@@ -7814,6 +8044,33 @@ install-info-dejagnu: \
 
 @endif dejagnu
 
+.PHONY: maybe-install-dvi-dejagnu install-dvi-dejagnu
+maybe-install-dvi-dejagnu:
+@if dejagnu
+maybe-install-dvi-dejagnu: install-dvi-dejagnu
+
+install-dvi-dejagnu: \
+    configure-dejagnu \
+    dvi-dejagnu 
+	@: $(MAKE); $(unstage)
+	@[ -f ./dejagnu/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in dejagnu"; \
+	(cd $(HOST_SUBDIR)/dejagnu && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif dejagnu
+
 .PHONY: maybe-install-pdf-dejagnu install-pdf-dejagnu
 maybe-install-pdf-dejagnu:
 @if dejagnu
@@ -8255,6 +8512,33 @@ install-info-etc: \
 
 @endif etc
 
+.PHONY: maybe-install-dvi-etc install-dvi-etc
+maybe-install-dvi-etc:
+@if etc
+maybe-install-dvi-etc: install-dvi-etc
+
+install-dvi-etc: \
+    configure-etc \
+    dvi-etc 
+	@: $(MAKE); $(unstage)
+	@[ -f ./etc/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in etc"; \
+	(cd $(HOST_SUBDIR)/etc && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif etc
+
 .PHONY: maybe-install-pdf-etc install-pdf-etc
 maybe-install-pdf-etc:
 @if etc
@@ -8699,6 +8983,33 @@ install-info-fastjar: \
 
 @endif fastjar
 
+.PHONY: maybe-install-dvi-fastjar install-dvi-fastjar
+maybe-install-dvi-fastjar:
+@if fastjar
+maybe-install-dvi-fastjar: install-dvi-fastjar
+
+install-dvi-fastjar: \
+    configure-fastjar \
+    dvi-fastjar 
+	@: $(MAKE); $(unstage)
+	@[ -f ./fastjar/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in fastjar"; \
+	(cd $(HOST_SUBDIR)/fastjar && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif fastjar
+
 .PHONY: maybe-install-pdf-fastjar install-pdf-fastjar
 maybe-install-pdf-fastjar:
 @if fastjar
@@ -9805,6 +10116,16 @@ install-info-fixincludes: \
 
 @endif fixincludes
 
+.PHONY: maybe-install-dvi-fixincludes install-dvi-fixincludes
+maybe-install-dvi-fixincludes:
+@if fixincludes
+maybe-install-dvi-fixincludes: install-dvi-fixincludes
+
+# fixincludes doesn't support install-dvi.
+install-dvi-fixincludes:
+
+@endif fixincludes
+
 .PHONY: maybe-install-pdf-fixincludes install-pdf-fixincludes
 maybe-install-pdf-fixincludes:
 @if fixincludes
@@ -10242,6 +10563,33 @@ install-info-flex: \
 
 @endif flex
 
+.PHONY: maybe-install-dvi-flex install-dvi-flex
+maybe-install-dvi-flex:
+@if flex
+maybe-install-dvi-flex: install-dvi-flex
+
+install-dvi-flex: \
+    configure-flex \
+    dvi-flex 
+	@: $(MAKE); $(unstage)
+	@[ -f ./flex/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in flex"; \
+	(cd $(HOST_SUBDIR)/flex && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif flex
+
 .PHONY: maybe-install-pdf-flex install-pdf-flex
 maybe-install-pdf-flex:
 @if flex
@@ -11363,6 +11711,32 @@ install-info-gas: \
 
 @endif gas
 
+.PHONY: maybe-install-dvi-gas install-dvi-gas
+maybe-install-dvi-gas:
+@if gas
+maybe-install-dvi-gas: install-dvi-gas
+
+install-dvi-gas: \
+    configure-gas \
+    dvi-gas 
+	@[ -f ./gas/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in gas"; \
+	(cd $(HOST_SUBDIR)/gas && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif gas
+
 .PHONY: maybe-install-pdf-gas install-pdf-gas
 maybe-install-pdf-gas:
 @if gas
@@ -12477,6 +12851,32 @@ install-info-gcc: \
 
 @endif gcc
 
+.PHONY: maybe-install-dvi-gcc install-dvi-gcc
+maybe-install-dvi-gcc:
+@if gcc
+maybe-install-dvi-gcc: install-dvi-gcc
+
+install-dvi-gcc: \
+    configure-gcc \
+    dvi-gcc 
+	@[ -f ./gcc/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) $(EXTRA_GCC_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in gcc"; \
+	(cd $(HOST_SUBDIR)/gcc && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif gcc
+
 .PHONY: maybe-install-pdf-gcc install-pdf-gcc
 maybe-install-pdf-gcc:
 @if gcc
@@ -13588,6 +13988,32 @@ install-info-gmp: \
 
 @endif gmp
 
+.PHONY: maybe-install-dvi-gmp install-dvi-gmp
+maybe-install-dvi-gmp:
+@if gmp
+maybe-install-dvi-gmp: install-dvi-gmp
+
+install-dvi-gmp: \
+    configure-gmp \
+    dvi-gmp 
+	@[ -f ./gmp/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) AM_CFLAGS="-DNO_ASM"; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in gmp"; \
+	(cd $(HOST_SUBDIR)/gmp && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif gmp
+
 .PHONY: maybe-install-pdf-gmp install-pdf-gmp
 maybe-install-pdf-gmp:
 @if gmp
@@ -14699,6 +15125,32 @@ install-info-mpfr: \
 
 @endif mpfr
 
+.PHONY: maybe-install-dvi-mpfr install-dvi-mpfr
+maybe-install-dvi-mpfr:
+@if mpfr
+maybe-install-dvi-mpfr: install-dvi-mpfr
+
+install-dvi-mpfr: \
+    configure-mpfr \
+    dvi-mpfr 
+	@[ -f ./mpfr/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) AM_CFLAGS="-DNO_ASM"; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in mpfr"; \
+	(cd $(HOST_SUBDIR)/mpfr && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif mpfr
+
 .PHONY: maybe-install-pdf-mpfr install-pdf-mpfr
 maybe-install-pdf-mpfr:
 @if mpfr
@@ -15810,6 +16262,32 @@ install-info-mpc: \
 
 @endif mpc
 
+.PHONY: maybe-install-dvi-mpc install-dvi-mpc
+maybe-install-dvi-mpc:
+@if mpc
+maybe-install-dvi-mpc: install-dvi-mpc
+
+install-dvi-mpc: \
+    configure-mpc \
+    dvi-mpc 
+	@[ -f ./mpc/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in mpc"; \
+	(cd $(HOST_SUBDIR)/mpc && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif mpc
+
 .PHONY: maybe-install-pdf-mpc install-pdf-mpc
 maybe-install-pdf-mpc:
 @if mpc
@@ -16921,6 +17399,32 @@ install-info-isl: \
 
 @endif isl
 
+.PHONY: maybe-install-dvi-isl install-dvi-isl
+maybe-install-dvi-isl:
+@if isl
+maybe-install-dvi-isl: install-dvi-isl
+
+install-dvi-isl: \
+    configure-isl \
+    dvi-isl 
+	@[ -f ./isl/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) V=1; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in isl"; \
+	(cd $(HOST_SUBDIR)/isl && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif isl
+
 .PHONY: maybe-install-pdf-isl install-pdf-isl
 maybe-install-pdf-isl:
 @if isl
@@ -18032,6 +18536,32 @@ install-info-libelf: \
 
 @endif libelf
 
+.PHONY: maybe-install-dvi-libelf install-dvi-libelf
+maybe-install-dvi-libelf:
+@if libelf
+maybe-install-dvi-libelf: install-dvi-libelf
+
+install-dvi-libelf: \
+    configure-libelf \
+    dvi-libelf 
+	@[ -f ./libelf/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in libelf"; \
+	(cd $(HOST_SUBDIR)/libelf && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif libelf
+
 .PHONY: maybe-install-pdf-libelf install-pdf-libelf
 maybe-install-pdf-libelf:
 @if libelf
@@ -19146,6 +19676,32 @@ install-info-gold: \
 
 @endif gold
 
+.PHONY: maybe-install-dvi-gold install-dvi-gold
+maybe-install-dvi-gold:
+@if gold
+maybe-install-dvi-gold: install-dvi-gold
+
+install-dvi-gold: \
+    configure-gold \
+    dvi-gold 
+	@[ -f ./gold/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in gold"; \
+	(cd $(HOST_SUBDIR)/gold && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif gold
+
 .PHONY: maybe-install-pdf-gold install-pdf-gold
 maybe-install-pdf-gold:
 @if gold
@@ -19580,6 +20136,33 @@ install-info-gprof: \
 
 @endif gprof
 
+.PHONY: maybe-install-dvi-gprof install-dvi-gprof
+maybe-install-dvi-gprof:
+@if gprof
+maybe-install-dvi-gprof: install-dvi-gprof
+
+install-dvi-gprof: \
+    configure-gprof \
+    dvi-gprof 
+	@: $(MAKE); $(unstage)
+	@[ -f ./gprof/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in gprof"; \
+	(cd $(HOST_SUBDIR)/gprof && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif gprof
+
 .PHONY: maybe-install-pdf-gprof install-pdf-gprof
 maybe-install-pdf-gprof:
 @if gprof
@@ -20701,6 +21284,32 @@ install-info-intl: \
 
 @endif intl
 
+.PHONY: maybe-install-dvi-intl install-dvi-intl
+maybe-install-dvi-intl:
+@if intl
+maybe-install-dvi-intl: install-dvi-intl
+
+install-dvi-intl: \
+    configure-intl \
+    dvi-intl 
+	@[ -f ./intl/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in intl"; \
+	(cd $(HOST_SUBDIR)/intl && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif intl
+
 .PHONY: maybe-install-pdf-intl install-pdf-intl
 maybe-install-pdf-intl:
 @if intl
@@ -21135,6 +21744,33 @@ install-info-tcl: \
 
 @endif tcl
 
+.PHONY: maybe-install-dvi-tcl install-dvi-tcl
+maybe-install-dvi-tcl:
+@if tcl
+maybe-install-dvi-tcl: install-dvi-tcl
+
+install-dvi-tcl: \
+    configure-tcl \
+    dvi-tcl 
+	@: $(MAKE); $(unstage)
+	@[ -f ./tcl/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in tcl"; \
+	(cd $(HOST_SUBDIR)/tcl && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif tcl
+
 .PHONY: maybe-install-pdf-tcl install-pdf-tcl
 maybe-install-pdf-tcl:
 @if tcl
@@ -21561,6 +22197,33 @@ install-info-itcl: \
 
 @endif itcl
 
+.PHONY: maybe-install-dvi-itcl install-dvi-itcl
+maybe-install-dvi-itcl:
+@if itcl
+maybe-install-dvi-itcl: install-dvi-itcl
+
+install-dvi-itcl: \
+    configure-itcl \
+    dvi-itcl 
+	@: $(MAKE); $(unstage)
+	@[ -f ./itcl/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in itcl"; \
+	(cd $(HOST_SUBDIR)/itcl && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif itcl
+
 .PHONY: maybe-install-pdf-itcl install-pdf-itcl
 maybe-install-pdf-itcl:
 @if itcl
@@ -22682,6 +23345,32 @@ install-info-ld: \
 
 @endif ld
 
+.PHONY: maybe-install-dvi-ld install-dvi-ld
+maybe-install-dvi-ld:
+@if ld
+maybe-install-dvi-ld: install-dvi-ld
+
+install-dvi-ld: \
+    configure-ld \
+    dvi-ld 
+	@[ -f ./ld/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in ld"; \
+	(cd $(HOST_SUBDIR)/ld && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif ld
+
 .PHONY: maybe-install-pdf-ld install-pdf-ld
 maybe-install-pdf-ld:
 @if ld
@@ -23796,6 +24485,32 @@ install-info-libbacktrace: \
 
 @endif libbacktrace
 
+.PHONY: maybe-install-dvi-libbacktrace install-dvi-libbacktrace
+maybe-install-dvi-libbacktrace:
+@if libbacktrace
+maybe-install-dvi-libbacktrace: install-dvi-libbacktrace
+
+install-dvi-libbacktrace: \
+    configure-libbacktrace \
+    dvi-libbacktrace 
+	@[ -f ./libbacktrace/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in libbacktrace"; \
+	(cd $(HOST_SUBDIR)/libbacktrace && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif libbacktrace
+
 .PHONY: maybe-install-pdf-libbacktrace install-pdf-libbacktrace
 maybe-install-pdf-libbacktrace:
 @if libbacktrace
@@ -24910,6 +25625,32 @@ install-info-libcpp: \
 
 @endif libcpp
 
+.PHONY: maybe-install-dvi-libcpp install-dvi-libcpp
+maybe-install-dvi-libcpp:
+@if libcpp
+maybe-install-dvi-libcpp: install-dvi-libcpp
+
+install-dvi-libcpp: \
+    configure-libcpp \
+    dvi-libcpp 
+	@[ -f ./libcpp/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in libcpp"; \
+	(cd $(HOST_SUBDIR)/libcpp && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif libcpp
+
 .PHONY: maybe-install-pdf-libcpp install-pdf-libcpp
 maybe-install-pdf-libcpp:
 @if libcpp
@@ -25876,23 +26617,8 @@ maybe-dvi-libcody:
 @if libcody
 maybe-dvi-libcody: dvi-libcody
 
-dvi-libcody: \
-    configure-libcody 
-	@[ -f ./libcody/Makefile ] || exit 0; \
-	r=`${PWD_COMMAND}`; export r; \
-	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
-	$(HOST_EXPORTS) \
-	for flag in $(EXTRA_HOST_FLAGS) ; do \
-	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
-	done; \
-	echo "Doing dvi in libcody"; \
-	(cd $(HOST_SUBDIR)/libcody && \
-	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
-	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
-	          "RANLIB=$${RANLIB}" \
-	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
-	          dvi) \
-	  || exit 1
+# libcody doesn't support dvi.
+dvi-libcody:
 
 @endif libcody
 
@@ -25936,6 +26662,16 @@ install-info-libcody:
 
 @endif libcody
 
+.PHONY: maybe-install-dvi-libcody install-dvi-libcody
+maybe-install-dvi-libcody:
+@if libcody
+maybe-install-dvi-libcody: install-dvi-libcody
+
+# libcody doesn't support install-dvi.
+install-dvi-libcody:
+
+@endif libcody
+
 .PHONY: maybe-install-pdf-libcody install-pdf-libcody
 maybe-install-pdf-libcody:
 @if libcody
@@ -27003,6 +27739,32 @@ install-info-libdecnumber: \
 
 @endif libdecnumber
 
+.PHONY: maybe-install-dvi-libdecnumber install-dvi-libdecnumber
+maybe-install-dvi-libdecnumber:
+@if libdecnumber
+maybe-install-dvi-libdecnumber: install-dvi-libdecnumber
+
+install-dvi-libdecnumber: \
+    configure-libdecnumber \
+    dvi-libdecnumber 
+	@[ -f ./libdecnumber/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in libdecnumber"; \
+	(cd $(HOST_SUBDIR)/libdecnumber && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif libdecnumber
+
 .PHONY: maybe-install-pdf-libdecnumber install-pdf-libdecnumber
 maybe-install-pdf-libdecnumber:
 @if libdecnumber
@@ -27437,6 +28199,33 @@ install-info-libgui: \
 
 @endif libgui
 
+.PHONY: maybe-install-dvi-libgui install-dvi-libgui
+maybe-install-dvi-libgui:
+@if libgui
+maybe-install-dvi-libgui: install-dvi-libgui
+
+install-dvi-libgui: \
+    configure-libgui \
+    dvi-libgui 
+	@: $(MAKE); $(unstage)
+	@[ -f ./libgui/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in libgui"; \
+	(cd $(HOST_SUBDIR)/libgui && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif libgui
+
 .PHONY: maybe-install-pdf-libgui install-pdf-libgui
 maybe-install-pdf-libgui:
 @if libgui
@@ -28567,6 +29356,32 @@ install-info-libiberty: \
 
 @endif libiberty
 
+.PHONY: maybe-install-dvi-libiberty install-dvi-libiberty
+maybe-install-dvi-libiberty:
+@if libiberty
+maybe-install-dvi-libiberty: install-dvi-libiberty
+
+install-dvi-libiberty: \
+    configure-libiberty \
+    dvi-libiberty 
+	@[ -f ./libiberty/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in libiberty"; \
+	(cd $(HOST_SUBDIR)/libiberty && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif libiberty
+
 .PHONY: maybe-install-pdf-libiberty install-pdf-libiberty
 maybe-install-pdf-libiberty:
 @if libiberty
@@ -29690,6 +30505,32 @@ install-info-libiberty-linker-plugin: \
 
 @endif libiberty-linker-plugin
 
+.PHONY: maybe-install-dvi-libiberty-linker-plugin install-dvi-libiberty-linker-plugin
+maybe-install-dvi-libiberty-linker-plugin:
+@if libiberty-linker-plugin
+maybe-install-dvi-libiberty-linker-plugin: install-dvi-libiberty-linker-plugin
+
+install-dvi-libiberty-linker-plugin: \
+    configure-libiberty-linker-plugin \
+    dvi-libiberty-linker-plugin 
+	@[ -f ./libiberty-linker-plugin/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) @extra_linker_plugin_flags@; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in libiberty-linker-plugin"; \
+	(cd $(HOST_SUBDIR)/libiberty-linker-plugin && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif libiberty-linker-plugin
+
 .PHONY: maybe-install-pdf-libiberty-linker-plugin install-pdf-libiberty-linker-plugin
 maybe-install-pdf-libiberty-linker-plugin:
 @if libiberty-linker-plugin
@@ -30740,6 +31581,32 @@ install-info-libiconv:
 
 @endif libiconv
 
+.PHONY: maybe-install-dvi-libiconv install-dvi-libiconv
+maybe-install-dvi-libiconv:
+@if libiconv
+maybe-install-dvi-libiconv: install-dvi-libiconv
+
+install-dvi-libiconv: \
+    configure-libiconv \
+    dvi-libiconv 
+	@[ -f ./libiconv/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in libiconv"; \
+	(cd $(HOST_SUBDIR)/libiconv && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif libiconv
+
 .PHONY: maybe-install-pdf-libiconv install-pdf-libiconv
 maybe-install-pdf-libiconv:
 @if libiconv
@@ -31142,6 +32009,33 @@ install-info-m4: \
 
 @endif m4
 
+.PHONY: maybe-install-dvi-m4 install-dvi-m4
+maybe-install-dvi-m4:
+@if m4
+maybe-install-dvi-m4: install-dvi-m4
+
+install-dvi-m4: \
+    configure-m4 \
+    dvi-m4 
+	@: $(MAKE); $(unstage)
+	@[ -f ./m4/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in m4"; \
+	(cd $(HOST_SUBDIR)/m4 && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif m4
+
 .PHONY: maybe-install-pdf-m4 install-pdf-m4
 maybe-install-pdf-m4:
 @if m4
@@ -31583,6 +32477,33 @@ install-info-readline: \
 
 @endif readline
 
+.PHONY: maybe-install-dvi-readline install-dvi-readline
+maybe-install-dvi-readline:
+@if readline
+maybe-install-dvi-readline: install-dvi-readline
+
+install-dvi-readline: \
+    configure-readline \
+    dvi-readline 
+	@: $(MAKE); $(unstage)
+	@[ -f ./readline/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in readline"; \
+	(cd $(HOST_SUBDIR)/readline && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif readline
+
 .PHONY: maybe-install-pdf-readline install-pdf-readline
 maybe-install-pdf-readline:
 @if readline
@@ -32024,6 +32945,33 @@ install-info-sid: \
 
 @endif sid
 
+.PHONY: maybe-install-dvi-sid install-dvi-sid
+maybe-install-dvi-sid:
+@if sid
+maybe-install-dvi-sid: install-dvi-sid
+
+install-dvi-sid: \
+    configure-sid \
+    dvi-sid 
+	@: $(MAKE); $(unstage)
+	@[ -f ./sid/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in sid"; \
+	(cd $(HOST_SUBDIR)/sid && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif sid
+
 .PHONY: maybe-install-pdf-sid install-pdf-sid
 maybe-install-pdf-sid:
 @if sid
@@ -32465,6 +33413,33 @@ install-info-sim: \
 
 @endif sim
 
+.PHONY: maybe-install-dvi-sim install-dvi-sim
+maybe-install-dvi-sim:
+@if sim
+maybe-install-dvi-sim: install-dvi-sim
+
+install-dvi-sim: \
+    configure-sim \
+    dvi-sim 
+	@: $(MAKE); $(unstage)
+	@[ -f ./sim/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in sim"; \
+	(cd $(HOST_SUBDIR)/sim && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif sim
+
 .PHONY: maybe-install-pdf-sim install-pdf-sim
 maybe-install-pdf-sim:
 @if sim
@@ -32894,6 +33869,33 @@ install-info-texinfo: \
 
 @endif texinfo
 
+.PHONY: maybe-install-dvi-texinfo install-dvi-texinfo
+maybe-install-dvi-texinfo:
+@if texinfo
+maybe-install-dvi-texinfo: install-dvi-texinfo
+
+install-dvi-texinfo: \
+    configure-texinfo \
+    dvi-texinfo 
+	@: $(MAKE); $(unstage)
+	@[ -f ./texinfo/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in texinfo"; \
+	(cd $(HOST_SUBDIR)/texinfo && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif texinfo
+
 .PHONY: maybe-install-pdf-texinfo install-pdf-texinfo
 maybe-install-pdf-texinfo:
 @if texinfo
@@ -34006,6 +35008,32 @@ install-info-zlib: \
 
 @endif zlib
 
+.PHONY: maybe-install-dvi-zlib install-dvi-zlib
+maybe-install-dvi-zlib:
+@if zlib
+maybe-install-dvi-zlib: install-dvi-zlib
+
+install-dvi-zlib: \
+    configure-zlib \
+    dvi-zlib 
+	@[ -f ./zlib/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in zlib"; \
+	(cd $(HOST_SUBDIR)/zlib && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif zlib
+
 .PHONY: maybe-install-pdf-zlib install-pdf-zlib
 maybe-install-pdf-zlib:
 @if zlib
@@ -34440,6 +35468,33 @@ install-info-gnulib: \
 
 @endif gnulib
 
+.PHONY: maybe-install-dvi-gnulib install-dvi-gnulib
+maybe-install-dvi-gnulib:
+@if gnulib
+maybe-install-dvi-gnulib: install-dvi-gnulib
+
+install-dvi-gnulib: \
+    configure-gnulib \
+    dvi-gnulib 
+	@: $(MAKE); $(unstage)
+	@[ -f ./gnulib/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in gnulib"; \
+	(cd $(HOST_SUBDIR)/gnulib && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif gnulib
+
 .PHONY: maybe-install-pdf-gnulib install-pdf-gnulib
 maybe-install-pdf-gnulib:
 @if gnulib
@@ -34881,6 +35936,33 @@ install-info-gdbsupport: \
 
 @endif gdbsupport
 
+.PHONY: maybe-install-dvi-gdbsupport install-dvi-gdbsupport
+maybe-install-dvi-gdbsupport:
+@if gdbsupport
+maybe-install-dvi-gdbsupport: install-dvi-gdbsupport
+
+install-dvi-gdbsupport: \
+    configure-gdbsupport \
+    dvi-gdbsupport 
+	@: $(MAKE); $(unstage)
+	@[ -f ./gdbsupport/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in gdbsupport"; \
+	(cd $(HOST_SUBDIR)/gdbsupport && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif gdbsupport
+
 .PHONY: maybe-install-pdf-gdbsupport install-pdf-gdbsupport
 maybe-install-pdf-gdbsupport:
 @if gdbsupport
@@ -35322,6 +36404,33 @@ install-info-gdbserver: \
 
 @endif gdbserver
 
+.PHONY: maybe-install-dvi-gdbserver install-dvi-gdbserver
+maybe-install-dvi-gdbserver:
+@if gdbserver
+maybe-install-dvi-gdbserver: install-dvi-gdbserver
+
+install-dvi-gdbserver: \
+    configure-gdbserver \
+    dvi-gdbserver 
+	@: $(MAKE); $(unstage)
+	@[ -f ./gdbserver/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in gdbserver"; \
+	(cd $(HOST_SUBDIR)/gdbserver && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif gdbserver
+
 .PHONY: maybe-install-pdf-gdbserver install-pdf-gdbserver
 maybe-install-pdf-gdbserver:
 @if gdbserver
@@ -35763,6 +36872,33 @@ install-info-gdb: \
 
 @endif gdb
 
+.PHONY: maybe-install-dvi-gdb install-dvi-gdb
+maybe-install-dvi-gdb:
+@if gdb
+maybe-install-dvi-gdb: install-dvi-gdb
+
+install-dvi-gdb: \
+    configure-gdb \
+    dvi-gdb 
+	@: $(MAKE); $(unstage)
+	@[ -f ./gdb/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in gdb"; \
+	(cd $(HOST_SUBDIR)/gdb && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif gdb
+
 .PHONY: maybe-install-pdf-gdb install-pdf-gdb
 maybe-install-pdf-gdb:
 @if gdb
@@ -36204,6 +37340,33 @@ install-info-expect: \
 
 @endif expect
 
+.PHONY: maybe-install-dvi-expect install-dvi-expect
+maybe-install-dvi-expect:
+@if expect
+maybe-install-dvi-expect: install-dvi-expect
+
+install-dvi-expect: \
+    configure-expect \
+    dvi-expect 
+	@: $(MAKE); $(unstage)
+	@[ -f ./expect/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in expect"; \
+	(cd $(HOST_SUBDIR)/expect && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif expect
+
 .PHONY: maybe-install-pdf-expect install-pdf-expect
 maybe-install-pdf-expect:
 @if expect
@@ -36645,6 +37808,33 @@ install-info-guile: \
 
 @endif guile
 
+.PHONY: maybe-install-dvi-guile install-dvi-guile
+maybe-install-dvi-guile:
+@if guile
+maybe-install-dvi-guile: install-dvi-guile
+
+install-dvi-guile: \
+    configure-guile \
+    dvi-guile 
+	@: $(MAKE); $(unstage)
+	@[ -f ./guile/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in guile"; \
+	(cd $(HOST_SUBDIR)/guile && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif guile
+
 .PHONY: maybe-install-pdf-guile install-pdf-guile
 maybe-install-pdf-guile:
 @if guile
@@ -37086,6 +38276,33 @@ install-info-tk: \
 
 @endif tk
 
+.PHONY: maybe-install-dvi-tk install-dvi-tk
+maybe-install-dvi-tk:
+@if tk
+maybe-install-dvi-tk: install-dvi-tk
+
+install-dvi-tk: \
+    configure-tk \
+    dvi-tk 
+	@: $(MAKE); $(unstage)
+	@[ -f ./tk/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in tk"; \
+	(cd $(HOST_SUBDIR)/tk && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif tk
+
 .PHONY: maybe-install-pdf-tk install-pdf-tk
 maybe-install-pdf-tk:
 @if tk
@@ -37521,6 +38738,33 @@ install-info-libtermcap: \
 
 @endif libtermcap
 
+.PHONY: maybe-install-dvi-libtermcap install-dvi-libtermcap
+maybe-install-dvi-libtermcap:
+@if libtermcap
+maybe-install-dvi-libtermcap: install-dvi-libtermcap
+
+install-dvi-libtermcap: \
+    configure-libtermcap \
+    dvi-libtermcap 
+	@: $(MAKE); $(unstage)
+	@[ -f ./libtermcap/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in libtermcap"; \
+	(cd $(HOST_SUBDIR)/libtermcap && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif libtermcap
+
 .PHONY: maybe-install-pdf-libtermcap install-pdf-libtermcap
 maybe-install-pdf-libtermcap:
 @if libtermcap
@@ -37896,6 +39140,33 @@ install-info-utils: \
 
 @endif utils
 
+.PHONY: maybe-install-dvi-utils install-dvi-utils
+maybe-install-dvi-utils:
+@if utils
+maybe-install-dvi-utils: install-dvi-utils
+
+install-dvi-utils: \
+    configure-utils \
+    dvi-utils 
+	@: $(MAKE); $(unstage)
+	@[ -f ./utils/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in utils"; \
+	(cd $(HOST_SUBDIR)/utils && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif utils
+
 .PHONY: maybe-install-pdf-utils install-pdf-utils
 maybe-install-pdf-utils:
 @if utils
@@ -38321,6 +39592,33 @@ install-info-c++tools: \
 
 @endif c++tools
 
+.PHONY: maybe-install-dvi-c++tools install-dvi-c++tools
+maybe-install-dvi-c++tools:
+@if c++tools
+maybe-install-dvi-c++tools: install-dvi-c++tools
+
+install-dvi-c++tools: \
+    configure-c++tools \
+    dvi-c++tools 
+	@: $(MAKE); $(unstage)
+	@[ -f ./c++tools/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in c++tools"; \
+	(cd $(HOST_SUBDIR)/c++tools && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif c++tools
+
 .PHONY: maybe-install-pdf-c++tools install-pdf-c++tools
 maybe-install-pdf-c++tools:
 @if c++tools
@@ -38762,6 +40060,33 @@ install-info-gnattools: \
 
 @endif gnattools
 
+.PHONY: maybe-install-dvi-gnattools install-dvi-gnattools
+maybe-install-dvi-gnattools:
+@if gnattools
+maybe-install-dvi-gnattools: install-dvi-gnattools
+
+install-dvi-gnattools: \
+    configure-gnattools \
+    dvi-gnattools 
+	@: $(MAKE); $(unstage)
+	@[ -f ./gnattools/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in gnattools"; \
+	(cd $(HOST_SUBDIR)/gnattools && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif gnattools
+
 .PHONY: maybe-install-pdf-gnattools install-pdf-gnattools
 maybe-install-pdf-gnattools:
 @if gnattools
@@ -39892,6 +41217,32 @@ install-info-lto-plugin: \
 
 @endif lto-plugin
 
+.PHONY: maybe-install-dvi-lto-plugin install-dvi-lto-plugin
+maybe-install-dvi-lto-plugin:
+@if lto-plugin
+maybe-install-dvi-lto-plugin: install-dvi-lto-plugin
+
+install-dvi-lto-plugin: \
+    configure-lto-plugin \
+    dvi-lto-plugin 
+	@[ -f ./lto-plugin/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) @extra_linker_plugin_flags@; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in lto-plugin"; \
+	(cd $(HOST_SUBDIR)/lto-plugin && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif lto-plugin
+
 .PHONY: maybe-install-pdf-lto-plugin install-pdf-lto-plugin
 maybe-install-pdf-lto-plugin:
 @if lto-plugin
@@ -40326,6 +41677,33 @@ install-info-libcc1: \
 
 @endif libcc1
 
+.PHONY: maybe-install-dvi-libcc1 install-dvi-libcc1
+maybe-install-dvi-libcc1:
+@if libcc1
+maybe-install-dvi-libcc1: install-dvi-libcc1
+
+install-dvi-libcc1: \
+    configure-libcc1 \
+    dvi-libcc1 
+	@: $(MAKE); $(unstage)
+	@[ -f ./libcc1/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in libcc1"; \
+	(cd $(HOST_SUBDIR)/libcc1 && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif libcc1
+
 .PHONY: maybe-install-pdf-libcc1 install-pdf-libcc1
 maybe-install-pdf-libcc1:
 @if libcc1
@@ -40767,6 +42145,33 @@ install-info-gotools: \
 
 @endif gotools
 
+.PHONY: maybe-install-dvi-gotools install-dvi-gotools
+maybe-install-dvi-gotools:
+@if gotools
+maybe-install-dvi-gotools: install-dvi-gotools
+
+install-dvi-gotools: \
+    configure-gotools \
+    dvi-gotools 
+	@: $(MAKE); $(unstage)
+	@[ -f ./gotools/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in gotools"; \
+	(cd $(HOST_SUBDIR)/gotools && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif gotools
+
 .PHONY: maybe-install-pdf-gotools install-pdf-gotools
 maybe-install-pdf-gotools:
 @if gotools
@@ -41888,6 +43293,32 @@ install-info-libctf: \
 
 @endif libctf
 
+.PHONY: maybe-install-dvi-libctf install-dvi-libctf
+maybe-install-dvi-libctf:
+@if libctf
+maybe-install-dvi-libctf: install-dvi-libctf
+
+install-dvi-libctf: \
+    configure-libctf \
+    dvi-libctf 
+	@[ -f ./libctf/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(HOST_EXPORTS) \
+	for flag in $(EXTRA_HOST_FLAGS) ; do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	echo "Doing install-dvi in libctf"; \
+	(cd $(HOST_SUBDIR)/libctf && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	          install-dvi) \
+	  || exit 1
+
+@endif libctf
+
 .PHONY: maybe-install-pdf-libctf install-pdf-libctf
 maybe-install-pdf-libctf:
 @if libctf
@@ -43129,6 +44560,33 @@ install-info-target-libstdc++-v3: \
 
 @endif target-libstdc++-v3
 
+.PHONY: maybe-install-dvi-target-libstdc++-v3 install-dvi-target-libstdc++-v3
+maybe-install-dvi-target-libstdc++-v3:
+@if target-libstdc++-v3
+maybe-install-dvi-target-libstdc++-v3: install-dvi-target-libstdc++-v3
+
+install-dvi-target-libstdc++-v3: \
+    configure-target-libstdc++-v3 \
+    dvi-target-libstdc++-v3 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libstdc++-v3/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(RAW_CXX_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libstdc++-v3"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libstdc++-v3 && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libstdc++-v3
+
 .PHONY: maybe-install-pdf-target-libstdc++-v3 install-pdf-target-libstdc++-v3
 maybe-install-pdf-target-libstdc++-v3:
 @if target-libstdc++-v3
@@ -44372,6 +45830,33 @@ install-info-target-libsanitizer: \
 
 @endif target-libsanitizer
 
+.PHONY: maybe-install-dvi-target-libsanitizer install-dvi-target-libsanitizer
+maybe-install-dvi-target-libsanitizer:
+@if target-libsanitizer
+maybe-install-dvi-target-libsanitizer: install-dvi-target-libsanitizer
+
+install-dvi-target-libsanitizer: \
+    configure-target-libsanitizer \
+    dvi-target-libsanitizer 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libsanitizer/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(RAW_CXX_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libsanitizer"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libsanitizer && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libsanitizer
+
 .PHONY: maybe-install-pdf-target-libsanitizer install-pdf-target-libsanitizer
 maybe-install-pdf-target-libsanitizer:
 @if target-libsanitizer
@@ -45615,6 +47100,33 @@ install-info-target-libvtv: \
 
 @endif target-libvtv
 
+.PHONY: maybe-install-dvi-target-libvtv install-dvi-target-libvtv
+maybe-install-dvi-target-libvtv:
+@if target-libvtv
+maybe-install-dvi-target-libvtv: install-dvi-target-libvtv
+
+install-dvi-target-libvtv: \
+    configure-target-libvtv \
+    dvi-target-libvtv 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libvtv/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(RAW_CXX_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libvtv"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libvtv && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libvtv
+
 .PHONY: maybe-install-pdf-target-libvtv install-pdf-target-libvtv
 maybe-install-pdf-target-libvtv:
 @if target-libvtv
@@ -46073,6 +47585,33 @@ install-info-target-liboffloadmic: \
 
 @endif target-liboffloadmic
 
+.PHONY: maybe-install-dvi-target-liboffloadmic install-dvi-target-liboffloadmic
+maybe-install-dvi-target-liboffloadmic:
+@if target-liboffloadmic
+maybe-install-dvi-target-liboffloadmic: install-dvi-target-liboffloadmic
+
+install-dvi-target-liboffloadmic: \
+    configure-target-liboffloadmic \
+    dvi-target-liboffloadmic 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/liboffloadmic/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/liboffloadmic"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/liboffloadmic && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-liboffloadmic
+
 .PHONY: maybe-install-pdf-target-liboffloadmic install-pdf-target-liboffloadmic
 maybe-install-pdf-target-liboffloadmic:
 @if target-liboffloadmic
@@ -46531,6 +48070,33 @@ install-info-target-libssp: \
 
 @endif target-libssp
 
+.PHONY: maybe-install-dvi-target-libssp install-dvi-target-libssp
+maybe-install-dvi-target-libssp:
+@if target-libssp
+maybe-install-dvi-target-libssp: install-dvi-target-libssp
+
+install-dvi-target-libssp: \
+    configure-target-libssp \
+    dvi-target-libssp 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libssp/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libssp"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libssp && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libssp
+
 .PHONY: maybe-install-pdf-target-libssp install-pdf-target-libssp
 maybe-install-pdf-target-libssp:
 @if target-libssp
@@ -46989,6 +48555,33 @@ install-info-target-newlib: \
 
 @endif target-newlib
 
+.PHONY: maybe-install-dvi-target-newlib install-dvi-target-newlib
+maybe-install-dvi-target-newlib:
+@if target-newlib
+maybe-install-dvi-target-newlib: install-dvi-target-newlib
+
+install-dvi-target-newlib: \
+    configure-target-newlib \
+    dvi-target-newlib 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/newlib/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/newlib"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/newlib && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-newlib
+
 .PHONY: maybe-install-pdf-target-newlib install-pdf-target-newlib
 maybe-install-pdf-target-newlib:
 @if target-newlib
@@ -48211,6 +49804,16 @@ install-info-target-libgcc: \
 
 @endif target-libgcc
 
+.PHONY: maybe-install-dvi-target-libgcc install-dvi-target-libgcc
+maybe-install-dvi-target-libgcc:
+@if target-libgcc
+maybe-install-dvi-target-libgcc: install-dvi-target-libgcc
+
+# libgcc doesn't support install-dvi.
+install-dvi-target-libgcc:
+
+@endif target-libgcc
+
 .PHONY: maybe-install-pdf-target-libgcc install-pdf-target-libgcc
 maybe-install-pdf-target-libgcc:
 @if target-libgcc
@@ -48669,6 +50272,33 @@ install-info-target-libbacktrace: \
 
 @endif target-libbacktrace
 
+.PHONY: maybe-install-dvi-target-libbacktrace install-dvi-target-libbacktrace
+maybe-install-dvi-target-libbacktrace:
+@if target-libbacktrace
+maybe-install-dvi-target-libbacktrace: install-dvi-target-libbacktrace
+
+install-dvi-target-libbacktrace: \
+    configure-target-libbacktrace \
+    dvi-target-libbacktrace 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libbacktrace/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libbacktrace"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libbacktrace && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libbacktrace
+
 .PHONY: maybe-install-pdf-target-libbacktrace install-pdf-target-libbacktrace
 maybe-install-pdf-target-libbacktrace:
 @if target-libbacktrace
@@ -49127,6 +50757,33 @@ install-info-target-libquadmath: \
 
 @endif target-libquadmath
 
+.PHONY: maybe-install-dvi-target-libquadmath install-dvi-target-libquadmath
+maybe-install-dvi-target-libquadmath:
+@if target-libquadmath
+maybe-install-dvi-target-libquadmath: install-dvi-target-libquadmath
+
+install-dvi-target-libquadmath: \
+    configure-target-libquadmath \
+    dvi-target-libquadmath 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libquadmath/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libquadmath"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libquadmath && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libquadmath
+
 .PHONY: maybe-install-pdf-target-libquadmath install-pdf-target-libquadmath
 maybe-install-pdf-target-libquadmath:
 @if target-libquadmath
@@ -49585,6 +51242,33 @@ install-info-target-libgfortran: \
 
 @endif target-libgfortran
 
+.PHONY: maybe-install-dvi-target-libgfortran install-dvi-target-libgfortran
+maybe-install-dvi-target-libgfortran:
+@if target-libgfortran
+maybe-install-dvi-target-libgfortran: install-dvi-target-libgfortran
+
+install-dvi-target-libgfortran: \
+    configure-target-libgfortran \
+    dvi-target-libgfortran 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libgfortran/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libgfortran"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libgfortran && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libgfortran
+
 .PHONY: maybe-install-pdf-target-libgfortran install-pdf-target-libgfortran
 maybe-install-pdf-target-libgfortran:
 @if target-libgfortran
@@ -50027,6 +51711,16 @@ install-info-target-libobjc: \
 
 @endif target-libobjc
 
+.PHONY: maybe-install-dvi-target-libobjc install-dvi-target-libobjc
+maybe-install-dvi-target-libobjc:
+@if target-libobjc
+maybe-install-dvi-target-libobjc: install-dvi-target-libobjc
+
+# libobjc doesn't support install-dvi.
+install-dvi-target-libobjc:
+
+@endif target-libobjc
+
 .PHONY: maybe-install-pdf-target-libobjc install-pdf-target-libobjc
 maybe-install-pdf-target-libobjc:
 @if target-libobjc
@@ -50485,6 +52179,33 @@ install-info-target-libgo: \
 
 @endif target-libgo
 
+.PHONY: maybe-install-dvi-target-libgo install-dvi-target-libgo
+maybe-install-dvi-target-libgo:
+@if target-libgo
+maybe-install-dvi-target-libgo: install-dvi-target-libgo
+
+install-dvi-target-libgo: \
+    configure-target-libgo \
+    dvi-target-libgo 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libgo/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libgo"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libgo && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libgo
+
 .PHONY: maybe-install-pdf-target-libgo install-pdf-target-libgo
 maybe-install-pdf-target-libgo:
 @if target-libgo
@@ -50943,6 +52664,33 @@ install-info-target-libphobos: \
 
 @endif target-libphobos
 
+.PHONY: maybe-install-dvi-target-libphobos install-dvi-target-libphobos
+maybe-install-dvi-target-libphobos:
+@if target-libphobos
+maybe-install-dvi-target-libphobos: install-dvi-target-libphobos
+
+install-dvi-target-libphobos: \
+    configure-target-libphobos \
+    dvi-target-libphobos 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libphobos/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libphobos"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libphobos && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libphobos
+
 .PHONY: maybe-install-pdf-target-libphobos install-pdf-target-libphobos
 maybe-install-pdf-target-libphobos:
 @if target-libphobos
@@ -51396,6 +53144,33 @@ install-info-target-libtermcap: \
 
 @endif target-libtermcap
 
+.PHONY: maybe-install-dvi-target-libtermcap install-dvi-target-libtermcap
+maybe-install-dvi-target-libtermcap:
+@if target-libtermcap
+maybe-install-dvi-target-libtermcap: install-dvi-target-libtermcap
+
+install-dvi-target-libtermcap: \
+    configure-target-libtermcap \
+    dvi-target-libtermcap 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libtermcap/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libtermcap"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libtermcap && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libtermcap
+
 .PHONY: maybe-install-pdf-target-libtermcap install-pdf-target-libtermcap
 maybe-install-pdf-target-libtermcap:
 @if target-libtermcap
@@ -51794,6 +53569,33 @@ install-info-target-winsup: \
 
 @endif target-winsup
 
+.PHONY: maybe-install-dvi-target-winsup install-dvi-target-winsup
+maybe-install-dvi-target-winsup:
+@if target-winsup
+maybe-install-dvi-target-winsup: install-dvi-target-winsup
+
+install-dvi-target-winsup: \
+    configure-target-winsup \
+    dvi-target-winsup 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/winsup/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/winsup"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/winsup && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-winsup
+
 .PHONY: maybe-install-pdf-target-winsup install-pdf-target-winsup
 maybe-install-pdf-target-winsup:
 @if target-winsup
@@ -52247,6 +54049,33 @@ install-info-target-libgloss: \
 
 @endif target-libgloss
 
+.PHONY: maybe-install-dvi-target-libgloss install-dvi-target-libgloss
+maybe-install-dvi-target-libgloss:
+@if target-libgloss
+maybe-install-dvi-target-libgloss: install-dvi-target-libgloss
+
+install-dvi-target-libgloss: \
+    configure-target-libgloss \
+    dvi-target-libgloss 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libgloss/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libgloss"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libgloss && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libgloss
+
 .PHONY: maybe-install-pdf-target-libgloss install-pdf-target-libgloss
 maybe-install-pdf-target-libgloss:
 @if target-libgloss
@@ -52695,6 +54524,33 @@ install-info-target-libffi: \
 
 @endif target-libffi
 
+.PHONY: maybe-install-dvi-target-libffi install-dvi-target-libffi
+maybe-install-dvi-target-libffi:
+@if target-libffi
+maybe-install-dvi-target-libffi: install-dvi-target-libffi
+
+install-dvi-target-libffi: \
+    configure-target-libffi \
+    dvi-target-libffi 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libffi/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libffi"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libffi && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libffi
+
 .PHONY: maybe-install-pdf-target-libffi install-pdf-target-libffi
 maybe-install-pdf-target-libffi:
 @if target-libffi
@@ -53153,6 +55009,33 @@ install-info-target-zlib: \
 
 @endif target-zlib
 
+.PHONY: maybe-install-dvi-target-zlib install-dvi-target-zlib
+maybe-install-dvi-target-zlib:
+@if target-zlib
+maybe-install-dvi-target-zlib: install-dvi-target-zlib
+
+install-dvi-target-zlib: \
+    configure-target-zlib \
+    dvi-target-zlib 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/zlib/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/zlib"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/zlib && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-zlib
+
 .PHONY: maybe-install-pdf-target-zlib install-pdf-target-zlib
 maybe-install-pdf-target-zlib:
 @if target-zlib
@@ -53611,6 +55494,33 @@ install-info-target-rda: \
 
 @endif target-rda
 
+.PHONY: maybe-install-dvi-target-rda install-dvi-target-rda
+maybe-install-dvi-target-rda:
+@if target-rda
+maybe-install-dvi-target-rda: install-dvi-target-rda
+
+install-dvi-target-rda: \
+    configure-target-rda \
+    dvi-target-rda 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/rda/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/rda"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/rda && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-rda
+
 .PHONY: maybe-install-pdf-target-rda install-pdf-target-rda
 maybe-install-pdf-target-rda:
 @if target-rda
@@ -54069,6 +55979,33 @@ install-info-target-libada: \
 
 @endif target-libada
 
+.PHONY: maybe-install-dvi-target-libada install-dvi-target-libada
+maybe-install-dvi-target-libada:
+@if target-libada
+maybe-install-dvi-target-libada: install-dvi-target-libada
+
+install-dvi-target-libada: \
+    configure-target-libada \
+    dvi-target-libada 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libada/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libada"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libada && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libada
+
 .PHONY: maybe-install-pdf-target-libada install-pdf-target-libada
 maybe-install-pdf-target-libada:
 @if target-libada
@@ -55312,6 +57249,33 @@ install-info-target-libgomp: \
 
 @endif target-libgomp
 
+.PHONY: maybe-install-dvi-target-libgomp install-dvi-target-libgomp
+maybe-install-dvi-target-libgomp:
+@if target-libgomp
+maybe-install-dvi-target-libgomp: install-dvi-target-libgomp
+
+install-dvi-target-libgomp: \
+    configure-target-libgomp \
+    dvi-target-libgomp 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libgomp/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libgomp"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libgomp && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libgomp
+
 .PHONY: maybe-install-pdf-target-libgomp install-pdf-target-libgomp
 maybe-install-pdf-target-libgomp:
 @if target-libgomp
@@ -55770,6 +57734,33 @@ install-info-target-libitm: \
 
 @endif target-libitm
 
+.PHONY: maybe-install-dvi-target-libitm install-dvi-target-libitm
+maybe-install-dvi-target-libitm:
+@if target-libitm
+maybe-install-dvi-target-libitm: install-dvi-target-libitm
+
+install-dvi-target-libitm: \
+    configure-target-libitm \
+    dvi-target-libitm 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libitm/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libitm"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libitm && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libitm
+
 .PHONY: maybe-install-pdf-target-libitm install-pdf-target-libitm
 maybe-install-pdf-target-libitm:
 @if target-libitm
@@ -56228,6 +58219,33 @@ install-info-target-libatomic: \
 
 @endif target-libatomic
 
+.PHONY: maybe-install-dvi-target-libatomic install-dvi-target-libatomic
+maybe-install-dvi-target-libatomic:
+@if target-libatomic
+maybe-install-dvi-target-libatomic: install-dvi-target-libatomic
+
+install-dvi-target-libatomic: \
+    configure-target-libatomic \
+    dvi-target-libatomic 
+	@: $(MAKE); $(unstage)
+	@[ -f $(TARGET_SUBDIR)/libatomic/Makefile ] || exit 0; \
+	r=`${PWD_COMMAND}`; export r; \
+	s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+	$(NORMAL_TARGET_EXPORTS) \
+	echo "Doing install-dvi in $(TARGET_SUBDIR)/libatomic"; \
+	for flag in $(EXTRA_TARGET_FLAGS); do \
+	  eval `echo "$$flag" | sed -e "s|^\([^=]*\)=\(.*\)|\1='\2'; export \1|"`; \
+	done; \
+	(cd $(TARGET_SUBDIR)/libatomic && \
+	  $(MAKE) $(BASE_FLAGS_TO_PASS) "AR=$${AR}" "AS=$${AS}" \
+	          "CC=$${CC}" "CXX=$${CXX}" "LD=$${LD}" "NM=$${NM}" \
+	          "RANLIB=$${RANLIB}" \
+	          "DLLTOOL=$${DLLTOOL}" "WINDRES=$${WINDRES}" "WINDMC=$${WINDMC}" \
+	           install-dvi) \
+	  || exit 1
+
+@endif target-libatomic
+
 .PHONY: maybe-install-pdf-target-libatomic install-pdf-target-libatomic
 maybe-install-pdf-target-libatomic:
 @if target-libatomic
@@ -60699,6 +62717,16 @@ install-ld: maybe-install-libctf
 install-strip-libctf: maybe-install-strip-bfd
 install-strip-ld: maybe-install-strip-bfd
 install-strip-ld: maybe-install-strip-libctf
+configure-opcodes: configure-bfd
+configure-stage1-opcodes: configure-stage1-bfd
+configure-stage2-opcodes: configure-stage2-bfd
+configure-stage3-opcodes: configure-stage3-bfd
+configure-stage4-opcodes: configure-stage4-bfd
+configure-stageprofile-opcodes: configure-stageprofile-bfd
+configure-stagetrain-opcodes: configure-stagetrain-bfd
+configure-stagefeedback-opcodes: configure-stagefeedback-bfd
+configure-stageautoprofile-opcodes: configure-stageautoprofile-bfd
+configure-stageautofeedback-opcodes: configure-stageautofeedback-bfd
 install-opcodes: maybe-install-bfd
 install-strip-opcodes: maybe-install-strip-bfd
 configure-gas: maybe-configure-intl
diff --git a/Makefile.tpl b/Makefile.tpl
index 9adf4f9..213052f 100644
--- a/Makefile.tpl
+++ b/Makefile.tpl
@@ -157,6 +157,8 @@ BUILD_EXPORTS = \
 	CC="$(CC_FOR_BUILD)"; export CC; \
 	CFLAGS="$(CFLAGS_FOR_BUILD)"; export CFLAGS; \
 	CONFIG_SHELL="$(SHELL)"; export CONFIG_SHELL; \
+	CPP="$(CPP_FOR_BUILD)"; export CPP; \
+	CPPFLAGS="$(CPPFLAGS_FOR_BUILD)"; export CPPFLAGS; \
 	CXX="$(CXX_FOR_BUILD)"; export CXX; \
 	CXXFLAGS="$(CXXFLAGS_FOR_BUILD)"; export CXXFLAGS; \
 	GFORTRAN="$(GFORTRAN_FOR_BUILD)"; export GFORTRAN; \
@@ -205,6 +207,8 @@ HOST_EXPORTS = \
 	AR="$(AR)"; export AR; \
 	AS="$(AS)"; export AS; \
 	CC_FOR_BUILD="$(CC_FOR_BUILD)"; export CC_FOR_BUILD; \
+	CPP_FOR_BUILD="$(CPP_FOR_BUILD)"; export CPP_FOR_BUILD; \
+	CPPFLAGS_FOR_BUILD="$(CPPFLAGS_FOR_BUILD)"; export CPPFLAGS_FOR_BUILD; \
 	CXX_FOR_BUILD="$(CXX_FOR_BUILD)"; export CXX_FOR_BUILD; \
 	DLLTOOL="$(DLLTOOL)"; export DLLTOOL; \
 	DSYMUTIL="$(DSYMUTIL)"; export DSYMUTIL; \
@@ -363,6 +367,8 @@ AR_FOR_BUILD = @AR_FOR_BUILD@
 AS_FOR_BUILD = @AS_FOR_BUILD@
 CC_FOR_BUILD = @CC_FOR_BUILD@
 CFLAGS_FOR_BUILD = @CFLAGS_FOR_BUILD@
+CPP_FOR_BUILD = @CPP_FOR_BUILD@
+CPPFLAGS_FOR_BUILD = @CPPFLAGS_FOR_BUILD@
 CXXFLAGS_FOR_BUILD = @CXXFLAGS_FOR_BUILD@
 CXX_FOR_BUILD = @CXX_FOR_BUILD@
 DLLTOOL_FOR_BUILD = @DLLTOOL_FOR_BUILD@
@@ -836,7 +842,7 @@ do-[+make_target+]:
 # Here are the targets which correspond to the do-X targets.
 
 .PHONY: info installcheck dvi pdf html
-.PHONY: install-info install-pdf install-html
+.PHONY: install-info install-dvi install-pdf install-html
 .PHONY: clean distclean mostlyclean maintainer-clean realclean
 .PHONY: local-clean local-distclean local-maintainer-clean
 info: do-info
@@ -855,6 +861,8 @@ install-info: do-install-info dir.info
 	  $(INSTALL_DATA) dir.info $(DESTDIR)$(infodir)/dir.info; \
 	else true; fi
 
+install-dvi: do-install-dvi
+
 install-pdf: do-install-pdf
 
 install-html: do-install-html
diff --git a/c++tools/ChangeLog b/c++tools/ChangeLog
index fe606e5..322b31b 100644
--- a/c++tools/ChangeLog
+++ b/c++tools/ChangeLog
@@ -1,3 +1,13 @@
+2021-10-26  Jonathan Wakely  <jwakely@redhat.com>
+
+	* resolver.cc (module_resolver::read_tuple_file): Use unique_ptr
+	to ensure memory is freed before returning.
+
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Makefile.in: Add dummy install-dvi target.
+
 2021-09-14  Iain Sandoe  <iain@sandoe.co.uk>
 
 	* resolver.cc (module_resolver::ModuleCompiledRequest):
diff --git a/c++tools/Makefile.in b/c++tools/Makefile.in
index 83a6ae5..824840a 100644
--- a/c++tools/Makefile.in
+++ b/c++tools/Makefile.in
@@ -59,6 +59,7 @@ html::
 info::
 install-info::
 install-pdf::
+install-dvi::
 install-man::
 install-html::
 
diff --git a/c++tools/resolver.cc b/c++tools/resolver.cc
index 421fdaa..a1837b3 100644
--- a/c++tools/resolver.cc
+++ b/c++tools/resolver.cc
@@ -23,6 +23,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "resolver.h"
 // C++
 #include <algorithm>
+#include <memory>
 // C
 #include <cstring>
 // OS
@@ -114,10 +115,17 @@ module_resolver::read_tuple_file (int fd, char const *prefix, bool force)
   buffer = mmap (nullptr, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
   if (buffer == MAP_FAILED)
     return -errno;
+  struct Deleter {
+    void operator()(void* p) const { munmap(p, size); }
+    size_t size;
+  };
+  std::unique_ptr<void, Deleter> guard(buffer, Deleter{(size_t)stat.st_size});
 #else
   buffer = xmalloc (stat.st_size);
   if (!buffer)
     return -errno;
+  struct Deleter { void operator()(void* p) const { free(p); } };
+  std::unique_ptr<void, Deleter> guard(buffer);
   if (read (fd, buffer, stat.st_size) != stat.st_size)
     return -errno;
 #endif
@@ -179,12 +187,6 @@ module_resolver::read_tuple_file (int fd, char const *prefix, bool force)
 	}
     }
 
-#if MAPPED_READING
-  munmap (buffer, stat.st_size);
-#else
-  free (buffer);
-#endif
-
   return 0;
 }
 
diff --git a/configure b/configure
index 785498e..58979d6 100755
--- a/configure
+++ b/configure
@@ -655,6 +655,8 @@ DSYMUTIL_FOR_BUILD
 DLLTOOL_FOR_BUILD
 CXX_FOR_BUILD
 CXXFLAGS_FOR_BUILD
+CPPFLAGS_FOR_BUILD
+CPP_FOR_BUILD
 CFLAGS_FOR_BUILD
 CC_FOR_BUILD
 AS_FOR_BUILD
@@ -4090,6 +4092,7 @@ if test "${build}" != "${host}" ; then
   AR_FOR_BUILD=${AR_FOR_BUILD-ar}
   AS_FOR_BUILD=${AS_FOR_BUILD-as}
   CC_FOR_BUILD=${CC_FOR_BUILD-gcc}
+  CPP_FOR_BUILD="${CPP_FOR_BUILD-\$(CPP)}"
   CXX_FOR_BUILD=${CXX_FOR_BUILD-g++}
   DSYMUTIL_FOR_BUILD=${DSYMUTIL_FOR_BUILD-dsymutil}
   GFORTRAN_FOR_BUILD=${GFORTRAN_FOR_BUILD-gfortran}
@@ -9999,6 +10002,7 @@ esac
 # our build compiler if desired.
 if test x"${build}" = x"${host}" ; then
   CFLAGS_FOR_BUILD=${CFLAGS_FOR_BUILD-${CFLAGS}}
+  CPPFLAGS_FOR_BUILD=${CPPFLAGS_FOR_BUILD-${CPPFLAGS}}
   CXXFLAGS_FOR_BUILD=${CXXFLAGS_FOR_BUILD-${CXXFLAGS}}
   LDFLAGS_FOR_BUILD=${LDFLAGS_FOR_BUILD-${LDFLAGS}}
 fi
diff --git a/configure.ac b/configure.ac
index c523083..550e699 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1334,6 +1334,7 @@ if test "${build}" != "${host}" ; then
   AR_FOR_BUILD=${AR_FOR_BUILD-ar}
   AS_FOR_BUILD=${AS_FOR_BUILD-as}
   CC_FOR_BUILD=${CC_FOR_BUILD-gcc}
+  CPP_FOR_BUILD="${CPP_FOR_BUILD-\$(CPP)}"
   CXX_FOR_BUILD=${CXX_FOR_BUILD-g++}
   DSYMUTIL_FOR_BUILD=${DSYMUTIL_FOR_BUILD-dsymutil}
   GFORTRAN_FOR_BUILD=${GFORTRAN_FOR_BUILD-gfortran}
@@ -3323,6 +3324,7 @@ esac
 # our build compiler if desired.
 if test x"${build}" = x"${host}" ; then
   CFLAGS_FOR_BUILD=${CFLAGS_FOR_BUILD-${CFLAGS}}
+  CPPFLAGS_FOR_BUILD=${CPPFLAGS_FOR_BUILD-${CPPFLAGS}}
   CXXFLAGS_FOR_BUILD=${CXXFLAGS_FOR_BUILD-${CXXFLAGS}}
   LDFLAGS_FOR_BUILD=${LDFLAGS_FOR_BUILD-${LDFLAGS}}
 fi
@@ -3389,6 +3391,8 @@ AC_SUBST(AR_FOR_BUILD)
 AC_SUBST(AS_FOR_BUILD)
 AC_SUBST(CC_FOR_BUILD)
 AC_SUBST(CFLAGS_FOR_BUILD)
+AC_SUBST(CPP_FOR_BUILD)
+AC_SUBST(CPPFLAGS_FOR_BUILD)
 AC_SUBST(CXXFLAGS_FOR_BUILD)
 AC_SUBST(CXX_FOR_BUILD)
 AC_SUBST(DLLTOOL_FOR_BUILD)
diff --git a/contrib/ChangeLog b/contrib/ChangeLog
index d36e6f1..e4b9ad0 100644
--- a/contrib/ChangeLog
+++ b/contrib/ChangeLog
@@ -1,3 +1,25 @@
+2021-11-08  Thomas Schwinge  <thomas@codesourcery.com>
+
+	* update-copyright.py (class BadYear): Derive from 'Exception'.
+
+2021-11-05  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* testsuite-management/validate_failures.py: 2to3
+
+2021-11-04  Martin Liska  <mliska@suse.cz>
+
+	* gcc-changelog/git_check_commit.py: Add -v option.
+	* gcc-changelog/git_commit.py: Print verbose diff for wrong
+	filename.
+
+2021-11-02  Martin Liska  <mliska@suse.cz>
+
+	* check-internal-format-escaping.py: Fix flake8 errors.
+
+2021-11-01  David Malcolm  <dmalcolm@redhat.com>
+
+	* unicode/utf8-dump.py: New file.
+
 2021-10-20  H.J. Lu  <hjl.tools@gmail.com>
 
 	* gcc-changelog/git_commit.py (ignored_prefixes): Add libffi/.
diff --git a/contrib/check-internal-format-escaping.py b/contrib/check-internal-format-escaping.py
index e067526..8bd54d8 100755
--- a/contrib/check-internal-format-escaping.py
+++ b/contrib/check-internal-format-escaping.py
@@ -66,7 +66,7 @@ def lint_gcc_internal_format(msg: polib.POEntry):
 
     def outside_quotes(m: Match[str]):
         before = msgid[:m.start(0)]
-        return before.count("%<") == before.count("%>")
+        return before.count('%<') == before.count('%>')
 
     def lint_matching_placeholders():
         """
@@ -127,7 +127,7 @@ def lint_gcc_internal_format(msg: polib.POEntry):
         to separate the words.
         """
 
-        for match in re.finditer("(.?[a-zA-Z0-9])%<", msgid):
+        for match in re.finditer('(.?[a-zA-Z0-9])%<', msgid):
             if match.group(1) != '%s':
                 warn(msg,
                      'no-space-before-quote',
@@ -143,7 +143,7 @@ def lint_gcc_internal_format(msg: polib.POEntry):
         * static_cast and the like (which are legitimate)
         """
 
-        for match in re.finditer("_", msgid):
+        for match in re.finditer('_', msgid):
             if outside_quotes(match):
                 warn(msg,
                      'underscore-outside-quotes',
@@ -163,13 +163,13 @@ def lint_gcc_internal_format(msg: polib.POEntry):
                  'the term "may not" is ambiguous')
 
     def lint_unbalanced_quotes():
-        if msgid.count("%<") != msgid.count("%>"):
+        if msgid.count('%<') != msgid.count('%>'):
             warn(msg,
                  'unbalanced-quotes',
                  'unbalanced %< and %> quotes')
 
         if msg.translated():
-            if msg.msgstr.count("%<") != msg.msgstr.count("%>"):
+            if msg.msgstr.count('%<') != msg.msgstr.count('%>'):
                 warn(msg,
                      'unbalanced-quotes',
                      'unbalanced %< and %> quotes')
diff --git a/contrib/gcc-changelog/git_check_commit.py b/contrib/gcc-changelog/git_check_commit.py
index 9a4c5d4..d6aff3c 100755
--- a/contrib/gcc-changelog/git_check_commit.py
+++ b/contrib/gcc-changelog/git_check_commit.py
@@ -29,6 +29,8 @@ parser.add_argument('-g', '--git-path', default='.',
                     help='Path to git repository')
 parser.add_argument('-p', '--print-changelog', action='store_true',
                     help='Print final changelog entires')
+parser.add_argument('-v', '--verbose', action='store_true',
+                    help='Print verbose information')
 args = parser.parse_args()
 
 retval = 0
@@ -41,6 +43,8 @@ for git_commit in parse_git_revisions(args.git_path, args.revisions):
     else:
         for error in git_commit.errors:
             print('ERR: %s' % error)
+            if args.verbose and error.details:
+                print(error.details)
         retval = 1
 
 exit(retval)
diff --git a/contrib/gcc-changelog/git_commit.py b/contrib/gcc-changelog/git_commit.py
index 60377b6..27a1d59 100755
--- a/contrib/gcc-changelog/git_commit.py
+++ b/contrib/gcc-changelog/git_commit.py
@@ -197,9 +197,10 @@ def decode_path(path):
 
 
 class Error:
-    def __init__(self, message, line=None):
+    def __init__(self, message, line=None, details=None):
         self.message = message
         self.line = line
+        self.details = details
 
     def __repr__(self):
         s = self.message
@@ -687,9 +688,11 @@ class GitCommit:
         for file in sorted(mentioned_files - changed_files):
             msg = 'unchanged file mentioned in a ChangeLog'
             candidates = difflib.get_close_matches(file, changed_files, 1)
+            details = None
             if candidates:
                 msg += f' (did you mean "{candidates[0]}"?)'
-            self.errors.append(Error(msg, file))
+                details = '\n'.join(difflib.Differ().compare([file], [candidates[0]])).rstrip()
+            self.errors.append(Error(msg, file, details))
         for file in sorted(changed_files - mentioned_files):
             if not self.in_ignored_location(file):
                 if file in self.new_files:
diff --git a/contrib/testsuite-management/validate_failures.py b/contrib/testsuite-management/validate_failures.py
index 2779050..925ba22 100755
--- a/contrib/testsuite-management/validate_failures.py
+++ b/contrib/testsuite-management/validate_failures.py
@@ -1,11 +1,7 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 # Script to compare testsuite failures against a list of known-to-fail
 # tests.
-#
-# NOTE: This script is used in installations that are running Python 2.4.
-#       Please stick to syntax features available in 2.4 and earlier
-#       versions.
 
 # Contributed by Diego Novillo <dnovillo@google.com>
 #
@@ -82,7 +78,7 @@ _MANIFEST_PATH_PATTERN = '%s/%s/%s.xfail'
 _OPTIONS = None
 
 def Error(msg):
-  print >>sys.stderr, 'error: %s' % msg
+  print('error: %s' % msg, file=sys.stderr)
   sys.exit(1)
 
 
@@ -126,7 +122,7 @@ class TestResult(object):
          self.description) = re.match(r'([A-Z]+):\s*(\S+)\s*(.*)',
                                       summary_line).groups()
       except:
-        print 'Failed to parse summary line: "%s"' % summary_line
+        print('Failed to parse summary line: "%s"' % summary_line)
         raise
       self.ordinal = ordinal
     except ValueError:
@@ -180,7 +176,7 @@ class TestResult(object):
 
 def GetMakefileValue(makefile_name, value_name):
   if os.path.exists(makefile_name):
-    makefile = open(makefile_name)
+    makefile = open(makefile_name, encoding='latin-1', mode='r')
     for line in makefile:
       if line.startswith(value_name):
         (_, value) = line.split('=', 1)
@@ -246,8 +242,8 @@ def GetNegativeResult(line):
 def ParseManifestWorker(result_set, manifest_path):
   """Read manifest_path, adding the contents to result_set."""
   if _OPTIONS.verbosity >= 1:
-    print 'Parsing manifest file %s.' % manifest_path
-  manifest_file = open(manifest_path)
+    print('Parsing manifest file %s.' % manifest_path)
+  manifest_file = open(manifest_path, encoding='latin-1', mode='r')
   for line in manifest_file:
     line = line.strip()
     if line == "":
@@ -278,7 +274,7 @@ def ParseSummary(sum_fname):
   # ordinal is used when sorting the results so that tests within each
   # .exp file are kept sorted.
   ordinal=0
-  sum_file = open(sum_fname)
+  sum_file = open(sum_fname, encoding='latin-1', mode='r')
   for line in sum_file:
     if IsInterestingResult(line):
       result = TestResult(line, ordinal)
@@ -287,7 +283,7 @@ def ParseSummary(sum_fname):
         # Tests that have expired are not added to the set of expected
         # results. If they are still present in the set of actual results,
         # they will cause an error to be reported.
-        print 'WARNING: Expected failure "%s" has expired.' % line.strip()
+        print('WARNING: Expected failure "%s" has expired.' % line.strip())
         continue
       result_set.add(result)
   sum_file.close()
@@ -324,7 +320,7 @@ def GetResults(sum_files):
   """Collect all the test results from the given .sum files."""
   build_results = set()
   for sum_fname in sum_files:
-    print '\t%s' % sum_fname
+    print('\t%s' % sum_fname)
     build_results |= ParseSummary(sum_fname)
   return build_results
 
@@ -387,23 +383,23 @@ def GetBuildData():
       return None, None
   srcdir = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'srcdir =')
   target = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'target_alias=')
-  print 'Source directory: %s' % srcdir
-  print 'Build target:     %s' % target
+  print('Source directory: %s' % srcdir)
+  print('Build target:     %s' % target)
   return srcdir, target
 
 
 def PrintSummary(msg, summary):
-  print '\n\n%s' % msg
+  print('\n\n%s' % msg)
   for result in sorted(summary):
-    print result
+    print(result)
 
 
 def GetSumFiles(results, build_dir):
   if not results:
-    print 'Getting actual results from build directory %s' % build_dir
+    print('Getting actual results from build directory %s' % build_dir)
     sum_files = CollectSumFiles(build_dir)
   else:
-    print 'Getting actual results from user-provided results'
+    print('Getting actual results from user-provided results')
     sum_files = results.split()
   return sum_files
 
@@ -425,7 +421,7 @@ def PerformComparison(expected, actual, ignore_missing_failures):
                  expected_vs_actual)
 
   if tests_ok:
-    print '\nSUCCESS: No unexpected failures.'
+    print('\nSUCCESS: No unexpected failures.')
 
   return tests_ok
 
@@ -433,7 +429,7 @@ def PerformComparison(expected, actual, ignore_missing_failures):
 def CheckExpectedResults():
   srcdir, target = GetBuildData()
   manifest_path = GetManifestPath(srcdir, target, True)
-  print 'Manifest:         %s' % manifest_path
+  print('Manifest:         %s' % manifest_path)
   manifest = GetManifest(manifest_path)
   sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
   actual = GetResults(sum_files)
@@ -448,16 +444,16 @@ def CheckExpectedResults():
 def ProduceManifest():
   (srcdir, target) = GetBuildData()
   manifest_path = GetManifestPath(srcdir, target, False)
-  print 'Manifest:         %s' % manifest_path
+  print('Manifest:         %s' % manifest_path)
   if os.path.exists(manifest_path) and not _OPTIONS.force:
     Error('Manifest file %s already exists.\nUse --force to overwrite.' %
           manifest_path)
 
   sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir)
   actual = GetResults(sum_files)
-  manifest_file = open(manifest_path, 'w')
+  manifest_file = open(manifest_path, encoding='latin-1', mode='w')
   for result in sorted(actual):
-    print result
+    print(result)
     manifest_file.write('%s\n' % result)
   manifest_file.close()
 
diff --git a/contrib/unicode/utf8-dump.py b/contrib/unicode/utf8-dump.py
new file mode 100755
index 0000000..f12ee79
--- /dev/null
+++ b/contrib/unicode/utf8-dump.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+#
+# Script to dump a UTF-8 file as a list of numbered lines (mimicking GCC's
+# diagnostic output format), interleaved with lines per character showing
+# the Unicode codepoints, the UTF-8 encoding bytes, the name of the
+# character, and, where printable, the characters themselves.
+# The lines are printed in logical order, which may help the reader to grok
+# the relationship between visual and logical ordering in bi-di files.
+#
+# SPDX-License-Identifier: MIT
+#
+# Copyright (C) 2021 David Malcolm <dmalcolm@redhat.com>.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
+# OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
+# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import sys
+import unicodedata
+
+
+def get_name(ch):
+    try:
+        return unicodedata.name(ch)
+    except ValueError:
+        if ch == '\n':
+            return 'LINE FEED (LF)'
+        return '(unknown)'
+
+
+def get_printable(ch):
+    cat = unicodedata.category(ch)
+    if cat == 'Cc':
+        return '(control character)'
+    elif cat == 'Cf':
+        return '(format control)'
+    elif cat[0] == 'Z':
+        return '(separator)'
+    return ch
+
+
+def dump_file(f_in):
+    line_num = 1
+    for line in f_in:
+        print('%4i | %s' % (line_num, line.rstrip()))
+        for ch in line:
+            utf8_desc = '%15s' % (' '.join(['0x%02x' % b
+                                            for b in ch.encode('utf-8')]))
+            print('%4s |   U+%04X %s %40s %s'
+                  % ('', ord(ch), utf8_desc, get_name(ch), get_printable(ch)))
+        line_num += 1
+
+
+with open(sys.argv[1], mode='r') as f_in:
+    dump_file(f_in)
diff --git a/contrib/update-copyright.py b/contrib/update-copyright.py
index 2b2bb11..d13b963 100755
--- a/contrib/update-copyright.py
+++ b/contrib/update-copyright.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (C) 2013-2020 Free Software Foundation, Inc.
+# Copyright (C) 2013-2021 Free Software Foundation, Inc.
 #
 # This script is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -233,7 +233,7 @@ class Copyright:
     def add_external_author (self, holder):
         self.holders[holder] = None
 
-    class BadYear():
+    class BadYear (Exception):
         def __init__ (self, year):
             self.year = year
 
diff --git a/fixincludes/ChangeLog b/fixincludes/ChangeLog
index b57d9f1..dfc6ee7 100644
--- a/fixincludes/ChangeLog
+++ b/fixincludes/ChangeLog
@@ -1,3 +1,11 @@
+2021-11-13  Xi Ruoyao  <xry111@mengyan1223.wang>
+
+	PR other/21823
+	PR bootstrap/80047
+	* fixincl.c (process): Simplify the handling for highly
+	  unlikely access() failure, to avoid using non-standard
+	  extensions.
+
 2021-08-30  John David Anglin  <danglin@gcc.gnu.org>
 
 	* inclhack.def (hpux_c99_inttypes5): New hack to define PRIdPTR, etc.
diff --git a/fixincludes/fixincl.c b/fixincludes/fixincl.c
index 6dba2f6..a17b658 100644
--- a/fixincludes/fixincl.c
+++ b/fixincludes/fixincl.c
@@ -1352,11 +1352,10 @@ process (void)
 
   if (access (pz_curr_file, R_OK) != 0)
     {
-      int erno = errno;
-      fprintf (stderr, "Cannot access %s from %s\n\terror %d (%s)\n",
-               pz_curr_file, getcwd ((char *) NULL, MAXPATHLEN),
-               erno, xstrerror (erno));
-      return;
+      /* Some really strange error happened.  */
+      fprintf (stderr, "Cannot access %s: %s\n", pz_curr_file,
+	       xstrerror (errno));
+      abort ();
     }
 
   pz_curr_data = load_file (pz_curr_file);
diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 115f32e..52de288 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,5008 @@
+2021-11-15  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	* gimple-fold.c (gimple_fold_builtin_strncat): Use ranges to
+	determine if it is safe to transform to strcat.
+	(gimple_fold_builtin_snprintf): Likewise.
+
+2021-11-15  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	* gimple-fold.c (known_lower): New function.
+	(gimple_fold_builtin_strncat_chk,
+	gimple_fold_builtin_memory_chk, gimple_fold_builtin_stxcpy_chk,
+	gimple_fold_builtin_stxncpy_chk,
+	gimple_fold_builtin_snprintf_chk,
+	gimple_fold_builtin_sprintf_chk): Use it.
+
+2021-11-15  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	* gimple-fold.c (dump_transformation): New function.
+	(gimple_fold_builtin_stxcpy_chk,
+	gimple_fold_builtin_stxncpy_chk): Use it.  Simplify to
+	BUILT_IN_STRNCPY if return value is not used.
+
+2021-11-15  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR middle-end/103184
+	* tree-ssa-ccp.c (optimize_atomic_bit_test_and): Check optab
+	before transforming equivalent, but slighly different cases to
+	their canonical forms.
+
+2021-11-15  Iain Sandoe  <iain@sandoe.co.uk>
+
+	PR fortran/102992
+	* config/darwin.h (TARGET_DTORS_FROM_CXA_ATEXIT): New.
+	* doc/tm.texi: Regenerated.
+	* doc/tm.texi.in: Add TARGET_DTORS_FROM_CXA_ATEXIT hook.
+	* ipa.c (cgraph_build_static_cdtor_1): Return the built
+	function decl.
+	(build_cxa_atexit_decl): New.
+	(build_dso_handle_decl): New.
+	(build_cxa_dtor_registrations): New.
+	(compare_cdtor_tu_order): New.
+	(build_cxa_atexit_fns): New.
+	(ipa_cdtor_merge): If dtors_from_cxa_atexit is set,
+	process the DTORs/CTORs accordingly.
+	(pass_ipa_cdtor_merge::gate): Also run if
+	dtors_from_cxa_atexit is set.
+	* target.def (dtors_from_cxa_atexit): New hook.
+
+2021-11-15  Iain Sandoe  <iain@sandoe.co.uk>
+
+	* config.in: Regenerate.
+	* configure: Regenerate.
+	* configure.ac: Test ld64 for -platform-version support.
+
+2021-11-15  Thomas Schwinge  <thomas@codesourcery.com>
+
+	* diagnostic-spec.h (typedef xint_hash_t)
+	(typedef xint_hash_map_t): Replace with...
+	(typedef nowarn_map_t): ... this.
+	(nowarn_map): Adjust.
+	* diagnostic-spec.c (nowarn_map, suppress_warning_at): Likewise.
+
+2021-11-15  Thomas Schwinge  <thomas@codesourcery.com>
+
+	* profile.c (branch_prob): Use 'location_hash' for
+	'seen_locations'.
+
+2021-11-15  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/103207
+	* value-range.cc (irange::set): Drop overflow.
+
+2021-11-15  Jakub Jelinek  <jakub@redhat.com>
+
+	* gimplify.c (optimize_target_teams): Only add OMP_CLAUSE_THREAD_LIMIT
+	to OMP_TARGET_CLAUSES if it isn't there already.
+
+2021-11-15  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gimple-range-path.cc (path_range_query::internal_range_of_expr):
+	Remove useless code.
+	(path_range_query::ssa_defined_in_bb): New.
+	(path_range_query::ssa_range_in_phi): Avoid fold_range call that
+	could trigger additional lookups.
+	Do not use the cache for ARGs defined in this block.
+	(path_range_query::compute_ranges_in_block): Use ssa_defined_in_bb.
+	(path_range_query::maybe_register_phi_relation): Same.
+	(path_range_query::range_of_stmt): Adjust comment.
+	* gimple-range-path.h (ssa_defined_in_bb): New.
+
+2021-11-15  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gimple-range-path.cc (path_range_query::internal_range_of_expr):
+	Default to global range if nothing found.
+
+2021-11-15  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/103237
+	* tree-vect-loop.c (vect_is_simple_reduction): Fail for
+	double reductions with multiple inner loop LC PHI nodes.
+
+2021-11-15  Hongyu Wang  <hongyu.wang@intel.com>
+
+	PR target/103069
+	* config/i386/i386-expand.c (ix86_expand_atomic_fetch_op_loop):
+	New expand function.
+	* config/i386/i386-options.c (ix86_target_string): Add
+	-mrelax-cmpxchg-loop flag.
+	(ix86_valid_target_attribute_inner_p): Likewise.
+	* config/i386/i386-protos.h (ix86_expand_atomic_fetch_op_loop):
+	New expand function prototype.
+	* config/i386/i386.opt: Add -mrelax-cmpxchg-loop.
+	* config/i386/sync.md (atomic_fetch_<logic><mode>): New expander
+	for SI,HI,QI modes.
+	(atomic_<logic>_fetch<mode>): Likewise.
+	(atomic_fetch_nand<mode>): Likewise.
+	(atomic_nand_fetch<mode>): Likewise.
+	(atomic_fetch_<logic><mode>): New expander for DI,TI modes.
+	(atomic_<logic>_fetch<mode>): Likewise.
+	(atomic_fetch_nand<mode>): Likewise.
+	(atomic_nand_fetch<mode>): Likewise.
+	* doc/invoke.texi: Document -mrelax-cmpxchg-loop.
+
+2021-11-15  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/103219
+	* gimple-loop-jam.c (tree_loop_unroll_and_jam): Use single_exit
+	to determine the exit for the VN region.
+
+2021-11-15  Prathamesh Kulkarni  <prathamesh.kulkarni@linaro.org>
+
+	* tree-ssa-loop.c (pass_vectorize): Move to tree-vectorizer.c.
+	(pass_data_vectorize): Likewise.
+	(make_pass_vectorize): Likewise.
+	* tree-vectorizer.c (vectorize_loops): Merge with
+	pass_vectorize::execute and replace cfun occurences with fun param.
+	(adjust_simduid_builtins): Add fun param, replace cfun occurences with
+	fun, and adjust callers approrpiately.
+	(note_simd_array_uses): Likewise.
+	(vect_loop_dist_alias_call): Likewise.
+	(set_uid_loop_bbs): Likewise.
+	(vect_transform_loops): Likewise.
+	(try_vectorize_loop_1): Likewise.
+	(try_vectorize_loop): Likewise.
+
+2021-11-15  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/103205
+	* config/i386/sync.md (atomic_bit_test_and_set<mode>,
+	atomic_bit_test_and_complement<mode>,
+	atomic_bit_test_and_reset<mode>): Use OPTAB_WIDEN instead of
+	OPTAB_DIRECT.
+
+2021-11-15  Maciej W. Rozycki  <macro@embecosm.com>
+
+	* config/vax/vax.c (vax_lra_p): New prototype and function.
+	(TARGET_LRA_P): Wire it.
+	* config/vax/vax.opt (mlra): New option.
+	* doc/invoke.texi (Option Summary, VAX Options): Document the
+	new option.
+
+2021-11-14  Jan Hubicka  <jh@suse.cz>
+
+	* ipa-modref.h (struct modref_summary): Add nondeterministic
+	and calls_interposable flags.
+	* ipa-modref.c (modref_summary::modref_summary): Initialize new flags.
+	(modref_summary::useful_p): Check new flags.
+	(struct modref_summary_lto): Add nondeterministic and
+	calls_interposable flags.
+	(modref_summary_lto::modref_summary_lto): Initialize new flags.
+	(modref_summary_lto::useful_p): Check new flags.
+	(modref_summary::dump): Dump new flags.
+	(modref_summary_lto::dump): Dump new flags.
+	(ignore_nondeterminism_p): New function.
+	(merge_call_side_effects): Merge new flags.
+	(process_fnspec): Likewise.
+	(analyze_load): Volatile access is nondeterministic.
+	(analyze_store): Liekwise.
+	(analyze_stmt): Volatile ASM is nondeterministic.
+	(analyze_function): Clear new flags.
+	(modref_summaries::duplicate): Duplicate new flags.
+	(modref_summaries_lto::duplicate): Duplicate new flags.
+	(modref_write): Stream new flags.
+	(read_section): Stream new flags.
+	(propagate_unknown_call): Update new flags.
+	(modref_propagate_in_scc): Propagate new flags.
+	* tree-ssa-alias.c (ref_maybe_used_by_call_p_1): Check
+	calls_interposable.
+	* tree-ssa-structalias.c (determine_global_memory_access):
+	Likewise.
+
+2021-11-14  Maciej W. Rozycki  <macro@embecosm.com>
+
+	* config/vax/vax.h (SET_RATIO): New macro.
+	* config/vax/vax.md (UNSPEC_SETMEM_FILL): New constant.
+	(setmemhi): New expander.
+	(setmemhi1): New insn and splitter.
+	(*setmemhi1): New insn.
+
+2021-11-14  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref-tree.c (modref_access_node::update_for_kills): New
+	member function.
+	(modref_access_node::merge_for_kills): Likewise.
+	(modref_access_node::insert_kill): Likewise.
+	* ipa-modref-tree.h (modref_access_node::update_for_kills,
+	modref_access_node::merge_for_kills, modref_access_node::insert_kill):
+	Declare.
+	(modref_access_node::useful_for_kill): New member function.
+	* ipa-modref.c (modref_summary::useful_p): Release useless kills.
+	(lto_modref_summary): Add kills.
+	(modref_summary::dump): Dump kills.
+	(record_access): Add mdoref_access_node parameter.
+	(record_access_lto): Likewise.
+	(merge_call_side_effects): Merge kills.
+	(analyze_call): Add ALWAYS_EXECUTED param and pass it around.
+	(struct summary_ptrs): Add always_executed filed.
+	(analyze_load): Update.
+	(analyze_store): Update; record kills.
+	(analyze_stmt): Add always_executed; record kills in clobbers.
+	(analyze_function): Track always_executed.
+	(modref_summaries::duplicate): Duplicate kills.
+	(update_signature): Release kills.
+	* ipa-modref.h (struct modref_summary): Add kills.
+	* tree-ssa-alias.c (alias_stats): Add kill stats.
+	(dump_alias_stats): Dump kill stats.
+	(store_kills_ref_p): Break out from ...
+	(stmt_kills_ref_p): Use it; handle modref info based kills.
+
+2021-11-14  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/103229
+	* gimple-range-cache.cc (ssa_global_cache::clear): Do not pass
+	null value to memset.
+
+2021-11-14  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref-tree.c (modref_access_node::get_call_arg): New member
+	function.
+	(modref_access_node::get_ao_ref): Likewise.
+	* ipa-modref-tree.h (modref_access_node::get_call_arg): Declare.
+	(modref_access_node::get_ao_ref): Declare.
+	* tree-ssa-alias.c (modref_may_conflict): Use new accessors.
+	* tree-ssa-dse.c (dse_optimize_call): Use new accessors.
+
+2021-11-13  Jan Hubicka  <jh@suse.cz>
+
+	PR lto/103211
+	* dbgcnt.def (ipa_attr): New counters.
+	* ipa-pure-const.c: Include dbgcnt.c
+	(ipa_make_function_const): Use debug counter.
+	(ipa_make_function_pure): Likewise.
+	(propagate_pure_const): Fix bug in my previous change.
+
+2021-11-13  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref-tree.c (modref_access_node::range_info_useful_p):
+	Offline from ipa-modref-tree.h.
+	(modref_access_node::dump): Move from ipa-modref.c; make member
+	function.
+	* ipa-modref-tree.h (modref_access_node::range_info_useful_p.
+	modref_access_node::dump): Declare.
+	* ipa-modref.c (dump_access): Remove.
+	(dump_records): Update.
+	(dump_lto_records): Update.
+	(record_access): Update.
+	(record_access_lto): Update.
+
+2021-11-13  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref.c (modref_summary::modref_summary): Clear new flags.
+	(modref_summary::dump): Dump try_dse.
+	(modref_summary::finalize): Add FUN attribute; compute try-dse.
+	(analyze_function): Update.
+	(read_section): Update.
+	(update_signature): Update.
+	(pass_ipa_modref::execute): Update.
+	* ipa-modref.h (struct modref_summary):
+	* tree-ssa-alias.c (ao_ref_init_from_ptr_and_range): Export.
+	* tree-ssa-alias.h (ao_ref_init_from_ptr_and_range): Declare.
+	* tree-ssa-dse.c (dse_optimize_call): New function.
+	(dse_optimize_stmt): Use it.
+
+2021-11-13  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref-tree.c: Move #if CHECKING_P to proper place.
+
+2021-11-13  Jan Hubicka  <jh@suse.cz>
+
+	* ipa-modref-tree.h
+	(struct modref_access_node): Move longer member functions to
+	ipa-modref-tree.c
+	(modref_ref_node::try_merge_with): Turn into modreef_acces_node member
+	function.
+	* ipa-modref-tree.c (modref_access_node::contains): Move here
+	from ipa-modref-tree.h.
+	(modref_access_node::update): Likewise.
+	(modref_access_node::merge): Likewise.
+	(modref_access_node::closer_pair_p): Likewise.
+	(modref_access_node::forced_merge): Likewise.
+	(modref_access_node::update2): Likewise.
+	(modref_access_node::combined_offsets): Likewise.
+	(modref_access_node::try_merge_with): Likewise.
+	(modref_access_node::insert): Likewise.
+
+2021-11-13  Jan Hubicka  <jh@suse.cz>
+
+	* ipa-modref.c (modref_summary::global_memory_read_p): Remove.
+	(modref_summary::global_memory_written_p): Remove.
+	(modref_summary::dump): Dump new flags.
+	(modref_summary::finalize): New member function.
+	(analyze_function): Call it.
+	(read_section): Call it.
+	(update_signature): Call it.
+	(pass_ipa_modref::execute): Call it.
+	* ipa-modref.h (struct modref_summary): Remove
+	global_memory_read_p and global_memory_written_p.
+	Add global_memory_read, global_memory_written.
+	* tree-ssa-structalias.c (determine_global_memory_access):
+	Update.
+
+2021-11-13  Jan Hubicka  <jh@suse.cz>
+
+	* ipa-fnsummary.c (compute_fn_summary): Use type_attribut_allowed_p
+	* ipa-param-manipulation.c
+	(ipa_param_adjustments::type_attribute_allowed_p):
+	New member function.
+	(drop_type_attribute_if_params_changed_p): New function.
+	(build_adjusted_function_type): Use it.
+	* ipa-param-manipulation.h: Add type_attribute_allowed_p.
+
+2021-11-13  David Malcolm  <dmalcolm@redhat.com>
+
+	* doc/invoke.texi (Static Analyzer Options): Add
+	-Wno-analyzer-tainted-allocation-size,
+	-Wno-analyzer-tainted-divisor, -Wno-analyzer-tainted-offset, and
+	-Wno-analyzer-tainted-size to list.  Add
+	-Wanalyzer-tainted-allocation-size, -Wanalyzer-tainted-divisor,
+	-Wanalyzer-tainted-offset, and -Wanalyzer-tainted-size to list
+	of options effectively enabled by -fanalyzer.
+	(-Wanalyzer-tainted-allocation-size): New.
+	(-Wanalyzer-tainted-array-index): Tweak wording; add link to CWE.
+	(-Wanalyzer-tainted-divisor): New.
+	(-Wanalyzer-tainted-offset): New.
+	(-Wanalyzer-tainted-size): New.
+
+2021-11-13  Jan Hubicka  <jh@suse.cz>
+
+	* attr-fnspec.h (attr_fnspec::arg_eaf_flags): Break out from ...
+	* gimple.c (gimple_call_arg_flags): ... here.
+	* ipa-modref.c (analyze_parms): Record flags known from fnspec.
+	(modref_merge_call_site_flags): Use arg_eaf_flags.
+
+2021-11-13  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/103222
+	* gimple-range-path.cc (path_range_query::compute_ranges_in_phis):
+	New.
+	(path_range_query::compute_ranges_in_block): Call
+	compute_ranges_in_phis.
+	* gimple-range-path.h (path_range_query::compute_ranges_in_phis):
+	New.
+
+2021-11-13  Jan Hubicka  <jh@suse.cz>
+
+	* ipa-fnsummary.c (compute_fn_summary): Do not give up on signature
+	changes on "fn spec" attribute; give up on varadic types.
+	* ipa-param-manipulation.c: Include attribs.h.
+	(build_adjusted_function_type): New parameter ARG_MODIFIED; if it is
+	true remove "fn spec" attribute.
+	(ipa_param_adjustments::build_new_function_type): Update.
+	(ipa_param_body_adjustments::modify_formal_parameters): update.
+	* ipa-sra.c: Include attribs.h.
+	(ipa_sra_preliminary_function_checks): Do not check for TYPE_ATTRIBUTES.
+
+2021-11-13  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gimple-range-path.cc (path_range_query::path_range_query): Merge
+	ctors.
+	(path_range_query::import_p): Move from header file.
+	(path_range_query::~path_range_query): Adjust for combined ctors.
+	* gimple-range-path.h: Merge ctors.
+	(path_range_query::import_p): Move to .cc file.
+
+2021-11-13  Jan Hubicka  <jh@suse.cz>
+
+	PR tree-optimization/103182
+	* ipa-modref.c (callee_to_caller_flags): Fix merging of flags.
+	(modref_eaf_analysis::analyze_ssa_name): Fix merging of flags.
+
+2021-11-12  Stafford Horne  <shorne@gmail.com>
+
+	* config/or1k/or1k-protos.h (or1k_profile_hook): New function.
+	* config/or1k/or1k.h (PROFILE_HOOK): Change macro to reference
+	new function or1k_profile_hook.
+	* config/or1k/or1k.c (struct machine_function): Add new field
+	set_mcount_arg_insn.
+	(or1k_profile_hook): New function.
+	(or1k_init_pic_reg): Update to inject pic rtx after _mcount arg
+	when profiling.
+	(or1k_frame_pointer_required): Frame pointer no longer needed
+	when profiling.
+
+2021-11-12  Jan Hubicka  <jh@suse.cz>
+
+	PR tree-optimization/103209
+	* tree-ssa-structalias.c (find_func_aliases_for_call): Fix
+	use of handle_rhs_call
+
+2021-11-12  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/103202
+	* gimple-range-path.cc
+	(path_range_query::compute_ranges_in_block): Solve PHI imports first.
+
+2021-11-12  Jan Hubicka  <jh@suse.cz>
+
+	* ipa-pure-const.c (propagate_pure_const): Remove redundant check;
+	fix call of ipa_make_function_const and ipa_make_function_pure.
+
+2021-11-12  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_vector_op::n_advsimd_ops): Delete.
+	(aarch64_vector_op::m_seen_loads): Likewise.
+	(aarch64_vector_costs::aarch64_vector_costs): Don't push to
+	m_advsimd_ops.
+	(aarch64_vector_op::count_ops): Remove vectype and factor parameters.
+	Remove code that tries to predict different vec_flags from the
+	current loop's.
+	(aarch64_vector_costs::add_stmt_cost): Update accordingly.
+	Remove m_advsimd_ops handling.
+
+2021-11-12  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_vector_costs::m_saw_sve_only_op)
+	(aarch64_sve_only_stmt_p): Delete.
+	(aarch64_vector_costs::prefer_unrolled_loop): New function,
+	extracted from adjust_body_cost.
+	(aarch64_vector_costs::better_main_loop_than_p): New function,
+	using heuristics extracted from adjust_body_cost and
+	adjust_body_cost_sve.
+	(aarch64_vector_costs::adjust_body_cost_sve): Remove
+	advsimd_cycles_per_iter and could_use_advsimd parameters.
+	Update after changes above.
+	(aarch64_vector_costs::adjust_body_cost): Update after changes above.
+
+2021-11-12  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_vec_op_count::m_vf_factor):
+	New member variable.
+	(aarch64_vec_op_count::aarch64_vec_op_count): Add a parameter for it.
+	(aarch64_vec_op_count::vf_factor): New function.
+	(aarch64_vector_costs::aarch64_vector_costs): When costing for
+	neoverse-512tvb, pass a vf_factor of 2 for the Neoverse V1 version
+	of an SVE loop.
+	(aarch64_vector_costs::adjust_body_cost): Read the vf factor
+	instead of hard-coding 2.
+
+2021-11-12  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c
+	(aarch64_vec_op_count::rename_cycles_per_iter): New function.
+	(aarch64_vec_op_count::min_nonpred_cycles_per_iter): Likewise.
+	(aarch64_vec_op_count::min_pred_cycles_per_iter): Likewise.
+	(aarch64_vec_op_count::min_cycles_per_iter): Likewise.
+	(aarch64_vec_op_count::dump): Move earlier in file.  Dump the
+	above properties too.
+	(aarch64_estimate_min_cycles_per_iter): Delete.
+	(adjust_body_cost): Use aarch64_vec_op_count::min_cycles_per_iter
+	instead of aarch64_estimate_min_cycles_per_iter.  Rely on the dump
+	routine to print CPI estimates.
+	(adjust_body_cost_sve): Likewise.  Use the other functions above
+	instead of doing the work inline.
+
+2021-11-12  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_vec_op_count): Allow default
+	initialization.
+	(aarch64_vec_op_count::base_issue_info): Remove handling of null
+	issue_infos.
+	(aarch64_vec_op_count::simd_issue_info): Likewise.
+	(aarch64_vec_op_count::sve_issue_info): Likewise.
+	(aarch64_vector_costs::m_ops): Turn into a vector.
+	(aarch64_vector_costs::m_advsimd_ops): Likewise.
+	(aarch64_vector_costs::aarch64_vector_costs): Add entries to
+	the vectors based on aarch64_tune_params.
+	(aarch64_vector_costs::analyze_loop_vinfo): Update the pred_ops
+	of all entries in m_ops.
+	(aarch64_vector_costs::add_stmt_cost): Call count_ops for all
+	entries in m_ops.
+	(aarch64_estimate_min_cycles_per_iter): Remove issue_info
+	parameter and get the information from the ops instead.
+	(aarch64_vector_costs::adjust_body_cost_sve): Take a
+	aarch64_vec_issue_info instead of a aarch64_vec_op_count.
+	(aarch64_vector_costs::adjust_body_cost): Update call accordingly.
+	Exit earlier if m_ops is empty for either cost structure.
+
+2021-11-12  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_vector_costs::m_scalar_ops)
+	(aarch64_vector_costs::m_sve_ops): Replace with...
+	(aarch64_vector_costs::m_ops): ...this.
+	(aarch64_vector_costs::analyze_loop_vinfo): Update accordingly.
+	(aarch64_vector_costs::adjust_body_cost_sve): Likewise.
+	(aarch64_vector_costs::aarch64_vector_costs): Likewise.
+	Initialize m_vec_flags here rather than in add_stmt_cost.
+	(aarch64_vector_costs::count_ops): Test for scalar reductions too.
+	Allow vectype to be null.
+	(aarch64_vector_costs::add_stmt_cost): Call count_ops for scalar
+	code too.  Don't require vectype to be nonnull.
+	(aarch64_vector_costs::adjust_body_cost): Take the loop_vec_info
+	and scalar costs as parameters.  Use the scalar costs to determine
+	the cycles per iteration of the scalar loop, then multiply it
+	by the estimated VF.
+	(aarch64_vector_costs::finish_cost): Update call accordingly.
+
+2021-11-12  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_dr_type): New function.
+	(aarch64_vector_costs::count_ops): Use it rather than the
+	vectype to determine floatness.
+
+2021-11-12  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_sve_in_loop_reduction_latency):
+	Remove vectype parameter and get floatness from the type of the
+	stmt lhs instead.
+	(arch64_in_loop_reduction_latency): Likewise.
+	(aarch64_detect_vector_stmt_subtype): Update caller.
+	(aarch64_vector_costs::count_ops): Likewise.
+
+2021-11-12  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_sve_op_count): Fold into...
+	(aarch64_vec_op_count): ...this.  Add a constructor.
+	(aarch64_vec_op_count::vec_flags): New function.
+	(aarch64_vec_op_count::base_issue_info): Likewise.
+	(aarch64_vec_op_count::simd_issue_info): Likewise.
+	(aarch64_vec_op_count::sve_issue_info): Likewise.
+	(aarch64_vec_op_count::m_issue_info): New member variable.
+	(aarch64_vec_op_count::m_vec_flags): Likewise.
+	(aarch64_vector_costs): Add a constructor.
+	(aarch64_vector_costs::m_sve_ops): Change type to aarch64_vec_op_count.
+	(aarch64_vector_costs::aarch64_vector_costs): New function.
+	Initialize m_scalar_ops, m_advsimd_ops and m_sve_ops.
+	(aarch64_vector_costs::count_ops): Remove vec_flags and
+	issue_info parameters, using the new aarch64_vec_op_count
+	functions instead.
+	(aarch64_vector_costs::add_stmt_cost): Update call accordingly.
+	(aarch64_sve_op_count::dump): Fold into...
+	(aarch64_vec_op_count::dump): ..here.
+
+2021-11-12  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c: Include tree-dfa.h.
+	(aarch64_check_consecutive_mems): New function that takes MEM_EXPR
+	and MEM_OFFSET into account.
+	(aarch64_swap_ldrstr_operands): Use it.
+	(aarch64_operands_ok_for_ldpstp): Likewise.  Check that the
+	address of the second memory doesn't depend on the result of
+	the first load.
+
+2021-11-12  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-pure-const.c (ipa_make_function_pure): Fix exit condition.
+
+2021-11-12  Jan Hubicka  <jh@suse.cz>
+
+	PR tree-optimization/103175
+	* ipa-modref.c (modref_lattice::merge): Add sanity check.
+	(callee_to_caller_flags): Make flags adjustment sane.
+	(modref_eaf_analysis::analyze_ssa_name): Likewise.
+
+2021-11-12  Jan Hubicka  <jh@suse.cz>
+
+	PR ipa/103200
+	* ipa-modref.c (analyze_function, modref_propagate_in_scc): Do
+	not mark pure/const function if there are side-effects.
+
+2021-11-12  Chung-Lin Tang  <cltang@codesourcery.com>
+
+	* tree.h (OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P): New access macro for
+	'implicit' bit, using 'base.deprecated_flag' field of tree_node.
+	* tree-pretty-print.c (dump_omp_clause): Add support for printing
+	implicit attribute in tree dumping.
+	* gimplify.c (gimplify_adjust_omp_clauses_1):
+	Set OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P to 1 if map clause is implicitly
+	created.
+	(gimplify_adjust_omp_clauses): Adjust place of adding implicitly created
+	clauses, from simple append, to starting of list, after non-map clauses.
+	* omp-low.c (lower_omp_target): Add GOMP_MAP_IMPLICIT bits into kind
+	values passed to libgomp for implicit maps.
+
+2021-11-12  Jakub Jelinek  <jakub@redhat.com>
+
+	* omp-builtins.def (BUILT_IN_GOMP_TEAMS): Remove.
+	(BUILT_IN_GOMP_TEAMS4): New.
+	* builtin-types.def (BT_FN_VOID_UINT_UINT): Remove.
+	(BT_FN_BOOL_UINT_UINT_UINT_BOOL): New.
+	* omp-low.c (lower_omp_teams): Use GOMP_teams4 instead of
+	GOMP_teams, pass to it also num_teams lower-bound expression
+	or a dup of upper-bound if it is missing and a flag whether
+	it is the first call or not.
+
+2021-11-12  Martin Liska  <mliska@suse.cz>
+
+	PR tree-optimization/102497
+	* gimple-predicate-analysis.cc (add_pred): Remove unused
+	function:
+
+2021-11-12  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/103204
+	* tree-ssa-sccvn.c (valueize_refs_1): Re-valueize the
+	top operand after folding in an address.
+
+2021-11-11  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gimple-range-path.cc (path_range_query::path_range_query): New
+	ctor without a ranger.
+	(path_range_query::~path_range_query): Free ranger if necessary.
+	(path_range_query::range_on_path_entry): Adjust m_ranger for pointer.
+	(path_range_query::ssa_range_in_phi): Same.
+	(path_range_query::compute_ranges_in_block): Same.
+	(path_range_query::compute_imports): Same.
+	(path_range_query::compute_ranges): Same.
+	(path_range_query::range_of_stmt): Same.
+	(path_range_query::compute_outgoing_relations): Same.
+	* gimple-range-path.h (class path_range_query): New ctor.
+	* tree-ssa-loop-ch.c (ch_base::copy_headers): Remove gimple_ranger
+	as path_range_query allocates one.
+	* tree-ssa-threadbackward.c (class back_threader): Remove m_ranger.
+	(back_threader::~back_threader): Same.
+
+2021-11-11  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c
+	(back_threader_profitability::profitable_path_p): Remove loop
+	crossing restriction.
+
+2021-11-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref.c (analyze_function): Do pure/const discovery, return
+	true on success.
+	(pass_modref::execute): If pure/const is discovered fixup cfg.
+	(ignore_edge): Do not ignore pure/const edges.
+	(modref_propagate_in_scc): Do pure/const discovery, return true if
+	cdtor was promoted pure/const.
+	(pass_ipa_modref::execute): If needed remove unreachable functions.
+	* ipa-pure-const.c (warn_function_noreturn): Fix whitespace.
+	(warn_function_cold): Likewise.
+	(skip_function_for_local_pure_const): Move earlier.
+	(ipa_make_function_const): Break out from ...
+	(ipa_make_function_pure): Break out from ...
+	(propagate_pure_const): ... here.
+	(pass_local_pure_const::execute): Use it.
+	* ipa-utils.h (ipa_make_function_const): Declare.
+	(ipa_make_function_pure): Declare.
+	* passes.def: Move early modref after pure-const.
+
+2021-11-11  David Malcolm  <dmalcolm@redhat.com>
+
+	PR other/103129
+	* diagnostic-show-locus.c (def_policy): Use def_tabstop.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-builtins.c (TYPES_COMBINE): Delete.
+	(TYPES_COMBINEP): Delete.
+	* config/aarch64/aarch64-simd-builtins.def: Declare type-
+	qualified builtins for vcombine_* intrinsics.
+	* config/aarch64/arm_neon.h (vcombine_s8): Remove unnecessary
+	cast.
+	(vcombine_s16): Likewise.
+	(vcombine_s32): Likewise.
+	(vcombine_f32): Likewise.
+	(vcombine_u8): Use type-qualified builtin and remove casts.
+	(vcombine_u16): Likewise.
+	(vcombine_u32): Likewise.
+	(vcombine_u64): Likewise.
+	(vcombine_p8): Likewise.
+	(vcombine_p16): Likewise.
+	(vcombine_p64): Likewise.
+	(vcombine_bf16): Remove unnecessary cast.
+	* config/aarch64/iterators.md (VD_I): New mode iterator.
+	(VDC_P): New mode iterator.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-builtins.c (TYPES_LOAD1_U): Define.
+	(TYPES_LOAD1_P): Define.
+	(TYPES_STORE1_U): Define.
+	(TYPES_STORE1P): Rename to...
+	(TYPES_STORE1_P): This.
+	(get_mem_type_for_load_store): Add unsigned and poly types.
+	(aarch64_general_gimple_fold_builtin): Add unsigned and poly
+	type-qualified builtin declarations.
+	* config/aarch64/aarch64-simd-builtins.def: Declare type-
+	qualified builtins for LD1/ST1.
+	* config/aarch64/arm_neon.h (vld1_p8): Use type-qualified
+	builtin and remove cast.
+	(vld1_p16): Likewise.
+	(vld1_u8): Likewise.
+	(vld1_u16): Likewise.
+	(vld1_u32): Likewise.
+	(vld1q_p8): Likewise.
+	(vld1q_p16): Likewise.
+	(vld1q_p64): Likewise.
+	(vld1q_u8): Likewise.
+	(vld1q_u16): Likewise.
+	(vld1q_u32): Likewise.
+	(vld1q_u64): Likewise.
+	(vst1_p8): Likewise.
+	(vst1_p16): Likewise.
+	(vst1_u8): Likewise.
+	(vst1_u16): Likewise.
+	(vst1_u32): Likewise.
+	(vst1q_p8): Likewise.
+	(vst1q_p16): Likewise.
+	(vst1q_p64): Likewise.
+	(vst1q_u8): Likewise.
+	(vst1q_u16): Likewise.
+	(vst1q_u32): Likewise.
+	(vst1q_u64): Likewise.
+	* config/aarch64/iterators.md (VALLP_NO_DI): New iterator.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def: Declare unsigned
+	builtins for vector reduction.
+	* config/aarch64/arm_neon.h (vaddv_u8): Use type-qualified
+	builtin and remove casts.
+	(vaddv_u16): Likewise.
+	(vaddv_u32): Likewise.
+	(vaddvq_u8): Likewise.
+	(vaddvq_u16): Likewise.
+	(vaddvq_u32): Likewise.
+	(vaddvq_u64): Likewise.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def:
+	* config/aarch64/arm_neon.h (vpaddq_u8): Use type-qualified
+	builtin and remove casts.
+	(vpaddq_u16): Likewise.
+	(vpaddq_u32): Likewise.
+	(vpaddq_u64): Likewise.
+	(vpadd_u8): Likewise.
+	(vpadd_u16): Likewise.
+	(vpadd_u32): Likewise.
+	(vpaddd_u64): Likewise.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def: Declare unsigned
+	builtins for [r]subhn[2].
+	* config/aarch64/arm_neon.h (vsubhn_s16): Remove unnecessary
+	cast.
+	(vsubhn_s32): Likewise.
+	(vsubhn_s64): Likewise.
+	(vsubhn_u16): Use type-qualified builtin and remove casts.
+	(vsubhn_u32): Likewise.
+	(vsubhn_u64): Likewise.
+	(vrsubhn_s16): Remove unnecessary cast.
+	(vrsubhn_s32): Likewise.
+	(vrsubhn_s64): Likewise.
+	(vrsubhn_u16): Use type-qualified builtin and remove casts.
+	(vrsubhn_u32): Likewise.
+	(vrsubhn_u64): Likewise.
+	(vrsubhn_high_s16): Remove unnecessary cast.
+	(vrsubhn_high_s32): Likewise.
+	(vrsubhn_high_s64): Likewise.
+	(vrsubhn_high_u16): Use type-qualified builtin and remove
+	casts.
+	(vrsubhn_high_u32): Likewise.
+	(vrsubhn_high_u64): Likewise.
+	(vsubhn_high_s16): Remove unnecessary cast.
+	(vsubhn_high_s32): Likewise.
+	(vsubhn_high_s64): Likewise.
+	(vsubhn_high_u16): Use type-qualified builtin and remove
+	casts.
+	(vsubhn_high_u32): Likewise.
+	(vsubhn_high_u64): Likewise.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def: Declare unsigned
+	builtins for [r]addhn[2].
+	* config/aarch64/arm_neon.h (vaddhn_s16): Remove unnecessary
+	cast.
+	(vaddhn_s32): Likewise.
+	(vaddhn_s64): Likewise.
+	(vaddhn_u16): Use type-qualified builtin and remove casts.
+	(vaddhn_u32): Likewise.
+	(vaddhn_u64): Likewise.
+	(vraddhn_s16): Remove unnecessary cast.
+	(vraddhn_s32): Likewise.
+	(vraddhn_s64): Likewise.
+	(vraddhn_u16): Use type-qualified builtin and remove casts.
+	(vraddhn_u32): Likewise.
+	(vraddhn_u64): Likewise.
+	(vaddhn_high_s16): Remove unnecessary cast.
+	(vaddhn_high_s32): Likewise.
+	(vaddhn_high_s64): Likewise.
+	(vaddhn_high_u16): Use type-qualified builtin and remove
+	casts.
+	(vaddhn_high_u32): Likewise.
+	(vaddhn_high_u64): Likewise.
+	(vraddhn_high_s16): Remove unnecessary cast.
+	(vraddhn_high_s32): Likewise.
+	(vraddhn_high_s64): Likewise.
+	(vraddhn_high_u16): Use type-qualified builtin and remove
+	casts.
+	(vraddhn_high_u32): Likewise.
+	(vraddhn_high_u64): Likewise.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def: Use BINOPU type
+	qualifiers in generator macros for uhsub builtins.
+	* config/aarch64/arm_neon.h (vhsub_s8): Remove unnecessary
+	cast.
+	(vhsub_s16): Likewise.
+	(vhsub_s32): Likewise.
+	(vhsub_u8): Use type-qualified builtin and remove casts.
+	(vhsub_u16): Likewise.
+	(vhsub_u32): Likewise.
+	(vhsubq_s8): Remove unnecessary cast.
+	(vhsubq_s16): Likewise.
+	(vhsubq_s32): Likewise.
+	(vhsubq_u8): Use type-qualified builtin and remove casts.
+	(vhsubq_u16): Likewise.
+	(vhsubq_u32): Likewise.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def: Use BINOPU type
+	qualifiers in generator macros for u[r]hadd builtins.
+	* config/aarch64/arm_neon.h (vhadd_s8): Remove unnecessary
+	cast.
+	(vhadd_s16): Likewise.
+	(vhadd_s32): Likewise.
+	(vhadd_u8): Use type-qualified builtin and remove casts.
+	(vhadd_u16): Likewise.
+	(vhadd_u32): Likewise.
+	(vhaddq_s8): Remove unnecessary cast.
+	(vhaddq_s16): Likewise.
+	(vhaddq_s32): Likewise.
+	(vhaddq_u8): Use type-qualified builtin and remove casts.
+	(vhaddq_u16): Likewise.
+	(vhaddq_u32): Likewise.
+	(vrhadd_s8): Remove unnecessary cast.
+	(vrhadd_s16): Likewise.
+	(vrhadd_s32): Likewise.
+	(vrhadd_u8): Use type-qualified builtin and remove casts.
+	(vrhadd_u16): Likewise.
+	(vrhadd_u32): Likewise.
+	(vrhaddq_s8): Remove unnecessary cast.
+	(vrhaddq_s16): Likewise.
+	(vrhaddq_s32): Likewise.
+	(vrhaddq_u8): Use type-wualified builtin and remove casts.
+	(vrhaddq_u16): Likewise.
+	(vrhaddq_u32): Likewise.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def: Use BINOPU type
+	qualifiers in generator macros for usub[lw][2] builtins.
+	* config/aarch64/arm_neon.h (vsubl_s8): Remove unnecessary
+	cast.
+	(vsubl_s16): Likewise.
+	(vsubl_s32): Likewise.
+	(vsubl_u8): Use type-qualified builtin and remove casts.
+	(vsubl_u16): Likewise.
+	(vsubl_u32): Likewise.
+	(vsubl_high_s8): Remove unnecessary cast.
+	(vsubl_high_s16): Likewise.
+	(vsubl_high_s32): Likewise.
+	(vsubl_high_u8): Use type-qualified builtin and remove casts.
+	(vsubl_high_u16): Likewise.
+	(vsubl_high_u32): Likewise.
+	(vsubw_s8): Remove unnecessary casts.
+	(vsubw_s16): Likewise.
+	(vsubw_s32): Likewise.
+	(vsubw_u8): Use type-qualified builtin and remove casts.
+	(vsubw_u16): Likewise.
+	(vsubw_u32): Likewise.
+	(vsubw_high_s8): Remove unnecessary cast.
+	(vsubw_high_s16): Likewise.
+	(vsubw_high_s32): Likewise.
+	(vsubw_high_u8): Use type-qualified builtin and remove casts.
+	(vsubw_high_u16): Likewise.
+	(vsubw_high_u32): Likewise.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def: Use BINOPU type
+	qualifiers in generator macros for uadd[lw][2] builtins.
+	* config/aarch64/arm_neon.h (vaddl_s8): Remove unnecessary
+	cast.
+	(vaddl_s16): Likewise.
+	(vaddl_s32): Likewise.
+	(vaddl_u8): Use type-qualified builtin and remove casts.
+	(vaddl_u16): Likewise.
+	(vaddl_u32): Likewise.
+	(vaddl_high_s8): Remove unnecessary cast.
+	(vaddl_high_s16): Likewise.
+	(vaddl_high_s32): Likewise.
+	(vaddl_high_u8): Use type-qualified builtin and remove casts.
+	(vaddl_high_u16): Likewise.
+	(vaddl_high_u32): Likewise.
+	(vaddw_s8): Remove unnecessary cast.
+	(vaddw_s16): Likewise.
+	(vaddw_s32): Likewise.
+	(vaddw_u8): Use type-qualified builtin and remove casts.
+	(vaddw_u16): Likewise.
+	(vaddw_u32): Likewise.
+	(vaddw_high_s8): Remove unnecessary cast.
+	(vaddw_high_s16): Likewise.
+	(vaddw_high_s32): Likewise.
+	(vaddw_high_u8): Use type-qualified builtin and remove casts.
+	(vaddw_high_u16): Likewise.
+	(vaddw_high_u32): Likewise.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def: Declare type-
+	qualified builtins for [R]SHRN[2].
+	* config/aarch64/arm_neon.h (vshrn_n_u16): Use type-qualified
+	builtin and remove casts.
+	(vshrn_n_u32): Likewise.
+	(vshrn_n_u64): Likewise.
+	(vrshrn_high_n_u16): Likewise.
+	(vrshrn_high_n_u32): Likewise.
+	(vrshrn_high_n_u64): Likewise.
+	(vrshrn_n_u16): Likewise.
+	(vrshrn_n_u32): Likewise.
+	(vrshrn_n_u64): Likewise.
+	(vshrn_high_n_u16): Likewise.
+	(vshrn_high_n_u32): Likewise.
+	(vshrn_high_n_u64): Likewise.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def: Declare unsigned
+	type-qualified builtins for XTN[2].
+	* config/aarch64/arm_neon.h (vmovn_high_u16): Use type-
+	qualified builtin and remove casts.
+	(vmovn_high_u32): Likewise.
+	(vmovn_high_u64): Likewise.
+	(vmovn_u16): Likewise.
+	(vmovn_u32): Likewise.
+	(vmovn_u64): Likewise.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def: Use poly type
+	qualifier in builtin generator macros.
+	* config/aarch64/arm_neon.h (vmul_p8): Use type-qualified
+	builtin and remove casts.
+	(vmulq_p8): Likewise.
+	(vmull_high_p8): Likewise.
+	(vmull_p8): Likewise.
+
+2021-11-11  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-simd-builtins.def: Declare type-
+	qualified builtin generators for unsigned MLA/MLS intrinsics.
+	* config/aarch64/arm_neon.h (vmla_n_u16): Use type-qualified
+	builtin.
+	(vmla_n_u32): Likewise.
+	(vmla_u8): Likewise.
+	(vmla_u16): Likewise.
+	(vmla_u32): Likewise.
+	(vmlaq_n_u16): Likewise.
+	(vmlaq_n_u32): Likewise.
+	(vmlaq_u8): Likewise.
+	(vmlaq_u16): Likewise.
+	(vmlaq_u32): Likewise.
+	(vmls_n_u16): Likewise.
+	(vmls_n_u32): Likewise.
+	(vmls_u8): Likewise.
+	(vmls_u16): Likewise.
+	(vmls_u32): Likewise.
+	(vmlsq_n_u16): Likewise.
+	(vmlsq_n_u32): Likewise.
+	(vmlsq_u8): Likewise.
+	(vmlsq_u16): Likewise.
+	(vmlsq_u32): Likewise.
+
+2021-11-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref.c (modref_summary::useful_p): Check also for side-effects
+	with looping const/pure.
+	(modref_summary_lto::useful_p): Likewise.
+	(merge_call_side_effects): Merge side effects before early exit
+	for pure/const.
+	(process_fnspec): Also handle pure functions.
+	(analyze_call): Do not early exit on looping pure const.
+	(propagate_unknown_call): Also handle nontrivial SCC as side-effect.
+	(modref_propagate_in_scc): Update.
+
+2021-11-11  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/103190
+	* tree-ssa-reassoc.c (insert_stmt_after): Only assert on asm goto.
+
+2021-11-11  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gimple-range-path.cc (path_range_query::add_copies_to_imports):
+	Rename to...
+	(path_range_query::compute_imports): ...this.  Adapt it so it can
+	be passed the imports bitmap instead of working on m_imports.
+	(path_range_query::compute_ranges): Call compute_imports in all
+	cases unless an imports bitmap is passed.
+	* gimple-range-path.h (path_range_query::compute_imports): New.
+	(path_range_query::add_copies_to_imports): Remove.
+	* tree-ssa-threadbackward.c (back_threader::resolve_def): Remove.
+	(back_threader::find_paths_to_names): Inline resolve_def.
+	(back_threader::find_paths): Call compute_imports.
+	(back_threader::resolve_phi): Adjust comment.
+
+2021-11-11  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/103188
+	* tree-ssa-loop-ch.c (should_duplicate_loop_header_p):
+	Remove query parameter, split out check for size
+	optimization.
+	(ch_base::m_ranger, cb_base::m_query): Remove.
+	(ch_base::copy_headers): Split processing loop into
+	analysis around which we allocate and use ranger and
+	transform where we do not.
+	(pass_ch::execute): Do not allocate/free ranger here.
+	(pass_ch_vect::execute): Likewise.
+
+2021-11-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-pure-const.c (propagate_pure_const): Self recursion is
+	a side effects.
+
+2021-11-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	* cgraph.c (set_noreturn_flag_1): New function.
+	(cgraph_node::set_noreturn_flag): New member function
+	* cgraph.h (cgraph_node::set_noreturn_flags): Declare.
+	* ipa-pure-const.c (pass_local_pure_const::execute): Use it.
+
+2021-11-11  Aldy Hernandez  <aldyh@redhat.com>
+	    Richard Biener  <rguenther@suse.de>
+
+	* tree-ssa-loop-ch.c (entry_loop_condition_is_static): Resolve
+	statically to the edge remaining in the loop.
+
+2021-11-11  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/103181
+	* tree-eh.c (operation_could_trap_helper_p): Properly
+	check vector constants for a zero element for integer
+	division.  Separate floating point and integer division code.
+
+2021-11-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR debug/101378
+	* dwarf2out.c (field_byte_offset): Do the PCC_BITFIELD_TYPE_MATTERS
+	handling only for DECL_BIT_FIELD_TYPE decls.
+
+2021-11-11  Prathamesh Kulkarni  <prathamesh.kulkarni@linaro.org>
+
+	PR target/102376
+	* config/aarch64/aarch64.c (aarch64_process_target_attr): Check if
+	token is arch extension without leading '+' and emit appropriate
+	diagnostic for the same.
+
+2021-11-11  Jakub Jelinek  <jakub@redhat.com>
+
+	* tree.h (OMP_CLAUSE_NUM_TEAMS_EXPR): Rename to ...
+	(OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR): ... this.
+	(OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR): Define.
+	* tree.c (omp_clause_num_ops): Increase num ops for
+	OMP_CLAUSE_NUM_TEAMS to 2.
+	* tree-pretty-print.c (dump_omp_clause): Print optional lower bound
+	for OMP_CLAUSE_NUM_TEAMS.
+	* gimplify.c (gimplify_scan_omp_clauses): Gimplify
+	OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR if non-NULL.
+	(optimize_target_teams): Use OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR instead
+	of OMP_CLAUSE_NUM_TEAMS_EXPR.  Handle OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR.
+	* omp-low.c (lower_omp_teams): Use OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR
+	instead of OMP_CLAUSE_NUM_TEAMS_EXPR.
+	* omp-expand.c (expand_teams_call, get_target_arguments): Likewise.
+
+2021-11-11  Richard Biener  <rguenther@suse.de>
+
+	* cfganal.c (find_pdom): Remove.
+	(control_dependences::find_control_dependence): Remove
+	special-casing of entry block, call get_immediate_dominator
+	directly.
+	* gimple-predicate-analysis.cc (find_pdom): Remove.
+	(find_dom): Likewise.
+	(find_control_equiv_block): Call get_immediate_dominator
+	directly.
+	(compute_control_dep_chain): Likewise.
+	(predicate::init_from_phi_def): Likewise.
+
+2021-11-11  Richard Biener  <rguenther@suse.de>
+
+	* cfganal.h (control_dependences::control_dependence_map):
+	Embed bitmap_head.
+	(control_dependences::m_bitmaps): New.
+	* cfganal.c (control_dependences::set_control_dependence_map_bit):
+	Adjust.
+	(control_dependences::clear_control_dependence_bitmap):
+	Likewise.
+	(control_dependences::find_control_dependence): Do not
+	find_edge for the abnormal edge test.
+	(control_dependences::control_dependences): Instead do not
+	add abnormal edges to the edge list.  Adjust.
+	(control_dependences::~control_dependences): Likewise.
+	(control_dependences::get_edges_dependent_on): Likewise.
+	* function-tests.c: Include bitmap.h.
+
+2021-11-11  Kewen Lin  <linkw@linux.ibm.com>
+
+	* doc/invoke.texi: Change references to "future cpu" to "power10",
+	"-mcpu=future" to "-mcpu=power10".  Adjust words for float128.
+
+2021-11-11  Cui,Lili  <lili.cui@intel.com>
+
+	* config/i386/i386-options.c (m_CORE_AVX2): Remove Alderlake
+	from m_CORE_AVX2.
+	(processor_cost_table): Use alderlake_cost for Alderlake.
+	* config/i386/i386.c (ix86_sched_init_global): Handle Alderlake.
+	* config/i386/x86-tune-costs.h (struct processor_costs): Add alderlake
+	cost.
+	* config/i386/x86-tune-sched.c (ix86_issue_rate): Change Alderlake
+	issue rate to 4.
+	(ix86_adjust_cost): Handle Alderlake.
+	* config/i386/x86-tune.def (X86_TUNE_SCHEDULE): Enable for Alderlake.
+	(X86_TUNE_PARTIAL_REG_DEPENDENCY): Likewise.
+	(X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY): Likewise.
+	(X86_TUNE_SSE_PARTIAL_REG_FP_CONVERTS_DEPENDENCY): Likewise.
+	(X86_TUNE_SSE_PARTIAL_REG_CONVERTS_DEPENDENCY): Likewise.
+	(X86_TUNE_MEMORY_MISMATCH_STALL): Likewise.
+	(X86_TUNE_USE_LEAVE): Likewise.
+	(X86_TUNE_PUSH_MEMORY): Likewise.
+	(X86_TUNE_USE_INCDEC): Likewise.
+	(X86_TUNE_INTEGER_DFMODE_MOVES): Likewise.
+	(X86_TUNE_MISALIGNED_MOVE_STRING_PRO_EPILOGUES): Likewise.
+	(X86_TUNE_USE_SAHF): Likewise.
+	(X86_TUNE_USE_BT): Likewise.
+	(X86_TUNE_AVOID_FALSE_DEP_FOR_BMI): Likewise.
+	(X86_TUNE_ONE_IF_CONV_INSN): Likewise.
+	(X86_TUNE_AVOID_MFENCE): Likewise.
+	(X86_TUNE_USE_SIMODE_FIOP): Likewise.
+	(X86_TUNE_EXT_80387_CONSTANTS): Likewise.
+	(X86_TUNE_SSE_UNALIGNED_LOAD_OPTIMAL): Likewise.
+	(X86_TUNE_SSE_UNALIGNED_STORE_OPTIMAL): Likewise.
+	(X86_TUNE_SSE_TYPELESS_STORES): Likewise.
+	(X86_TUNE_SSE_LOAD0_BY_PXOR): Likewise.
+	(X86_TUNE_AVOID_4BYTE_PREFIXES): Likewise.
+	(X86_TUNE_USE_GATHER): Disable for Alderlake.
+	(X86_TUNE_AVX256_MOVE_BY_PIECES): Likewise.
+	(X86_TUNE_AVX256_STORE_BY_PIECES): Likewise.
+
+2021-11-11  liuhongt  <hongtao.liu@intel.com>
+
+	PR target/103151
+	* config/i386/sse.md (V_128_256): Extend to V8HF/V16HF.
+	(avxsizesuffix): Ditto.
+
+2021-11-11  Kito Cheng  <kito.cheng@sifive.com>
+
+	* common/config/riscv/riscv-common.c (riscv_subset_list::to_string): Fix
+	wrong marco checking.
+
+2021-11-10  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/102906
+	* tree-ssa-loop-ch.c (entry_loop_condition_is_static): New.
+	(should_duplicate_loop_header_p): Call entry_loop_condition_is_static.
+	(class ch_base): Add m_ranger and m_query.
+	(ch_base::copy_headers): Pass m_query to
+	entry_loop_condition_is_static.
+	(pass_ch::execute): Allocate and deallocate m_ranger and
+	m_query.
+	(pass_ch_vect::execute): Same.
+
+2021-11-10  Andrew Pinski  <apinski@marvell.com>
+
+	PR target/103170
+	* config/aarch64/aarch64-simd.md (aarch64_simd_dup<mode>):
+	Use vwcore iterator for the r constraint output string.
+
+2021-11-10  qing zhao  <qing.zhao@oracle.com>
+
+	* internal-fn.c (expand_DEFERRED_INIT): Apply pattern initialization
+	only when have_insn_for return true for the mode. Fix a memory leak.
+
+2021-11-10  Christophe Lyon  <christophe.lyon@foss.st.com>
+
+	* config/arm/arm.c (cortexa9_extra_costs, cortexa8_extra_costs,
+	cortexa5_extra_costs, cortexa7_extra_costs,
+	cortexa12_extra_costs, cortexa15_extra_costs, v7m_extra_costs):
+	Initialize movi, dup and extract costing fields.
+
+2021-11-10  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gimple-range-path.cc (path_range_query::path_range_query): Do
+	not init m_path.
+	(path_range_query::dump): Change m_path uses to non-pointer.
+	(path_range_query::defined_outside_path):  Same.
+	(path_range_query::set_path): Same.
+	(path_range_query::add_copies_to_imports): Same.
+	(path_range_query::range_of_stmt): Same.
+	(path_range_query::compute_outgoing_relations): Same.
+	(path_range_query::compute_ranges): Imports are now optional.
+	Implement overload that takes an edge.
+	* gimple-range-path.h (class path_range_query): Make imports
+	optional for compute_ranges.  Add compute_ranges(edge) overload.
+	Make m_path an auto_vec instead of a pointer and adjust
+	accordingly.
+
+2021-11-10  Tamar Christina  <tamar.christina@arm.com>
+
+	* tree-vectorizer.h (struct scalar_cond_masked_key): Add inverted_p.
+	(default_hash_traits<scalar_conf_masked_key>): Likewise.
+	* tree-vect-stmts.c (vectorizable_condition): Check if inverse of mask
+	is live.
+	* tree-vectorizer.c (scalar_cond_masked_key::get_cond_ops_from_tree):
+	Register mask inverses.
+
+2021-11-10  Tamar Christina  <tamar.christina@arm.com>
+
+	* tree-vectorizer.c (vectorize_loops): Do local CSE through RPVN upon
+	successful vectorization.
+
+2021-11-10  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gimple-range-cache.cc (sbr_vector::grow): New.
+	(sbr_vector::set_bb_range): Call grow.
+	(sbr_vector::get_bb_range): Same.
+	(sbr_vector::bb_range_p): Remove assert.
+
+2021-11-10  Tamar Christina  <tamar.christina@arm.com>
+
+	* config/aarch64/aarch64-simd.md (*aarch64_topbits_shuffle<mode>_le
+	,*aarch64_topbits_shuffle<mode>_be): Remove.
+
+2021-11-10  Jan Hubicka  <jh@suse.cz>
+
+	* ipa-modref.c: Include tree-eh.h
+	(modref_summary::modref_summary): Initialize side_effects.
+	(struct modref_summary_lto): New bool field side_effects.
+	(modref_summary_lto::modref_summary_lto): Initialize side_effects.
+	(modref_summary::dump): Dump side_effects.
+	(modref_summary_lto::dump): Dump side_effects.
+	(merge_call_side_effects): Merge side effects.
+	(process_fnspec): Calls to non-const/pure or looping
+	function is a side effect.
+	(analyze_call): Self-recursion is a side-effect; handle
+	special builtins.
+	(analyze_load): Watch for volatile and throwing memory.
+	(analyze_store): Likewise.
+	(analyze_stmt): Watch for volatitle asm.
+	(analyze_function): Handle side_effects.
+	(modref_summaries::duplicate): Duplicate side_effects.
+	(modref_summaries_lto::duplicate): Likewise.
+	(modref_write): Stream side_effects.
+	(read_section): Likewise.
+	(update_signature): Update.
+	(propagate_unknown_call): Handle side_effects.
+	(modref_propagate_in_scc): Likewise.
+	* ipa-modref.h (struct modref_summary): Add side_effects.
+	* ipa-pure-const.c (special_builtin_state): Rename to ...
+	(builtin_safe_for_const_function_p): ... this one.
+	(check_call): Update.
+	(finite_function_p): Break out from ...
+	(propagate_pure_const): ... here
+	* ipa-utils.h (finite_function): Declare.
+
+2021-11-10  Lucas A. M. Magalhães  <lamm@linux.ibm.com>
+
+	* config.gcc (powerpc*-*-*): Remove -rpath from
+	--with-advance-toolchain.
+
+2021-11-10  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/101940
+	* attribs.c (struct scoped_attributes): Add a bool member.
+	(lookup_scoped_attribute_spec): Forward declare.
+	(register_scoped_attributes): New bool parameter, defaulted to
+	false.  Use it.
+	(handle_ignored_attributes_option): New function.
+	(free_attr_data): New function.
+	(init_attributes): Call handle_ignored_attributes_option.
+	(attr_namespace_ignored_p): New function.
+	(decl_attributes): Check attr_namespace_ignored_p before
+	warning.
+	* attribs.h (free_attr_data): Declare.
+	(register_scoped_attributes): Adjust declaration.
+	(handle_ignored_attributes_option): Declare.
+	(canonicalize_attr_name): New function template.
+	(canonicalize_attr_name): Use it.
+	* common.opt (Wattributes=): New option with a variable.
+	* doc/extend.texi: Document #pragma GCC diagnostic ignored_attributes.
+	* doc/invoke.texi: Document -Wno-attributes=.
+	* opts.c (common_handle_option) <case OPT_Wattributes_>: Handle.
+	* plugin.h (register_scoped_attributes): Adjust declaration.
+	* toplev.c (compile_file): Call free_attr_data.
+
+2021-11-10  Przemyslaw Wirkus  <przemyslaw.wirkus@arm.com>
+
+	* config/arm/arm-cpus.in (cortex-a710): New CPU.
+	* config/arm/arm-tables.opt: Regenerate.
+	* config/arm/arm-tune.md: Regenerate.
+	* doc/invoke.texi: Update docs.
+
+2021-11-10  Andre Vieira  <andre.simoesdiasvieira@arm.com>
+
+	* config/aarch64/aarch64-builtins.c
+	(aarch64_general_gimple_fold_builtin): Mark argument as unused.
+
+2021-11-10  Martin Liska  <mliska@suse.cz>
+
+	* lto-wrapper.c (merge_and_complain): Make the first argument
+	a reference type.
+
+2021-11-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/iterators.md (optab): Use fmax_nan instead of
+	smax_nan and fmin_nan instead of smin_nan.
+	(maxmin_uns): Rename to...
+	(fmaxmin): ...this and make the same changes.  Remove entries
+	unrelated to fmax* and fmin*.
+	* config/aarch64/aarch64.md (<maxmin_uns><mode>3): Rename to...
+	(<fmaxmin><mode>3): ...this.
+	* config/aarch64/aarch64-simd.md (aarch64_<maxmin_uns>p<mode>):
+	Rename to...
+	(aarch64_<optab>p<mode>): ...this.
+	(<maxmin_uns><mode>3): Rename to...
+	(<fmaxmin><mode>3): ...this.
+	(reduc_<maxmin_uns>_scal_<mode>): Rename to...
+	(reduc_<optab>_scal_<mode>): ...this and update gen* call.
+	(aarch64_reduc_<maxmin_uns>_internal<mode>): Rename to...
+	(aarch64_reduc_<optab>_internal<mode>): ...this.
+	(aarch64_reduc_<maxmin_uns>_internalv2si): Rename to...
+	(aarch64_reduc_<optab>_internalv2si): ...this.
+	* config/aarch64/aarch64-sve.md (<maxmin_uns><mode>3): Rename to...
+	(<fmaxmin><mode>3): ...this.
+	* config/aarch64/aarch64-simd-builtins.def (smax_nan, smin_nan)
+	Rename to...
+	(fmax_nan, fmin_nan): ...this.
+	* config/aarch64/arm_neon.h (vmax_f32, vmax_f64, vmaxq_f32, vmaxq_f64)
+	(vmin_f32, vmin_f64, vminq_f32, vminq_f64, vmax_f16, vmaxq_f16)
+	(vmin_f16, vminq_f16): Update accordingly.
+
+2021-11-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* tree-vectorizer.h (vector_costs::finish_cost): Take the
+	corresponding scalar costs as a parameter.
+	(finish_cost): Likewise.
+	* tree-vect-loop.c (vect_compute_single_scalar_iteration_cost)
+	(vect_estimate_min_profitable_iters): Update accordingly.
+	* tree-vect-slp.c (vect_bb_vectorization_profitable_p): Likewise.
+	* tree-vectorizer.c (vector_costs::finish_cost): Likewise.
+	* config/aarch64/aarch64.c (aarch64_vector_costs::finish_cost):
+	Likewise.
+	* config/rs6000/rs6000.c (rs6000_cost_data::finish_cost): Likewise.
+
+2021-11-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* tree-vectorizer.h (_loop_vec_info::scalar_costs): New member
+	variable.
+	(_loop_vec_info::single_scalar_iteration_cost): Delete.
+	(LOOP_VINFO_SINGLE_SCALAR_ITERATION_COST): Delete.
+	(vector_costs::total_cost): New function.
+	* tree-vect-loop.c (_loop_vec_info::_loop_vec_info): Update
+	after above changes.
+	(_loop_vec_info::~_loop_vec_info): Delete scalar_costs.
+	(vect_compute_single_scalar_iteration_cost): Store the costs
+	in loop_vinfo->scalar_costs.
+	(vect_estimate_min_profitable_iters): Get the scalar cost from
+	loop_vinfo->scalar_costs.
+
+2021-11-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* tree-vectorizer.h (vector_costs::better_main_loop_than_p)
+	(vector_costs::better_epilogue_loop_than_p)
+	(vector_costs::compare_inside_loop_cost)
+	(vector_costs::compare_outside_loop_cost): Likewise.
+	* tree-vectorizer.c (vector_costs::better_main_loop_than_p)
+	(vector_costs::better_epilogue_loop_than_p)
+	(vector_costs::compare_inside_loop_cost)
+	(vector_costs::compare_outside_loop_cost): New functions,
+	containing code moved from...
+	* tree-vect-loop.c (vect_better_loop_vinfo_p): ...here.
+
+2021-11-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* tree-vectorizer.h (_loop_vec_info): Remove vec_outside_cost
+	and vec_inside_cost.
+	(vector_costs::outside_cost): New function.
+	* tree-vect-loop.c (_loop_vec_info::_loop_vec_info): Update
+	after above.
+	(vect_estimate_min_profitable_iters): Likewise.
+	(vect_better_loop_vinfo_p): Get the inside and outside costs
+	from the loop_vec_infos' vector_costs.
+
+2021-11-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* tree-vectorizer.h (vec_info::target_cost_data): Replace with...
+	(_loop_vec_info::vector_costs): ...this.
+	(LOOP_VINFO_TARGET_COST_DATA): Delete.
+	* tree-vectorizer.c (vec_info::vec_info): Remove target_cost_data
+	initialization.
+	(vec_info::~vec_info): Remove corresponding delete.
+	* tree-vect-loop.c (_loop_vec_info::_loop_vec_info): Initialize
+	vector_costs to null.
+	(_loop_vec_info::~_loop_vec_info): Delete vector_costs.
+	(vect_analyze_loop_operations): Update after above changes.
+	(vect_analyze_loop_2): Likewise.
+	(vect_estimate_min_profitable_iters): Likewise.
+	* tree-vect-slp.c (vect_slp_analyze_operations): Likewise.
+
+2021-11-10  Jan Hubicka  <hubicka@ucw.cz>
+
+	* tree-core.h (EAF_DIRECT): Remove.
+	(EAF_NOCLOBBER): Remove.
+	(EAF_UNUSED): Remove.
+	(EAF_NOESCAPE): Remove.
+	(EAF_NO_DIRECT_CLOBBER): New.
+	(EAF_NO_INDIRECT_CLOBBER): New.
+	(EAF_NODIRECTESCAPE): Remove.
+	(EAF_NO_DIRECT_ESCAPE): New.
+	(EAF_NO_INDIRECT_ESCAPE): New.
+	(EAF_NOT_RETURNED): Remove.
+	(EAF_NOT_RETURNED_INDIRECTLY): New.
+	(EAF_NOREAD): Remove.
+	(EAF_NO_DIRECT_READ): New.
+	(EAF_NO_INDIRECT_READ): New.
+	* gimple.c (gimple_call_arg_flags): Update for new flags.
+	(gimple_call_retslot_flags): Update for new flags.
+	* ipa-modref.c (dump_eaf_flags): Likewise.
+	(remove_useless_eaf_flags): Likewise.
+	(deref_flags): Likewise.
+	(modref_lattice::init): Likewise.
+	(modref_lattice::merge): Likewise.
+	(modref_lattice::merge_direct_load): Likewise.
+	(modref_lattice::merge_direct_store): Likewise.
+	(modref_eaf_analysis::merge_call_lhs_flags): Likewise.
+	(callee_to_caller_flags): Likewise.
+	(modref_eaf_analysis::analyze_ssa_name): Likewise.
+	(modref_eaf_analysis::propagate): Likewise.
+	(modref_merge_call_site_flags): Likewise.
+	* ipa-modref.h (interposable_eaf_flags): Likewise.
+	* tree-ssa-alias.c: (ref_maybe_used_by_call_p_1) Likewise.
+	* tree-ssa-structalias.c (handle_call_arg): Likewise.
+	(handle_rhs_call): Likewise.
+	* tree-ssa-uninit.c (maybe_warn_pass_by_reference): Likewise.
+
+2021-11-10  Tamar Christina  <tamar.christina@arm.com>
+
+	* tree-vect-slp-patterns.c (complex_mul_pattern::matches): Remove l1node.
+
+2021-11-10  Andre Vieira  <andre.simoesdiasvieira@arm.com>
+
+	* config/aarch64/aarch64-builtins.c
+	(aarch64_general_gimple_fold_builtin): Change pointer alignment and
+	alias.
+
+2021-11-10  Jan Hubicka  <jh@suse.cz>
+
+	* ipa-modref-tree.h (modref_tree::remap_params): Fix off-by-one error.
+
+2021-11-10  H.J. Lu  <hongjiu.lu@intel.com>
+	    Hongtao Liu  <hongtao.liu@intel.com>
+
+	PR middle-end/102566
+	* match.pd (nop_atomic_bit_test_and_p): New match.
+	* tree-ssa-ccp.c (convert_atomic_bit_not): New function.
+	(gimple_nop_atomic_bit_test_and_p): New prototype.
+	(optimize_atomic_bit_test_and): Transform equivalent, but slighly
+	different cases to their canonical forms.
+
+2021-11-10  liuhongt  <hongtao.liu@intel.com>
+
+	PR tree-optimization/103126
+	* tree-vect-loop.c (neutral_op_for_reduction): Remove static.
+	* tree-vectorizer.h (neutral_op_for_reduction): Declare.
+	* tree-if-conv.c : Include tree-vectorizer.h.
+	(is_cond_scalar_reduction): Handle
+	BIT_XOR_EXPR/BIT_IOR_EXPR/BIT_AND_EXPR.
+	(convert_scalar_cond_reduction): Ditto.
+
+2021-11-10  konglin1  <lingling.kong@intel.com>
+
+	* config/i386/sse.md (cmul<conj_op><mode>3): add new define_expand.
+	(cmla<conj_op><mode>4): Likewise
+
+2021-11-10  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadedge.c: Do not include
+	gimple-ssa-evrp-analyze.h.
+	* value-pointer-equiv.cc: Same.
+
+2021-11-10  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c
+	(back_threader::maybe_register_path_dump): Abstract path dumping...
+	(dump_path): ...here.
+	(back_threader::resolve_phi): Call dump_path.
+	(debug): Same.
+
+2021-11-10  konglin1  <lingling.kong@intel.com>
+
+	* config/i386/sse.md (fma_<complexpairopname>_<mode>_pair):
+	Add new define_insn.
+	(fma_<mode>_fmaddc_bcst): Add new define_insn_and_split.
+	(fma_<mode>_fcmaddc_bcst): Likewise
+
+2021-11-10  liuhongt  <hongtao.liu@intel.com>
+
+	PR target/102464
+	* match.pd: Simplify (trunc)fmax/fmin((extend)a, (extend)b) to
+	MAX/MIN(a,b)
+
+2021-11-10  Andrew Pinski  <apinski@marvell.com>
+
+	PR target/101529
+	* config/aarch64/aarch64.c (aarch64_evpc_ins): Don't use target
+	as an input, use original one.
+
+2021-11-10  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* config/nios2/nios2.c (nios2_can_inline_p): New.
+	(TARGET_CAN_INLINE_P): Define.
+
+2021-11-09  Bill Schmidt  <wschmidt@linux.ibm.com>
+
+	* config/rs6000/rs6000-call.c (rs6000_gimple_fold_new_builtin):
+	Disable gimple fold for RS6000_BIF_{XVMINDP,XVMINSP,VMINFP} and
+	RS6000_BIF_{XVMAXDP,XVMAXSP,VMAXFP} when fast-math is not set.
+	(lxvrse_expand_builtin): Modify the expansion for sign extension.
+	All extensions are done within VSX registers.
+
+2021-11-09  Andrew Pinski  <apinski@marvell.com>
+
+	PR tree-optimization/10352
+	* match.pd: Remove check of TYPE_PRECISION for
+	the a?0:pow2 case.
+
+2021-11-09  Andrew MacLeod  <amacleod@redhat.com>
+
+	* function.c (allocate_struct_function): Don't set x_range_query.
+	* function.h (get_range_query): Move to value-query.h.
+	* gimple-range.cc (enable_ranger): Check that query is currently NULL.
+	(disable_ranger): Clear function current query field.
+	* value-query.cc (get_global_range_query): Relocate to:
+	* value-query.h (get_global_range_query): Here and inline.
+	(get_range_query): Relocate here from function.h.
+
+2021-11-09  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c
+	(back_threader::maybe_register_path_dump): New.
+	(back_threader::maybe_register_path): Call maybe_register_path_dump.
+
+2021-11-09  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c (back_threader::maybe_register_path):
+	Return NULL when unprofitable.
+
+2021-11-09  Martin Jambor  <mjambor@suse.cz>
+
+	* tree.h (build_debug_expr_decl): Declare.
+	* tree.c (build_debug_expr_decl): New function.
+	* cfgexpand.c (avoid_deep_ter_for_debug): Use build_debug_expr_decl
+	instead of building a DEBUG_EXPR_DECL.
+	* ipa-param-manipulation.c
+	(ipa_param_body_adjustments::prepare_debug_expressions): Likewise.
+	* omp-simd-clone.c (ipa_simd_modify_stmt_ops): Likewise.
+	* tree-ssa-ccp.c (optimize_atomic_bit_test_and): Likewise.
+	* tree-ssa-phiopt.c (spaceship_replacement): Likewise.
+	* tree-ssa-reassoc.c (make_new_ssa_for_def): Likewise.
+
+2021-11-09  Bill Schmidt  <wschmidt@linux.ibm.com>
+
+	* config/rs6000/rs6000-call.c (rs6000_type_string): New function.
+	(def_builtin): Change debug formatting for easier parsing and
+	include more information.
+	(rs6000_init_builtins): Add dump of autogenerated builtins.
+	(altivec_init_builtins): Dump __builtin_altivec_mask_for_load for
+	completeness.
+
+2021-11-09  Marek Polacek  <polacek@redhat.com>
+
+	* ipa-modref.c (analyze_parms): Fix a typo.
+
+2021-11-09  Kito Cheng  <kito.cheng@sifive.com>
+
+	PR target/102957
+	* common/config/riscv/riscv-common.c (multi_letter_subset_rank): Remove
+	assertion for Z*-ext.
+
+2021-11-09  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref.c (analyze_parms): Add past_flags, past_retslot_flags
+	and past_static_chain; merge past summary with current one.
+	(analyze_function): Update.
+
+2021-11-09  Aldy Hernandez  <aldyh@redhat.com>
+
+	* doc/invoke.texi (Invoking GCC): Document --param=threader-debug.
+
+2021-11-09  Martin Liska  <mliska@suse.cz>
+
+	* print-rtl.c (rtx_writer::rtx_writer): Make the compilation
+	conditional based on
+	* print-rtl.h (class rtx_writer): Likewise.
+
+2021-11-09  Thomas Schwinge  <thomas@codesourcery.com>
+
+	* input.h (location_hash): Use 'BUILTINS_LOCATION' as spare value
+	for 'Deleted'.  Turn into a '#define'.
+
+2021-11-09  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gimple-range-path.cc (path_range_query::dump): Clean up.
+	(path_range_query::compute_ranges): Same.
+	* value-relation.cc (path_oracle::dump): Same.
+
+2021-11-09  Aldy Hernandez  <aldyh@redhat.com>
+
+	* dumpfile.c (dump_options): Remove TDF_THREADING entry.
+	* dumpfile.h (enum dump_flag): Remove TDF_THREADING and adjust
+	remaining entries.
+	* flag-types.h (enum threader_debug): New.
+	* gimple-range-path.cc (DEBUG_SOLVER): Use param_threader_debug.
+	* params.opt: Add entry for --param=threader-debug=.
+
+2021-11-09  Thomas Schwinge  <thomas@codesourcery.com>
+
+	PR middle-end/101204
+	PR other/103157
+	* diagnostic-spec.h (typedef xint_hash_t): Turn into...
+	(struct xint_hash_t): ... this.
+	* doc/gty.texi: Update.
+
+2021-11-09  Przemyslaw Wirkus  <przemyslaw.wirkus@arm.com>
+
+	* config/arm/arm-cpus.in (armv9): New define.
+	(ARMv9a): New group.
+	(armv9-a): New arch definition.
+	* config/arm/arm-tables.opt: Regenerate.
+	* config/arm/arm.h (BASE_ARCH_9A): New arch enum value.
+	* config/arm/t-aprofile: Added armv9-a and armv9+simd.
+	* config/arm/t-arm-elf: Added arm9-a, v9_fps and all_v9_archs
+	to MULTILIB_MATCHES.
+	* config/arm/t-multilib: Added v9_a_nosimd_variants and
+	v9_a_simd_variants to MULTILIB_MATCHES.
+	* doc/invoke.texi: Update docs.
+
+2021-11-09  Martin Jambor  <mjambor@suse.cz>
+
+	PR ipa/103132
+	* ipa-param-manipulation.c (replace_with_mapped_expr): Early
+	return with error_mark_mode when part of expression is mapped to
+	NULL.
+	(ipa_param_body_adjustments::remap_with_debug_expressions): Set
+	mapped value to NULL if walk_tree returns error_mark_mode.
+
+2021-11-09  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-strlen.c (compare_nonzero_chars): Pass statement
+	context to ranger.
+	(get_addr_stridx): Same.
+	(get_stridx): Same.
+	(get_range_strlen_dynamic): Same.
+	(handle_builtin_strlen): Same.
+	(handle_builtin_strchr): Same.
+	(handle_builtin_strcpy): Same.
+	(maybe_diag_stxncpy_trunc): Same.
+	(handle_builtin_stxncpy_strncat): Same.
+	(handle_builtin_memcpy): Same.
+	(handle_builtin_strcat): Same.
+	(handle_alloc_call): Same.
+	(handle_builtin_memset): Same.
+	(handle_builtin_string_cmp): Same.
+	(handle_pointer_plus): Same.
+	(count_nonzero_bytes_addr): Same.
+	(count_nonzero_bytes): Same.
+	(handle_store): Same.
+	(fold_strstr_to_strncmp): Same.
+	(handle_integral_assign): Same.
+	(check_and_optimize_stmt): Same.
+	(class strlen_dom_walker): Replace evrp with ranger.
+	(strlen_dom_walker::before_dom_children): Remove evrp.
+	(strlen_dom_walker::after_dom_children): Remove evrp.
+	* gimple-ssa-warn-access.cc (maybe_check_access_sizes):
+	Restrict sprintf output.
+
+2021-11-09  Martin Liska  <mliska@suse.cz>
+
+	* genconditions.c (write_one_condition): Add const qualifier
+	to pointer casting.
+
+2021-11-09  Jeff Law  <jeffreyalaw@gmail.com>
+
+	* match.pd: New pattern to simplify (1 << n) & M ==/!= 0 for M
+	being a power of 2.
+
+2021-11-08  Bill Schmidt  <wschmidt@linux.ibm.com>
+
+	* config/rs6000/rs6000-builtin-new.def (VEC_INIT_V16QI): Use
+	escape-newline support.
+	(VEC_INIT_V4SI): Likewise.
+	(VEC_INIT_V8HI): Likewise.
+	(PACK_V1TI): Likewise.
+	(DIVDEU): Likewise.
+	(VFIRSTMISMATCHOREOSINDEX_V16QI): Likewise.
+	(VFIRSTMISMATCHOREOSINDEX_V8HI): Likewise.
+	(VFIRSTMISMATCHOREOSINDEX_V4SI): Likewise.
+	(CMPRB2): Likewise.
+	(VSTDCP): Likewise.
+	(VSIEDP): Likewise.
+	(FMAF128_ODD): Likewise.
+	(VSCEQPUO): Likewise.
+	(VSIEQP): Likewise.
+	(VSIEQPF): Likewise.
+	(VSTDCQP): Likewise.
+	(PACK_TD): Likewise.
+	(TABORTDC): Likewise.
+	(TABORTDCI): Likewise.
+	(SE_LXVRBX): Likewise.
+	(SE_LXVRHX): Likewise.
+	(SE_LXVRWX): Likewise.
+	(SE_LXVRDX): Likewise.
+	(VREPLACE_UN_UV2DI): Likewise.
+	(VREPLACE_UN_UV4SI): Likewise.
+	(VREPLACE_UN_V2DI): Likewise.
+	(VREPLACE_ELT_UV2DI): Likewise.
+	(VREPLACE_ELT_V2DI): Likewise.
+	(ZE_LXVRBX): Likewise.
+	(ZE_LXVRHX): Likewise.
+	(ZE_LXVRWX): Likewise.
+	(ZE_LXVRDX): Likewise.
+	(CFUGED): Likewise.
+	(CNTLZDM): Likewise.
+	(CNTTZDM): Likewise.
+	(PDEPD): Likewise.
+	(PEXTD): Likewise.
+	(PMXVBF16GER2): Likewise.
+	(PMXVBF16GER2_INTERNAL): Likewise.
+	(PMXVBF16GER2NN): Likewise.
+	(PMXVBF16GER2NN_INTERNAL): Likewise.
+	(PMXVBF16GER2NP): Likewise.
+	(PMXVBF16GER2NP_INTERNAL): Likewise.
+	(PMXVBF16GER2PN): Likewise.
+	(PMXVBF16GER2PN_INTERNAL): Likewise.
+	(PMXVBF16GER2PP): Likewise.
+	(PMXVBF16GER2PP_INTERNAL): Likewise.
+	(PMXVF16GER2): Likewise.
+	(PMXVF16GER2_INTERNAL): Likewise.
+	(PMXVF16GER2NN): Likewise.
+	(PMXVF16GER2NN_INTERNAL): Likewise.
+	(PMXVF16GER2NP): Likewise.
+	(PMXVF16GER2NP_INTERNAL): Likewise.
+	(PMXVF16GER2PN): Likewise.
+	(PMXVF16GER2PN_INTERNAL): Likewise.
+	(PMXVF16GER2PP): Likewise.
+	(PMXVF16GER2PP_INTERNAL): Likewise.
+	(PMXVF32GER_INTERNAL): Likewise.
+	(PMXVF32GERNN): Likewise.
+	(PMXVF32GERNN_INTERNAL): Likewise.
+	(PMXVF32GERNP): Likewise.
+	(PMXVF32GERNP_INTERNAL): Likewise.
+	(PMXVF32GERPN): Likewise.
+	(PMXVF32GERPN_INTERNAL): Likewise.
+	(PMXVF32GERPP): Likewise.
+	(PMXVF32GERPP_INTERNAL): Likewise.
+	(PMXVF64GER): Likewise.
+	(PMXVF64GER_INTERNAL): Likewise.
+	(PMXVF64GERNN): Likewise.
+	(PMXVF64GERNN_INTERNAL): Likewise.
+	(PMXVF64GERNP): Likewise.
+	(PMXVF64GERNP_INTERNAL): Likewise.
+	(PMXVF64GERPN): Likewise.
+	(PMXVF64GERPN_INTERNAL): Likewise.
+	(PMXVF64GERPP): Likewise.
+	(PMXVF64GERPP_INTERNAL): Likewise.
+	(PMXVI16GER2): Likewise.
+	(PMXVI16GER2_INTERNAL): Likewise.
+	(PMXVI16GER2PP): Likewise.
+	(PMXVI16GER2PP_INTERNAL): Likewise.
+	(PMXVI16GER2S): Likewise.
+	(PMXVI16GER2S_INTERNAL): Likewise.
+	(PMXVI16GER2SPP): Likewise.
+	(PMXVI16GER2SPP_INTERNAL): Likewise.
+	(PMXVI4GER8): Likewise.
+	(PMXVI4GER8_INTERNAL): Likewise.
+	(PMXVI4GER8PP): Likewise.
+	(PMXVI4GER8PP_INTERNAL): Likewise.
+	(PMXVI8GER4): Likewise.
+	(PMXVI8GER4_INTERNAL): Likewise.
+	(PMXVI8GER4PP): Likewise.
+	(PMXVI8GER4PP_INTERNAL): Likewise.
+	(PMXVI8GER4SPP): Likewise.
+	(PMXVI8GER4SPP_INTERNAL): Likewise.
+	* config/rs6000/rs6000-gen-builtins.c (MAXLINES): New macro.
+	(linebuf): Increase size.
+	(lines): New variable.
+	(lastline): Likewise.
+	(real_line_pos): New function.
+	(diag): Change signature.
+	(bif_diag): Change signature; support escape-newline handling.
+	(ovld_diag): Likewise.
+	(fatal): Move earlier.
+	(consume_whitespace): Adjust diag call.
+	(advance_line): Add escape-newline handling; call fatal.
+	(safe_inc_pos): Adjust diag call.
+	(match_identifier): Likewise.
+	(match_integer): Likewise.
+	(match_to_right_bracket): Call fatal instead of diag; adjust diag
+	call.
+	(match_basetype): Adjust diag calls.
+	(match_bracketed_pair): Likewise.
+	(match_const_restriction): Likewise.
+	(match_type): Likewise.
+	(parse_args): Likewise.
+	(parse_bif_attrs): Likewise.
+	(complete_vector_type): Likewise.
+	(complete_base_type): Likewise.
+	(parse_prototype): Likewise.
+	(parse_bif_entry): Likewise.
+	(parse_bif_stanza): Likewise.
+	(parse_ovld_entry): Likewise.
+	(parse_ovld_stanza): Likewise.
+	(main): Allocate buffers for lines[].
+
+2021-11-08  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* config/i386/i386.c (legitimize_pic_address): Adjust comment and
+	use the REG argument on the CM_LARGE_PIC code path as well.
+	* config/i386/predicates.md (gotoff_operand): Do not treat VxWorks
+	specially with the large code models.
+
+2021-11-08  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gimple.c (gimple_call_static_chain_flags): Revert the workaround
+	allowing interposition since issues with binds_to_local_def were
+	hopefully solved.
+
+2021-11-08  Andrew MacLeod  <amacleod@redhat.com>
+
+	PR tree-optimization/103122
+	* gimple-range.cc (gimple_ranger::range_of_expr): Request the cache
+	entry with "calulate new values" set to false.
+
+2021-11-08  Jan Hubicka  <hubicka@ucw.cz>
+
+	* builtins.c (is_simple_builtin): Add builitin_dwarf_cfa
+	and builtin_return_address.
+	(builtin_fnspec): Annotate builtin_return,
+	bulitin_eh_pointer, builtin_eh_filter, builtin_unwind_resume,
+	builtin_cxa_end_cleanup, builtin_eh_copy_values,
+	builtin_frame_address, builtin_apply_args,
+	builtin_asan_before_dynamic_init, builtin_asan_after_dynamic_init,
+	builtin_prefetch, builtin_dwarf_cfa, builtin_return_addrss
+	as ".c"
+	* ipa-pure-const.c (special_builtin_state): Add builtin_dwarf_cfa
+	and builtin_return_address.
+
+2021-11-08  Jan Hubicka  <hubicka@ucw.cz>
+
+	PR tree-optimization/103177
+	* passes.def: Move uncprop after pure/const and modref.
+
+2021-11-08  Martin Jambor  <mjambor@suse.cz>
+
+	PR ipa/103099
+	PR ipa/103107
+	* tree-inline.c (remap_gimple_stmt): Unshare the expression without
+	location before invoking remap_with_debug_expressions on it.
+	* ipa-param-manipulation.c
+	(ipa_param_body_adjustments::prepare_debug_expressions): Likewise.
+
+2021-11-08  David Edelsohn  <dje.gcc@gmail.com>
+
+	* config/rs6000/vsx.md (vsx_splat_v4si_di): Revert "wa"
+	constraint to "we".
+
+2021-11-08  Richard Biener  <rguenther@suse.de>
+
+	* cfgloop.c (verify_loop_structure): Use a temporary BB flag
+	instead of an sbitmap to cache irreducible state.
+
+2021-11-08  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/103120
+	* gimple-range-path.cc (path_range_query::range_defined_in_block):
+	Bail if there's a cache entry.
+
+2021-11-08  Bill Schmidt  <wschmidt@linux.ibm.com>
+
+	* config/rs6000/rs6000.c (rs6000_builtin_reciprocal): Use
+	rs6000_builtin_decls_x when appropriate.
+	(add_condition_to_bb): Likewise.
+	(rs6000_atomic_assign_expand_fenv): Likewise.
+
+2021-11-08  Bill Schmidt  <wschmidt@linux.ibm.com>
+
+	* config/rs6000/rs6000-call.c (rs6000_new_builtin_decl): New function.
+	(rs6000_builtin_decl): Call it.
+
+2021-11-08  Martin Liska  <mliska@suse.cz>
+
+	* cgraph.c (cgraph_node::dump): Dump it from decl.
+
+2021-11-08  Martin Liska  <mliska@suse.cz>
+
+	PR gcov-profile/100520
+	* coverage.c (coverage_compute_profile_id): Strip .gk when
+	compare debug is used.
+	* system.h (endswith): New function.
+
+2021-11-08  Martin Liska  <mliska@suse.cz>
+
+	* cgraph.c (cgraph_node::dump): Dump static_chain_decl.
+
+2021-11-08  Thomas Schwinge  <thomas@codesourcery.com>
+
+	* config/rs6000/rbtree.c: Fix 'Copyright (C) 2020-21' into '2020-2021'
+	* config/rs6000/rbtree.h: Likewise.
+	* config/rs6000/rs6000-builtin-new.def: Likewise.
+	* config/rs6000/rs6000-gen-builtins.c: Likewise.
+	* config/rs6000/rs6000-overload.def: Likewise.
+
+2021-11-08  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_ldn_stn_vectors): New function.
+	(aarch64_address_cost): Use it instead of testing for CImode and
+	XImode directly.
+
+2021-11-08  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* genmodes.c (emit_insn_modes_h): Define NUM_MODE_* macros.
+	* expmed.h (NUM_MODE_INT): Delete in favor of genmodes definitions.
+	(NUM_MODE_PARTIAL_INT, NUM_MODE_VECTOR_INT): Likewise.
+	* real.h (real_format_for_mode): Use NUM_MODE_FLOAT and
+	NUM_MODE_DECIMAL_FLOAT.
+	(REAL_MODE_FORMAT): Likewise.
+
+2021-11-08  Andre Vieira  <andre.simoesdiasvieira@arm.com>
+
+	* tree-vect-loop.c (vect_better_loop_vinfo_p): Change how epilogue loop
+	costs are compared.
+
+2021-11-08  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_create_loop_vinfo): Add main_loop_info
+	parameter.
+	* tree-vect-loop.c (vect_create_loop_vinfo): Likewise.  Set
+	LOOP_VINFO_ORIG_LOOP_INFO and conditionalize set of
+	LOOP_VINFO_NITERS_ASSUMPTIONS.
+	(vect_analyze_loop_1): Adjust.
+	(vect_analyze_loop): Move loop constraint setting and
+	SCEV/niter reset here from vect_create_loop_vinfo to perform
+	it only once.
+	(vect_analyze_loop_form): Move dumping of symbolic niters
+	here from vect_create_loop_vinfo.
+
+2021-11-08  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref-tree.h (enum modref_special_parms): New enum.
+	(struct modref_access_node): update for special parms.
+	(struct modref_ref_node): Likewise.
+	(struct modref_parm_map): Likewise.
+	(struct modref_tree): Likewise.
+	* ipa-modref.c (dump_access): Likewise.
+	(get_access): Detect static chain.
+	(parm_map_for_arg): Take tree as arg instead of
+	stmt and index.
+	(merge_call_side_effects): Compute map for static chain.
+	(process_fnspec): Update.
+	(struct escape_point): Remove retslot_arg and static_chain_arg.
+	(analyze_parms): Update.
+	(compute_parm_map): Update.
+	(propagate_unknown_call): Update.
+	(modref_propagate_in_scc): Update.
+	(modref_merge_call_site_flags): Update.
+	(ipa_merge_modref_summary_after_inlining): Update.
+	* tree-ssa-alias.c (modref_may_conflict): Handle static chain.
+	* ipa-modref-tree.c (test_merge): Update.
+
+2021-11-08  Haochen Gui  <guihaoc@gcc.gnu.org>
+
+	* config/rs6000/rs6000-call.c (rs6000_gimple_fold_builtin): Disable
+	gimple fold for VSX_BUILTIN_XVMINDP, ALTIVEC_BUILTIN_VMINFP,
+	VSX_BUILTIN_XVMAXDP, ALTIVEC_BUILTIN_VMAXFP when fast-math is not
+	set.
+
+2021-11-08  liuhongt  <hongtao.liu@intel.com>
+
+	PR tree-optimization/103077
+	* doc/invoke.texi (Options That Control Optimization):
+	Update documentation for -ftree-loop-vectorize and
+	-ftree-slp-vectorize which are enabled by default at -02.
+
+2021-11-08  liuhongt  <hongtao.liu@intel.com>
+
+	PR target/102464
+	* match.pd (Simplifcation (trunc)copysign((extend)a, (extend)b)
+	to .COPYSIGN (a, b)): Add !HONOR_SNANS.
+
+2021-11-08  liuhongt  <hongtao.liu@intel.com>
+
+	PR target/102464
+	* match.pd: Simplify
+	(trunc)fma ((extend)a, (extend)b, (extend)c) to IFN_FMA (a, b,
+	c) under flag_unsafe_math_optimizations.
+
+2021-11-07  Jan Hubicka  <hubicka@ucw.cz>
+
+	PR ipa/103055
+	* params.opt (modref-max-depth): Add range.
+	(modref-max-adjustments): Fix range.
+
+2021-11-07  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-pass.h (make_pass_vrp_threader): Remove.
+	* tree-ssa-threadbackward.c
+	(back_threader_profitability::profitable_path_p): Remove
+	ASSERT_EXPR references.
+	* tree-ssa-threadedge.c (jt_state::register_equivs_stmt): Same.
+	* tree-vrp.c (vrp_folder::simplify_casted_conds): Same.
+	(execute_vrp): Same.
+	(class hybrid_threader): Remove.
+	(hybrid_threader::hybrid_threader): Remove.
+	(hybrid_threader::~hybrid_threader): Remove.
+	(hybrid_threader::before_dom_children): Remove.
+	(hybrid_threader::after_dom_children): Remove.
+	(execute_vrp_threader): Remove.
+	(class pass_vrp_threader): Remove.
+	(make_pass_vrp_threader): Remove.
+
+2021-11-07  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gimple.c (gimple_call_arg_flags): Use interposable_eaf_flags.
+	(gimple_call_retslot_flags): Likewise.
+	(gimple_call_static_chain_flags): Likewise.
+	* ipa-modref.c (remove_useless_eaf_flags): Do not remove everything for
+	NOVOPS.
+	(modref_summary::useful_p): Likewise.
+	(modref_summary_lto::useful_p): Likewise.
+	(analyze_parms): Do not give up on NOVOPS.
+	(analyze_function): When dumping report chnages in EAF flags
+	between IPA and local pass.
+	(modref_merge_call_site_flags): Compute implicit eaf flags
+	based on callee ecf_flags and fnspec; if the function does not
+	bind to current defs use interposable_eaf_flags.
+	(modref_propagate_flags_in_scc): Update.
+	* ipa-modref.h (interposable_eaf_flags): New function.
+
+2021-11-07  Bill Schmidt  <wschmidt@linux.ibm.com>
+
+	* config/rs6000/rs6000-call.c (rs6000_expand_new_builtin): New
+	forward decl.
+	(rs6000_invalid_new_builtin): New function.
+	(rs6000_expand_builtin): Call rs6000_expand_new_builtin.
+	(rs6000_expand_ldst_mask): New function.
+	(new_cpu_expand_builtin): Likewise.
+	(elemrev_icode): Likewise.
+	(ldv_expand_builtin): Likewise.
+	(lxvrse_expand_builtin): Likewise.
+	(lxvrze_expand_builtin): Likewise.
+	(stv_expand_builtin): Likewise.
+	(new_mma_expand_builtin): Likewise.
+	(new_htm_spr_num): Likewise.
+	(new_htm_expand_builtin): Likewise.
+	(rs6000_expand_new_builtin): Likewise.
+	(rs6000_init_builtins): Initialize altivec_builtin_mask_for_load.
+
+2021-11-07  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref.c (modref_lattice): Add do_dataflow,
+	changed and propagate_to fields.
+	(modref_lattice::release): Free propagate_to
+	(modref_lattice::merge): Do not give up early on unknown
+	lattice values.
+	(modref_lattice::merge_deref): Likewise.
+	(modref_eaf_analysis): Update toplevel comment.
+	(modref_eaf_analysis::analyze_ssa_name): Record postponned ssa names;
+	do optimistic dataflow initialization.
+	(modref_eaf_analysis::merge_with_ssa_name): Build dataflow graph.
+	(modref_eaf_analysis::propagate): New member function.
+	(analyze_parms): Update to new API of modref_eaf_analysis.
+
+2021-11-06  Jan Hubicka  <hubicka@ucw.cz>
+
+	* cgraph.h (cgraph_node::can_be_discarded_p): Do not
+	return true on functions from other partition.
+
+2021-11-06  David Edelsohn  <dje.gcc@gmail.com>
+
+	* config/rs6000/vsx.md (vsx_splat_v4si): Change constraints to "wa".
+	(vsx_splat_v4si_di): Change constraint to "wa".
+
+2021-11-06  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/103061
+	* value-relation.cc (path_oracle::path_oracle): Initialize
+	m_killed_defs.
+	(path_oracle::killing_def): Set m_killed_defs.
+	(path_oracle::query_relation): Do not look at the root oracle for
+	killed defs.
+	* value-relation.h (class path_oracle): Add m_killed_defs.
+
+2021-11-06  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c (back_threader::find_paths_to_names):
+	Remove gotos and other cleanups.
+
+2021-11-05  Jan Hubicka  <hubicka@ucw.cz>
+
+	PR ipa/103073
+	* ipa-modref-tree.h (modref_tree::insert): Do nothing for
+	paradoxical and zero sized accesses.
+
+2021-11-05  Jan Hubicka  <hubicka@ucw.cz>
+
+	PR ipa/103082
+	* ipa-modref-tree.h (struct modref_access_node): Avoid left shift
+	of negative value
+
+2021-11-05  Iain Sandoe  <iain@sandoe.co.uk>
+
+	* config/darwin.h (ASM_GENERATE_INTERNAL_LABEL): Add LTRAMP
+	to the list of symbol prefixes that must be made linker-
+	visible.
+
+2021-11-05  Iain Sandoe  <iain@sandoe.co.uk>
+
+	* config.host: Add support for aarch64-*-darwin.
+	* config/aarch64/host-aarch64-darwin.c: New file.
+	* config/aarch64/x-darwin: New file.
+
+2021-11-05  Andrew MacLeod  <amacleod@redhat.com>
+
+	PR tree-optimization/103093
+	* gimple-range-gori.cc (range_def_chain::get_imports): Remove assert.
+
+2021-11-05  Andrew MacLeod  <amacleod@redhat.com>
+
+	PR tree-optimization/102943
+	* gimple-range-cache.cc (class update_list): New.
+	(update_list::add): Replace add_to_update.
+	(update_list::pop): New.
+	(ranger_cache::ranger_cache): Adjust.
+	(ranger_cache::~ranger_cache): Adjust.
+	(ranger_cache::add_to_update): Delete.
+	(ranger_cache::propagate_cache): Adjust to new class.
+	(ranger_cache::propagate_updated_value): Ditto.
+	(ranger_cache::fill_block_cache): Ditto.
+	* gimple-range-cache.h (class ranger_cache): Adjust to update class.
+
+2021-11-05  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.c (vect_analyze_loop): Remove obsolete
+	comment and expand on another one.  Combine nested if.
+
+2021-11-05  John David Anglin  <danglin@gcc.gnu.org>
+
+	PR libgomp/96661
+	* config/pa/pa-modes.def: Add OImode integer type.
+	* config/pa/pa.c (pa_scalar_mode_supported_p): Allow TImode
+	for TARGET_64BIT.
+	* config/pa/pa.h (MIN_UNITS_PER_WORD) Define to MIN_UNITS_PER_WORD
+	to UNITS_PER_WORD if IN_LIBGCC2.
+	* config/pa/pa.md (addti3, addvti3, subti3, subvti3, negti2,
+	negvti2, ashlti3, shrpd_internal): New patterns.
+	Change some multi instruction types to multi.
+
+2021-11-05  Jakub Jelinek  <jakub@redhat.com>
+
+	PR bootstrap/100246
+	* config/i386/i386.h
+	(stringop_algs::stringop_strategy::stringop_strategy): Make the ctor
+	constexpr.
+
+2021-11-05  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	PR target/103085
+	* config/aarch64/aarch64.c (aarch64_mov_operand_p): Strip the salt
+	first.
+	* config/aarch64/constraints.md: Support const in Usw.
+
+2021-11-05  John David Anglin  <danglin@gcc.gnu.org>
+
+	* config/pa/pa.h (PREFERRED_DEBUGGING_TYPE): Define to DWARF2_DEBUG.
+	* config/pa/pa64-hpux.h (PREFERRED_DEBUGGING_TYPE): Remove define.
+
+2021-11-05  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vec_info_shared::n_stmts): Add.
+	(LOOP_VINFO_N_STMTS): Likewise.
+	(vec_info_for_bb): Remove unused function.
+	* tree-vectorizer.c (vec_info_shared::vec_info_shared):
+	Initialize n_stmts member.
+	* tree-vect-loop.c: Remove INCLUDE_FUNCTIONAL.
+	(vect_create_loop_vinfo): Do not set loop->aux.
+	(vect_analyze_loop_2): Do not get n_stmts as argument,
+	instead use LOOP_VINFO_N_STMTS.  Set LOOP_VINFO_VECTORIZABLE_P
+	here.
+	(vect_analyze_loop_1): Remove callback, get the mode iterator
+	and autodetected_vector_mode as argument, advancing the
+	iterator and initializing autodetected_vector_mode here.
+	(vect_analyze_loop): Split analysis loop into two, first
+	processing main loops only and then epilogues.
+
+2021-11-05  Martin Jambor  <mjambor@suse.cz>
+
+	* ipa-prop.c (compute_complex_assign_jump_func): Remove
+	unnecessary check for RECORD_TYPE.
+
+2021-11-05  Gerald Pfeifer  <gerald@pfeifer.com>
+
+	* doc/install.texi2html: Do not generate old.html any longer.
+
+2021-11-05  Martin Liska  <mliska@suse.cz>
+
+	PR debug/102955
+	* opts.c (finish_options): Reset flag_gtoggle when it is used.
+
+2021-11-05  Jakub Jelinek  <jakub@redhat.com>
+
+	PR debug/103046
+	* dwarf2out.c (add_const_value_attribute): Add MODE argument, use it
+	in CONST_WIDE_INT handling.  Adjust recursive calls.
+	(add_location_or_const_value_attribute): Pass DECL_MODE (decl) to
+	new add_const_value_attribute argument.
+	(tree_add_const_value_attribute): Pass TYPE_MODE (type) to new
+	add_const_value_attribute argument.
+
+2021-11-05  Rasmus Villemoes  <rasmus.villemoes@prevas.dk>
+
+	* config/vx-common.h: Test value of TARGET_VXWORKS7 rather
+	than definedness.
+
+2021-11-05  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (struct vect_loop_form_info): New.
+	(vect_analyze_loop_form): Adjust.
+	(vect_create_loop_vinfo): New.
+	* tree-parloops.c (gather_scalar_reductions): Adjust for
+	vect_analyze_loop_form API change.
+	* tree-vect-loop.c: Include <functional>.
+	(vect_analyze_loop_form_1): Rename to vect_analyze_loop_form,
+	take struct vect_loop_form_info as output parameter and adjust.
+	(vect_analyze_loop_form): Rename to vect_create_loop_vinfo and
+	split out call to the original vect_analyze_loop_form_1.
+	(vect_reanalyze_as_main_loop): Rename to...
+	(vect_analyze_loop_1): ... this, factor out the call to
+	vect_analyze_loop_form and generalize to be able to use it twice ...
+	(vect_analyze_loop): ... here.  Perform vect_analyze_loop_form
+	once only and here.
+
+2021-11-05  Xionghu Luo  <luoxhu@linux.ibm.com>
+
+	PR target/102991
+	* config/rs6000/fusion.md: Regenerate.
+	* config/rs6000/genfusion.pl: Fix incorrect clobber constraint.
+
+2021-11-04  Andreas Krebbel  <krebbel@linux.ibm.com>
+
+	* config/s390/s390.h (STACK_CHECK_MOVING_SP): New macro
+	definition.
+
+2021-11-04  Tamar Christina  <tamar.christina@arm.com>
+
+	* config/aarch64/aarch64-builtins.c
+	(aarch64_general_gimple_fold_builtin): Add ashl, sshl, ushl, ashr,
+	ashr_simd, lshr, lshr_simd.
+	* config/aarch64/aarch64-simd-builtins.def (lshr): Use USHIFTIMM.
+	* config/aarch64/arm_neon.h (vshr_n_u8, vshr_n_u16, vshr_n_u32,
+	vshrq_n_u8, vshrq_n_u16, vshrq_n_u32, vshrq_n_u64): Fix type hack.
+
+2021-11-04  Tamar Christina  <tamar.christina@arm.com>
+
+	* match.pd: New negate+shift pattern.
+
+2021-11-04  Andrew MacLeod  <amacleod@redhat.com>
+
+	PR tree-optimization/103079
+	* gimple-range-gori.cc (gimple_range_calc_op1): Treat undefined as
+	varying.
+	(gimple_range_calc_op2): Ditto.
+
+2021-11-04  Martin Jambor  <mjambor@suse.cz>
+
+	PR ipa/93385
+	* ipa-param-manipulation.h (class ipa_param_body_adjustments): New
+	members remap_with_debug_expressions, m_dead_ssa_debug_equiv,
+	m_dead_stmt_debug_equiv and prepare_debug_expressions.  Added
+	parameter to mark_dead_statements.
+	* ipa-param-manipulation.c: Include tree-phinodes.h and cfgexpand.h.
+	(ipa_param_body_adjustments::mark_dead_statements): New parameter
+	debugstack, push into it all SSA names used in debug statements,
+	produce m_dead_ssa_debug_equiv mapping for the removed param.
+	(replace_with_mapped_expr): New function.
+	(ipa_param_body_adjustments::remap_with_debug_expressions): Likewise.
+	(ipa_param_body_adjustments::prepare_debug_expressions): Likewise.
+	(ipa_param_body_adjustments::common_initialization): Gather and
+	procecc SSA which will be removed but are in debug statements. Simplify.
+	(ipa_param_body_adjustments::ipa_param_body_adjustments): Initialize
+	new members.
+	* tree-inline.c (remap_gimple_stmt): Create a debug bind when possible
+	when avoiding a copy of an unnecessary statement.  Remap removed SSA
+	names in existing debug statements.
+	(tree_function_versioning): Do not create DEBUG_EXPR_DECL for removed
+	parameters if we have already done so.
+
+2021-11-04  Jan Hubicka  <hubicka@ucw.cz>
+
+	PR ipa/103058
+	* gimple.c (gimple_call_static_chain_flags): Handle case when
+	nested function does not bind locally.
+
+2021-11-04  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_function_value): Generate
+	a register rtx for Neon vector-tuple modes.
+	(aarch64_layout_arg): Likewise.
+
+2021-11-04  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* lower-subreg.c (simple_move): Prevent decomposition if
+	modes are not tieable.
+
+2021-11-04  Jonathan Wright  <jonathan.wright@arm.com>
+	    Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-builtins.c (v2x8qi_UP): Define.
+	(v2x4hi_UP): Likewise.
+	(v2x4hf_UP): Likewise.
+	(v2x4bf_UP): Likewise.
+	(v2x2si_UP): Likewise.
+	(v2x2sf_UP): Likewise.
+	(v2x1di_UP): Likewise.
+	(v2x1df_UP): Likewise.
+	(v2x16qi_UP): Likewise.
+	(v2x8hi_UP): Likewise.
+	(v2x8hf_UP): Likewise.
+	(v2x8bf_UP): Likewise.
+	(v2x4si_UP): Likewise.
+	(v2x4sf_UP): Likewise.
+	(v2x2di_UP): Likewise.
+	(v2x2df_UP): Likewise.
+	(v3x8qi_UP): Likewise.
+	(v3x4hi_UP): Likewise.
+	(v3x4hf_UP): Likewise.
+	(v3x4bf_UP): Likewise.
+	(v3x2si_UP): Likewise.
+	(v3x2sf_UP): Likewise.
+	(v3x1di_UP): Likewise.
+	(v3x1df_UP): Likewise.
+	(v3x16qi_UP): Likewise.
+	(v3x8hi_UP): Likewise.
+	(v3x8hf_UP): Likewise.
+	(v3x8bf_UP): Likewise.
+	(v3x4si_UP): Likewise.
+	(v3x4sf_UP): Likewise.
+	(v3x2di_UP): Likewise.
+	(v3x2df_UP): Likewise.
+	(v4x8qi_UP): Likewise.
+	(v4x4hi_UP): Likewise.
+	(v4x4hf_UP): Likewise.
+	(v4x4bf_UP): Likewise.
+	(v4x2si_UP): Likewise.
+	(v4x2sf_UP): Likewise.
+	(v4x1di_UP): Likewise.
+	(v4x1df_UP): Likewise.
+	(v4x16qi_UP): Likewise.
+	(v4x8hi_UP): Likewise.
+	(v4x8hf_UP): Likewise.
+	(v4x8bf_UP): Likewise.
+	(v4x4si_UP): Likewise.
+	(v4x4sf_UP): Likewise.
+	(v4x2di_UP): Likewise.
+	(v4x2df_UP): Likewise.
+	(TYPES_GETREGP): Delete.
+	(TYPES_SETREGP): Likewise.
+	(TYPES_LOADSTRUCT_U): Define.
+	(TYPES_LOADSTRUCT_P): Likewise.
+	(TYPES_LOADSTRUCT_LANE_U): Likewise.
+	(TYPES_LOADSTRUCT_LANE_P): Likewise.
+	(TYPES_STORE1P): Move for consistency.
+	(TYPES_STORESTRUCT_U): Define.
+	(TYPES_STORESTRUCT_P): Likewise.
+	(TYPES_STORESTRUCT_LANE_U): Likewise.
+	(TYPES_STORESTRUCT_LANE_P): Likewise.
+	(aarch64_simd_tuple_types): Define.
+	(aarch64_lookup_simd_builtin_type): Handle tuple type lookup.
+	(aarch64_init_simd_builtin_functions): Update frontend lookup
+	for builtin functions after handling arm_neon.h pragma.
+	(register_tuple_type): Manually set modes of single-integer
+	tuple types. Record tuple types.
+	* config/aarch64/aarch64-modes.def
+	(ADV_SIMD_D_REG_STRUCT_MODES): Define D-register tuple modes.
+	(ADV_SIMD_Q_REG_STRUCT_MODES): Define Q-register tuple modes.
+	(SVE_MODES): Give single-vector modes priority over vector-
+	tuple modes.
+	(VECTOR_MODES_WITH_PREFIX): Set partial-vector mode order to
+	be after all single-vector modes.
+	* config/aarch64/aarch64-simd-builtins.def: Update builtin
+	generator macros to reflect modifications to the backend
+	patterns.
+	* config/aarch64/aarch64-simd.md (aarch64_simd_ld2<mode>):
+	Use vector-tuple mode iterator and rename to...
+	(aarch64_simd_ld2<vstruct_elt>): This.
+	(aarch64_simd_ld2r<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_simd_ld2r<vstruct_elt>): This.
+	(aarch64_vec_load_lanesoi_lane<mode>): Use vector-tuple mode
+	iterator and rename to...
+	(aarch64_vec_load_lanes<mode>_lane<vstruct_elt>): This.
+	(vec_load_lanesoi<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(vec_load_lanes<mode><vstruct_elt>): This.
+	(aarch64_simd_st2<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_simd_st2<vstruct_elt>): This.
+	(aarch64_vec_store_lanesoi_lane<mode>): Use vector-tuple mode
+	iterator and rename to...
+	(aarch64_vec_store_lanes<mode>_lane<vstruct_elt>): This.
+	(vec_store_lanesoi<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(vec_store_lanes<mode><vstruct_elt>): This.
+	(aarch64_simd_ld3<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_simd_ld3<vstruct_elt>): This.
+	(aarch64_simd_ld3r<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_simd_ld3r<vstruct_elt>): This.
+	(aarch64_vec_load_lanesci_lane<mode>): Use vector-tuple mode
+	iterator and rename to...
+	(vec_load_lanesci<mode>): This.
+	(aarch64_simd_st3<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_simd_st3<vstruct_elt>): This.
+	(aarch64_vec_store_lanesci_lane<mode>): Use vector-tuple mode
+	iterator and rename to...
+	(vec_store_lanesci<mode>): This.
+	(aarch64_simd_ld4<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_simd_ld4<vstruct_elt>): This.
+	(aarch64_simd_ld4r<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_simd_ld4r<vstruct_elt>): This.
+	(aarch64_vec_load_lanesxi_lane<mode>): Use vector-tuple mode
+	iterator and rename to...
+	(vec_load_lanesxi<mode>): This.
+	(aarch64_simd_st4<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_simd_st4<vstruct_elt>): This.
+	(aarch64_vec_store_lanesxi_lane<mode>): Use vector-tuple mode
+	iterator and rename to...
+	(vec_store_lanesxi<mode>): This.
+	(mov<mode>): Define for Neon vector-tuple modes.
+	(aarch64_ld1x3<VALLDIF:mode>): Use vector-tuple mode iterator
+	and rename to...
+	(aarch64_ld1x3<vstruct_elt>): This.
+	(aarch64_ld1_x3_<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_ld1_x3_<vstruct_elt>): This.
+	(aarch64_ld1x4<VALLDIF:mode>): Use vector-tuple mode iterator
+	and rename to...
+	(aarch64_ld1x4<vstruct_elt>): This.
+	(aarch64_ld1_x4_<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_ld1_x4_<vstruct_elt>): This.
+	(aarch64_st1x2<VALLDIF:mode>): Use vector-tuple mode iterator
+	and rename to...
+	(aarch64_st1x2<vstruct_elt>): This.
+	(aarch64_st1_x2_<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_st1_x2_<vstruct_elt>): This.
+	(aarch64_st1x3<VALLDIF:mode>): Use vector-tuple mode iterator
+	and rename to...
+	(aarch64_st1x3<vstruct_elt>): This.
+	(aarch64_st1_x3_<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_st1_x3_<vstruct_elt>): This.
+	(aarch64_st1x4<VALLDIF:mode>): Use vector-tuple mode iterator
+	and rename to...
+	(aarch64_st1x4<vstruct_elt>): This.
+	(aarch64_st1_x4_<mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_st1_x4_<vstruct_elt>): This.
+	(*aarch64_mov<mode>): Define for vector-tuple modes.
+	(*aarch64_be_mov<mode>): Likewise.
+	(aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>): Use vector-tuple
+	mode iterator and rename to...
+	(aarch64_ld<nregs>r<vstruct_elt>): This.
+	(aarch64_ld2<mode>_dreg): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_ld2<vstruct_elt>_dreg): This.
+	(aarch64_ld3<mode>_dreg): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_ld3<vstruct_elt>_dreg): This.
+	(aarch64_ld4<mode>_dreg): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_ld4<vstruct_elt>_dreg): This.
+	(aarch64_ld<VSTRUCT:nregs><VDC:mode>): Use vector-tuple mode
+	iterator and rename to...
+	(aarch64_ld<nregs><vstruct_elt>): Use vector-tuple mode
+	iterator and rename to...
+	(aarch64_ld<VSTRUCT:nregs><VQ:mode>): Use vector-tuple mode
+	(aarch64_ld1x2<VQ:mode>): Delete.
+	(aarch64_ld1x2<VDC:mode>): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_ld1x2<vstruct_elt>): This.
+	(aarch64_ld<VSTRUCT:nregs>_lane<VALLDIF:mode>): Use vector-
+	tuple mode iterator and rename to...
+	(aarch64_ld<nregs>_lane<vstruct_elt>): This.
+	(aarch64_get_dreg<VSTRUCT:mode><VDC:mode>): Delete.
+	(aarch64_get_qreg<VSTRUCT:mode><VQ:mode>): Likewise.
+	(aarch64_st2<mode>_dreg): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_st2<vstruct_elt>_dreg): This.
+	(aarch64_st3<mode>_dreg): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_st3<vstruct_elt>_dreg): This.
+	(aarch64_st4<mode>_dreg): Use vector-tuple mode iterator and
+	rename to...
+	(aarch64_st4<vstruct_elt>_dreg): This.
+	(aarch64_st<VSTRUCT:nregs><VDC:mode>): Use vector-tuple mode
+	iterator and rename to...
+	(aarch64_st<nregs><vstruct_elt>): This.
+	(aarch64_st<VSTRUCT:nregs><VQ:mode>): Use vector-tuple mode
+	iterator and rename to aarch64_st<nregs><vstruct_elt>.
+	(aarch64_st<VSTRUCT:nregs>_lane<VALLDIF:mode>): Use vector-
+	tuple mode iterator and rename to...
+	(aarch64_st<nregs>_lane<vstruct_elt>): This.
+	(aarch64_set_qreg<VSTRUCT:mode><VQ:mode>): Delete.
+	(aarch64_simd_ld1<mode>_x2): Use vector-tuple mode iterator
+	and rename to...
+	(aarch64_simd_ld1<vstruct_elt>_x2): This.
+	* config/aarch64/aarch64.c (aarch64_advsimd_struct_mode_p):
+	Refactor to include new vector-tuple modes.
+	(aarch64_classify_vector_mode): Add cases for new vector-
+	tuple modes.
+	(aarch64_advsimd_partial_struct_mode_p): Define.
+	(aarch64_advsimd_full_struct_mode_p): Likewise.
+	(aarch64_advsimd_vector_array_mode): Likewise.
+	(aarch64_sve_data_mode): Change location in file.
+	(aarch64_array_mode): Handle case of Neon vector-tuple modes.
+	(aarch64_hard_regno_nregs): Handle case of partial Neon
+	vector structures.
+	(aarch64_classify_address): Refactor to include handling of
+	Neon vector-tuple modes.
+	(aarch64_print_operand): Print "d" for "%R" for a partial
+	Neon vector structure.
+	(aarch64_expand_vec_perm_1): Use new vector-tuple mode.
+	(aarch64_modes_tieable_p): Prevent tieing Neon partial struct
+	modes with scalar machines modes larger than 8 bytes.
+	(aarch64_can_change_mode_class): Don't allow changes between
+	partial and full Neon vector-structure modes.
+	* config/aarch64/arm_neon.h (vst2_lane_f16): Use updated
+	builtin and remove boiler-plate code for opaque mode.
+	(vst2_lane_f32): Likewise.
+	(vst2_lane_f64): Likewise.
+	(vst2_lane_p8): Likewise.
+	(vst2_lane_p16): Likewise.
+	(vst2_lane_p64): Likewise.
+	(vst2_lane_s8): Likewise.
+	(vst2_lane_s16): Likewise.
+	(vst2_lane_s32): Likewise.
+	(vst2_lane_s64): Likewise.
+	(vst2_lane_u8): Likewise.
+	(vst2_lane_u16): Likewise.
+	(vst2_lane_u32): Likewise.
+	(vst2_lane_u64): Likewise.
+	(vst2q_lane_f16): Likewise.
+	(vst2q_lane_f32): Likewise.
+	(vst2q_lane_f64): Likewise.
+	(vst2q_lane_p8): Likewise.
+	(vst2q_lane_p16): Likewise.
+	(vst2q_lane_p64): Likewise.
+	(vst2q_lane_s8): Likewise.
+	(vst2q_lane_s16): Likewise.
+	(vst2q_lane_s32): Likewise.
+	(vst2q_lane_s64): Likewise.
+	(vst2q_lane_u8): Likewise.
+	(vst2q_lane_u16): Likewise.
+	(vst2q_lane_u32): Likewise.
+	(vst2q_lane_u64): Likewise.
+	(vst3_lane_f16): Likewise.
+	(vst3_lane_f32): Likewise.
+	(vst3_lane_f64): Likewise.
+	(vst3_lane_p8): Likewise.
+	(vst3_lane_p16): Likewise.
+	(vst3_lane_p64): Likewise.
+	(vst3_lane_s8): Likewise.
+	(vst3_lane_s16): Likewise.
+	(vst3_lane_s32): Likewise.
+	(vst3_lane_s64): Likewise.
+	(vst3_lane_u8): Likewise.
+	(vst3_lane_u16): Likewise.
+	(vst3_lane_u32): Likewise.
+	(vst3_lane_u64): Likewise.
+	(vst3q_lane_f16): Likewise.
+	(vst3q_lane_f32): Likewise.
+	(vst3q_lane_f64): Likewise.
+	(vst3q_lane_p8): Likewise.
+	(vst3q_lane_p16): Likewise.
+	(vst3q_lane_p64): Likewise.
+	(vst3q_lane_s8): Likewise.
+	(vst3q_lane_s16): Likewise.
+	(vst3q_lane_s32): Likewise.
+	(vst3q_lane_s64): Likewise.
+	(vst3q_lane_u8): Likewise.
+	(vst3q_lane_u16): Likewise.
+	(vst3q_lane_u32): Likewise.
+	(vst3q_lane_u64): Likewise.
+	(vst4_lane_f16): Likewise.
+	(vst4_lane_f32): Likewise.
+	(vst4_lane_f64): Likewise.
+	(vst4_lane_p8): Likewise.
+	(vst4_lane_p16): Likewise.
+	(vst4_lane_p64): Likewise.
+	(vst4_lane_s8): Likewise.
+	(vst4_lane_s16): Likewise.
+	(vst4_lane_s32): Likewise.
+	(vst4_lane_s64): Likewise.
+	(vst4_lane_u8): Likewise.
+	(vst4_lane_u16): Likewise.
+	(vst4_lane_u32): Likewise.
+	(vst4_lane_u64): Likewise.
+	(vst4q_lane_f16): Likewise.
+	(vst4q_lane_f32): Likewise.
+	(vst4q_lane_f64): Likewise.
+	(vst4q_lane_p8): Likewise.
+	(vst4q_lane_p16): Likewise.
+	(vst4q_lane_p64): Likewise.
+	(vst4q_lane_s8): Likewise.
+	(vst4q_lane_s16): Likewise.
+	(vst4q_lane_s32): Likewise.
+	(vst4q_lane_s64): Likewise.
+	(vst4q_lane_u8): Likewise.
+	(vst4q_lane_u16): Likewise.
+	(vst4q_lane_u32): Likewise.
+	(vst4q_lane_u64): Likewise.
+	(vtbl3_s8): Likewise.
+	(vtbl3_u8): Likewise.
+	(vtbl3_p8): Likewise.
+	(vtbl4_s8): Likewise.
+	(vtbl4_u8): Likewise.
+	(vtbl4_p8): Likewise.
+	(vld1_u8_x3): Likewise.
+	(vld1_s8_x3): Likewise.
+	(vld1_u16_x3): Likewise.
+	(vld1_s16_x3): Likewise.
+	(vld1_u32_x3): Likewise.
+	(vld1_s32_x3): Likewise.
+	(vld1_u64_x3): Likewise.
+	(vld1_s64_x3): Likewise.
+	(vld1_f16_x3): Likewise.
+	(vld1_f32_x3): Likewise.
+	(vld1_f64_x3): Likewise.
+	(vld1_p8_x3): Likewise.
+	(vld1_p16_x3): Likewise.
+	(vld1_p64_x3): Likewise.
+	(vld1q_u8_x3): Likewise.
+	(vld1q_s8_x3): Likewise.
+	(vld1q_u16_x3): Likewise.
+	(vld1q_s16_x3): Likewise.
+	(vld1q_u32_x3): Likewise.
+	(vld1q_s32_x3): Likewise.
+	(vld1q_u64_x3): Likewise.
+	(vld1q_s64_x3): Likewise.
+	(vld1q_f16_x3): Likewise.
+	(vld1q_f32_x3): Likewise.
+	(vld1q_f64_x3): Likewise.
+	(vld1q_p8_x3): Likewise.
+	(vld1q_p16_x3): Likewise.
+	(vld1q_p64_x3): Likewise.
+	(vld1_u8_x2): Likewise.
+	(vld1_s8_x2): Likewise.
+	(vld1_u16_x2): Likewise.
+	(vld1_s16_x2): Likewise.
+	(vld1_u32_x2): Likewise.
+	(vld1_s32_x2): Likewise.
+	(vld1_u64_x2): Likewise.
+	(vld1_s64_x2): Likewise.
+	(vld1_f16_x2): Likewise.
+	(vld1_f32_x2): Likewise.
+	(vld1_f64_x2): Likewise.
+	(vld1_p8_x2): Likewise.
+	(vld1_p16_x2): Likewise.
+	(vld1_p64_x2): Likewise.
+	(vld1q_u8_x2): Likewise.
+	(vld1q_s8_x2): Likewise.
+	(vld1q_u16_x2): Likewise.
+	(vld1q_s16_x2): Likewise.
+	(vld1q_u32_x2): Likewise.
+	(vld1q_s32_x2): Likewise.
+	(vld1q_u64_x2): Likewise.
+	(vld1q_s64_x2): Likewise.
+	(vld1q_f16_x2): Likewise.
+	(vld1q_f32_x2): Likewise.
+	(vld1q_f64_x2): Likewise.
+	(vld1q_p8_x2): Likewise.
+	(vld1q_p16_x2): Likewise.
+	(vld1q_p64_x2): Likewise.
+	(vld1_s8_x4): Likewise.
+	(vld1q_s8_x4): Likewise.
+	(vld1_s16_x4): Likewise.
+	(vld1q_s16_x4): Likewise.
+	(vld1_s32_x4): Likewise.
+	(vld1q_s32_x4): Likewise.
+	(vld1_u8_x4): Likewise.
+	(vld1q_u8_x4): Likewise.
+	(vld1_u16_x4): Likewise.
+	(vld1q_u16_x4): Likewise.
+	(vld1_u32_x4): Likewise.
+	(vld1q_u32_x4): Likewise.
+	(vld1_f16_x4): Likewise.
+	(vld1q_f16_x4): Likewise.
+	(vld1_f32_x4): Likewise.
+	(vld1q_f32_x4): Likewise.
+	(vld1_p8_x4): Likewise.
+	(vld1q_p8_x4): Likewise.
+	(vld1_p16_x4): Likewise.
+	(vld1q_p16_x4): Likewise.
+	(vld1_s64_x4): Likewise.
+	(vld1_u64_x4): Likewise.
+	(vld1_p64_x4): Likewise.
+	(vld1q_s64_x4): Likewise.
+	(vld1q_u64_x4): Likewise.
+	(vld1q_p64_x4): Likewise.
+	(vld1_f64_x4): Likewise.
+	(vld1q_f64_x4): Likewise.
+	(vld2_s64): Likewise.
+	(vld2_u64): Likewise.
+	(vld2_f64): Likewise.
+	(vld2_s8): Likewise.
+	(vld2_p8): Likewise.
+	(vld2_p64): Likewise.
+	(vld2_s16): Likewise.
+	(vld2_p16): Likewise.
+	(vld2_s32): Likewise.
+	(vld2_u8): Likewise.
+	(vld2_u16): Likewise.
+	(vld2_u32): Likewise.
+	(vld2_f16): Likewise.
+	(vld2_f32): Likewise.
+	(vld2q_s8): Likewise.
+	(vld2q_p8): Likewise.
+	(vld2q_s16): Likewise.
+	(vld2q_p16): Likewise.
+	(vld2q_p64): Likewise.
+	(vld2q_s32): Likewise.
+	(vld2q_s64): Likewise.
+	(vld2q_u8): Likewise.
+	(vld2q_u16): Likewise.
+	(vld2q_u32): Likewise.
+	(vld2q_u64): Likewise.
+	(vld2q_f16): Likewise.
+	(vld2q_f32): Likewise.
+	(vld2q_f64): Likewise.
+	(vld3_s64): Likewise.
+	(vld3_u64): Likewise.
+	(vld3_f64): Likewise.
+	(vld3_s8): Likewise.
+	(vld3_p8): Likewise.
+	(vld3_s16): Likewise.
+	(vld3_p16): Likewise.
+	(vld3_s32): Likewise.
+	(vld3_u8): Likewise.
+	(vld3_u16): Likewise.
+	(vld3_u32): Likewise.
+	(vld3_f16): Likewise.
+	(vld3_f32): Likewise.
+	(vld3_p64): Likewise.
+	(vld3q_s8): Likewise.
+	(vld3q_p8): Likewise.
+	(vld3q_s16): Likewise.
+	(vld3q_p16): Likewise.
+	(vld3q_s32): Likewise.
+	(vld3q_s64): Likewise.
+	(vld3q_u8): Likewise.
+	(vld3q_u16): Likewise.
+	(vld3q_u32): Likewise.
+	(vld3q_u64): Likewise.
+	(vld3q_f16): Likewise.
+	(vld3q_f32): Likewise.
+	(vld3q_f64): Likewise.
+	(vld3q_p64): Likewise.
+	(vld4_s64): Likewise.
+	(vld4_u64): Likewise.
+	(vld4_f64): Likewise.
+	(vld4_s8): Likewise.
+	(vld4_p8): Likewise.
+	(vld4_s16): Likewise.
+	(vld4_p16): Likewise.
+	(vld4_s32): Likewise.
+	(vld4_u8): Likewise.
+	(vld4_u16): Likewise.
+	(vld4_u32): Likewise.
+	(vld4_f16): Likewise.
+	(vld4_f32): Likewise.
+	(vld4_p64): Likewise.
+	(vld4q_s8): Likewise.
+	(vld4q_p8): Likewise.
+	(vld4q_s16): Likewise.
+	(vld4q_p16): Likewise.
+	(vld4q_s32): Likewise.
+	(vld4q_s64): Likewise.
+	(vld4q_u8): Likewise.
+	(vld4q_u16): Likewise.
+	(vld4q_u32): Likewise.
+	(vld4q_u64): Likewise.
+	(vld4q_f16): Likewise.
+	(vld4q_f32): Likewise.
+	(vld4q_f64): Likewise.
+	(vld4q_p64): Likewise.
+	(vld2_dup_s8): Likewise.
+	(vld2_dup_s16): Likewise.
+	(vld2_dup_s32): Likewise.
+	(vld2_dup_f16): Likewise.
+	(vld2_dup_f32): Likewise.
+	(vld2_dup_f64): Likewise.
+	(vld2_dup_u8): Likewise.
+	(vld2_dup_u16): Likewise.
+	(vld2_dup_u32): Likewise.
+	(vld2_dup_p8): Likewise.
+	(vld2_dup_p16): Likewise.
+	(vld2_dup_p64): Likewise.
+	(vld2_dup_s64): Likewise.
+	(vld2_dup_u64): Likewise.
+	(vld2q_dup_s8): Likewise.
+	(vld2q_dup_p8): Likewise.
+	(vld2q_dup_s16): Likewise.
+	(vld2q_dup_p16): Likewise.
+	(vld2q_dup_s32): Likewise.
+	(vld2q_dup_s64): Likewise.
+	(vld2q_dup_u8): Likewise.
+	(vld2q_dup_u16): Likewise.
+	(vld2q_dup_u32): Likewise.
+	(vld2q_dup_u64): Likewise.
+	(vld2q_dup_f16): Likewise.
+	(vld2q_dup_f32): Likewise.
+	(vld2q_dup_f64): Likewise.
+	(vld2q_dup_p64): Likewise.
+	(vld3_dup_s64): Likewise.
+	(vld3_dup_u64): Likewise.
+	(vld3_dup_f64): Likewise.
+	(vld3_dup_s8): Likewise.
+	(vld3_dup_p8): Likewise.
+	(vld3_dup_s16): Likewise.
+	(vld3_dup_p16): Likewise.
+	(vld3_dup_s32): Likewise.
+	(vld3_dup_u8): Likewise.
+	(vld3_dup_u16): Likewise.
+	(vld3_dup_u32): Likewise.
+	(vld3_dup_f16): Likewise.
+	(vld3_dup_f32): Likewise.
+	(vld3_dup_p64): Likewise.
+	(vld3q_dup_s8): Likewise.
+	(vld3q_dup_p8): Likewise.
+	(vld3q_dup_s16): Likewise.
+	(vld3q_dup_p16): Likewise.
+	(vld3q_dup_s32): Likewise.
+	(vld3q_dup_s64): Likewise.
+	(vld3q_dup_u8): Likewise.
+	(vld3q_dup_u16): Likewise.
+	(vld3q_dup_u32): Likewise.
+	(vld3q_dup_u64): Likewise.
+	(vld3q_dup_f16): Likewise.
+	(vld3q_dup_f32): Likewise.
+	(vld3q_dup_f64): Likewise.
+	(vld3q_dup_p64): Likewise.
+	(vld4_dup_s64): Likewise.
+	(vld4_dup_u64): Likewise.
+	(vld4_dup_f64): Likewise.
+	(vld4_dup_s8): Likewise.
+	(vld4_dup_p8): Likewise.
+	(vld4_dup_s16): Likewise.
+	(vld4_dup_p16): Likewise.
+	(vld4_dup_s32): Likewise.
+	(vld4_dup_u8): Likewise.
+	(vld4_dup_u16): Likewise.
+	(vld4_dup_u32): Likewise.
+	(vld4_dup_f16): Likewise.
+	(vld4_dup_f32): Likewise.
+	(vld4_dup_p64): Likewise.
+	(vld4q_dup_s8): Likewise.
+	(vld4q_dup_p8): Likewise.
+	(vld4q_dup_s16): Likewise.
+	(vld4q_dup_p16): Likewise.
+	(vld4q_dup_s32): Likewise.
+	(vld4q_dup_s64): Likewise.
+	(vld4q_dup_u8): Likewise.
+	(vld4q_dup_u16): Likewise.
+	(vld4q_dup_u32): Likewise.
+	(vld4q_dup_u64): Likewise.
+	(vld4q_dup_f16): Likewise.
+	(vld4q_dup_f32): Likewise.
+	(vld4q_dup_f64): Likewise.
+	(vld4q_dup_p64): Likewise.
+	(vld2_lane_u8): Likewise.
+	(vld2_lane_u16): Likewise.
+	(vld2_lane_u32): Likewise.
+	(vld2_lane_u64): Likewise.
+	(vld2_lane_s8): Likewise.
+	(vld2_lane_s16): Likewise.
+	(vld2_lane_s32): Likewise.
+	(vld2_lane_s64): Likewise.
+	(vld2_lane_f16): Likewise.
+	(vld2_lane_f32): Likewise.
+	(vld2_lane_f64): Likewise.
+	(vld2_lane_p8): Likewise.
+	(vld2_lane_p16): Likewise.
+	(vld2_lane_p64): Likewise.
+	(vld2q_lane_u8): Likewise.
+	(vld2q_lane_u16): Likewise.
+	(vld2q_lane_u32): Likewise.
+	(vld2q_lane_u64): Likewise.
+	(vld2q_lane_s8): Likewise.
+	(vld2q_lane_s16): Likewise.
+	(vld2q_lane_s32): Likewise.
+	(vld2q_lane_s64): Likewise.
+	(vld2q_lane_f16): Likewise.
+	(vld2q_lane_f32): Likewise.
+	(vld2q_lane_f64): Likewise.
+	(vld2q_lane_p8): Likewise.
+	(vld2q_lane_p16): Likewise.
+	(vld2q_lane_p64): Likewise.
+	(vld3_lane_u8): Likewise.
+	(vld3_lane_u16): Likewise.
+	(vld3_lane_u32): Likewise.
+	(vld3_lane_u64): Likewise.
+	(vld3_lane_s8): Likewise.
+	(vld3_lane_s16): Likewise.
+	(vld3_lane_s32): Likewise.
+	(vld3_lane_s64): Likewise.
+	(vld3_lane_f16): Likewise.
+	(vld3_lane_f32): Likewise.
+	(vld3_lane_f64): Likewise.
+	(vld3_lane_p8): Likewise.
+	(vld3_lane_p16): Likewise.
+	(vld3_lane_p64): Likewise.
+	(vld3q_lane_u8): Likewise.
+	(vld3q_lane_u16): Likewise.
+	(vld3q_lane_u32): Likewise.
+	(vld3q_lane_u64): Likewise.
+	(vld3q_lane_s8): Likewise.
+	(vld3q_lane_s16): Likewise.
+	(vld3q_lane_s32): Likewise.
+	(vld3q_lane_s64): Likewise.
+	(vld3q_lane_f16): Likewise.
+	(vld3q_lane_f32): Likewise.
+	(vld3q_lane_f64): Likewise.
+	(vld3q_lane_p8): Likewise.
+	(vld3q_lane_p16): Likewise.
+	(vld3q_lane_p64): Likewise.
+	(vld4_lane_u8): Likewise.
+	(vld4_lane_u16): Likewise.
+	(vld4_lane_u32): Likewise.
+	(vld4_lane_u64): Likewise.
+	(vld4_lane_s8): Likewise.
+	(vld4_lane_s16): Likewise.
+	(vld4_lane_s32): Likewise.
+	(vld4_lane_s64): Likewise.
+	(vld4_lane_f16): Likewise.
+	(vld4_lane_f32): Likewise.
+	(vld4_lane_f64): Likewise.
+	(vld4_lane_p8): Likewise.
+	(vld4_lane_p16): Likewise.
+	(vld4_lane_p64): Likewise.
+	(vld4q_lane_u8): Likewise.
+	(vld4q_lane_u16): Likewise.
+	(vld4q_lane_u32): Likewise.
+	(vld4q_lane_u64): Likewise.
+	(vld4q_lane_s8): Likewise.
+	(vld4q_lane_s16): Likewise.
+	(vld4q_lane_s32): Likewise.
+	(vld4q_lane_s64): Likewise.
+	(vld4q_lane_f16): Likewise.
+	(vld4q_lane_f32): Likewise.
+	(vld4q_lane_f64): Likewise.
+	(vld4q_lane_p8): Likewise.
+	(vld4q_lane_p16): Likewise.
+	(vld4q_lane_p64): Likewise.
+	(vqtbl2_s8): Likewise.
+	(vqtbl2_u8): Likewise.
+	(vqtbl2_p8): Likewise.
+	(vqtbl2q_s8): Likewise.
+	(vqtbl2q_u8): Likewise.
+	(vqtbl2q_p8): Likewise.
+	(vqtbl3_s8): Likewise.
+	(vqtbl3_u8): Likewise.
+	(vqtbl3_p8): Likewise.
+	(vqtbl3q_s8): Likewise.
+	(vqtbl3q_u8): Likewise.
+	(vqtbl3q_p8): Likewise.
+	(vqtbl4_s8): Likewise.
+	(vqtbl4_u8): Likewise.
+	(vqtbl4_p8): Likewise.
+	(vqtbl4q_s8): Likewise.
+	(vqtbl4q_u8): Likewise.
+	(vqtbl4q_p8): Likewise.
+	(vqtbx2_s8): Likewise.
+	(vqtbx2_u8): Likewise.
+	(vqtbx2_p8): Likewise.
+	(vqtbx2q_s8): Likewise.
+	(vqtbx2q_u8): Likewise.
+	(vqtbx2q_p8): Likewise.
+	(vqtbx3_s8): Likewise.
+	(vqtbx3_u8): Likewise.
+	(vqtbx3_p8): Likewise.
+	(vqtbx3q_s8): Likewise.
+	(vqtbx3q_u8): Likewise.
+	(vqtbx3q_p8): Likewise.
+	(vqtbx4_s8): Likewise.
+	(vqtbx4_u8): Likewise.
+	(vqtbx4_p8): Likewise.
+	(vqtbx4q_s8): Likewise.
+	(vqtbx4q_u8): Likewise.
+	(vqtbx4q_p8): Likewise.
+	(vst1_s64_x2): Likewise.
+	(vst1_u64_x2): Likewise.
+	(vst1_f64_x2): Likewise.
+	(vst1_s8_x2): Likewise.
+	(vst1_p8_x2): Likewise.
+	(vst1_s16_x2): Likewise.
+	(vst1_p16_x2): Likewise.
+	(vst1_s32_x2): Likewise.
+	(vst1_u8_x2): Likewise.
+	(vst1_u16_x2): Likewise.
+	(vst1_u32_x2): Likewise.
+	(vst1_f16_x2): Likewise.
+	(vst1_f32_x2): Likewise.
+	(vst1_p64_x2): Likewise.
+	(vst1q_s8_x2): Likewise.
+	(vst1q_p8_x2): Likewise.
+	(vst1q_s16_x2): Likewise.
+	(vst1q_p16_x2): Likewise.
+	(vst1q_s32_x2): Likewise.
+	(vst1q_s64_x2): Likewise.
+	(vst1q_u8_x2): Likewise.
+	(vst1q_u16_x2): Likewise.
+	(vst1q_u32_x2): Likewise.
+	(vst1q_u64_x2): Likewise.
+	(vst1q_f16_x2): Likewise.
+	(vst1q_f32_x2): Likewise.
+	(vst1q_f64_x2): Likewise.
+	(vst1q_p64_x2): Likewise.
+	(vst1_s64_x3): Likewise.
+	(vst1_u64_x3): Likewise.
+	(vst1_f64_x3): Likewise.
+	(vst1_s8_x3): Likewise.
+	(vst1_p8_x3): Likewise.
+	(vst1_s16_x3): Likewise.
+	(vst1_p16_x3): Likewise.
+	(vst1_s32_x3): Likewise.
+	(vst1_u8_x3): Likewise.
+	(vst1_u16_x3): Likewise.
+	(vst1_u32_x3): Likewise.
+	(vst1_f16_x3): Likewise.
+	(vst1_f32_x3): Likewise.
+	(vst1_p64_x3): Likewise.
+	(vst1q_s8_x3): Likewise.
+	(vst1q_p8_x3): Likewise.
+	(vst1q_s16_x3): Likewise.
+	(vst1q_p16_x3): Likewise.
+	(vst1q_s32_x3): Likewise.
+	(vst1q_s64_x3): Likewise.
+	(vst1q_u8_x3): Likewise.
+	(vst1q_u16_x3): Likewise.
+	(vst1q_u32_x3): Likewise.
+	(vst1q_u64_x3): Likewise.
+	(vst1q_f16_x3): Likewise.
+	(vst1q_f32_x3): Likewise.
+	(vst1q_f64_x3): Likewise.
+	(vst1q_p64_x3): Likewise.
+	(vst1_s8_x4): Likewise.
+	(vst1q_s8_x4): Likewise.
+	(vst1_s16_x4): Likewise.
+	(vst1q_s16_x4): Likewise.
+	(vst1_s32_x4): Likewise.
+	(vst1q_s32_x4): Likewise.
+	(vst1_u8_x4): Likewise.
+	(vst1q_u8_x4): Likewise.
+	(vst1_u16_x4): Likewise.
+	(vst1q_u16_x4): Likewise.
+	(vst1_u32_x4): Likewise.
+	(vst1q_u32_x4): Likewise.
+	(vst1_f16_x4): Likewise.
+	(vst1q_f16_x4): Likewise.
+	(vst1_f32_x4): Likewise.
+	(vst1q_f32_x4): Likewise.
+	(vst1_p8_x4): Likewise.
+	(vst1q_p8_x4): Likewise.
+	(vst1_p16_x4): Likewise.
+	(vst1q_p16_x4): Likewise.
+	(vst1_s64_x4): Likewise.
+	(vst1_u64_x4): Likewise.
+	(vst1_p64_x4): Likewise.
+	(vst1q_s64_x4): Likewise.
+	(vst1q_u64_x4): Likewise.
+	(vst1q_p64_x4): Likewise.
+	(vst1_f64_x4): Likewise.
+	(vst1q_f64_x4): Likewise.
+	(vst2_s64): Likewise.
+	(vst2_u64): Likewise.
+	(vst2_f64): Likewise.
+	(vst2_s8): Likewise.
+	(vst2_p8): Likewise.
+	(vst2_s16): Likewise.
+	(vst2_p16): Likewise.
+	(vst2_s32): Likewise.
+	(vst2_u8): Likewise.
+	(vst2_u16): Likewise.
+	(vst2_u32): Likewise.
+	(vst2_f16): Likewise.
+	(vst2_f32): Likewise.
+	(vst2_p64): Likewise.
+	(vst2q_s8): Likewise.
+	(vst2q_p8): Likewise.
+	(vst2q_s16): Likewise.
+	(vst2q_p16): Likewise.
+	(vst2q_s32): Likewise.
+	(vst2q_s64): Likewise.
+	(vst2q_u8): Likewise.
+	(vst2q_u16): Likewise.
+	(vst2q_u32): Likewise.
+	(vst2q_u64): Likewise.
+	(vst2q_f16): Likewise.
+	(vst2q_f32): Likewise.
+	(vst2q_f64): Likewise.
+	(vst2q_p64): Likewise.
+	(vst3_s64): Likewise.
+	(vst3_u64): Likewise.
+	(vst3_f64): Likewise.
+	(vst3_s8): Likewise.
+	(vst3_p8): Likewise.
+	(vst3_s16): Likewise.
+	(vst3_p16): Likewise.
+	(vst3_s32): Likewise.
+	(vst3_u8): Likewise.
+	(vst3_u16): Likewise.
+	(vst3_u32): Likewise.
+	(vst3_f16): Likewise.
+	(vst3_f32): Likewise.
+	(vst3_p64): Likewise.
+	(vst3q_s8): Likewise.
+	(vst3q_p8): Likewise.
+	(vst3q_s16): Likewise.
+	(vst3q_p16): Likewise.
+	(vst3q_s32): Likewise.
+	(vst3q_s64): Likewise.
+	(vst3q_u8): Likewise.
+	(vst3q_u16): Likewise.
+	(vst3q_u32): Likewise.
+	(vst3q_u64): Likewise.
+	(vst3q_f16): Likewise.
+	(vst3q_f32): Likewise.
+	(vst3q_f64): Likewise.
+	(vst3q_p64): Likewise.
+	(vst4_s64): Likewise.
+	(vst4_u64): Likewise.
+	(vst4_f64): Likewise.
+	(vst4_s8): Likewise.
+	(vst4_p8): Likewise.
+	(vst4_s16): Likewise.
+	(vst4_p16): Likewise.
+	(vst4_s32): Likewise.
+	(vst4_u8): Likewise.
+	(vst4_u16): Likewise.
+	(vst4_u32): Likewise.
+	(vst4_f16): Likewise.
+	(vst4_f32): Likewise.
+	(vst4_p64): Likewise.
+	(vst4q_s8): Likewise.
+	(vst4q_p8): Likewise.
+	(vst4q_s16): Likewise.
+	(vst4q_p16): Likewise.
+	(vst4q_s32): Likewise.
+	(vst4q_s64): Likewise.
+	(vst4q_u8): Likewise.
+	(vst4q_u16): Likewise.
+	(vst4q_u32): Likewise.
+	(vst4q_u64): Likewise.
+	(vst4q_f16): Likewise.
+	(vst4q_f32): Likewise.
+	(vst4q_f64): Likewise.
+	(vst4q_p64): Likewise.
+	(vtbx4_s8): Likewise.
+	(vtbx4_u8): Likewise.
+	(vtbx4_p8): Likewise.
+	(vld1_bf16_x2): Likewise.
+	(vld1q_bf16_x2): Likewise.
+	(vld1_bf16_x3): Likewise.
+	(vld1q_bf16_x3): Likewise.
+	(vld1_bf16_x4): Likewise.
+	(vld1q_bf16_x4): Likewise.
+	(vld2_bf16): Likewise.
+	(vld2q_bf16): Likewise.
+	(vld2_dup_bf16): Likewise.
+	(vld2q_dup_bf16): Likewise.
+	(vld3_bf16): Likewise.
+	(vld3q_bf16): Likewise.
+	(vld3_dup_bf16): Likewise.
+	(vld3q_dup_bf16): Likewise.
+	(vld4_bf16): Likewise.
+	(vld4q_bf16): Likewise.
+	(vld4_dup_bf16): Likewise.
+	(vld4q_dup_bf16): Likewise.
+	(vst1_bf16_x2): Likewise.
+	(vst1q_bf16_x2): Likewise.
+	(vst1_bf16_x3): Likewise.
+	(vst1q_bf16_x3): Likewise.
+	(vst1_bf16_x4): Likewise.
+	(vst1q_bf16_x4): Likewise.
+	(vst2_bf16): Likewise.
+	(vst2q_bf16): Likewise.
+	(vst3_bf16): Likewise.
+	(vst3q_bf16): Likewise.
+	(vst4_bf16): Likewise.
+	(vst4q_bf16): Likewise.
+	(vld2_lane_bf16): Likewise.
+	(vld2q_lane_bf16): Likewise.
+	(vld3_lane_bf16): Likewise.
+	(vld3q_lane_bf16): Likewise.
+	(vld4_lane_bf16): Likewise.
+	(vld4q_lane_bf16): Likewise.
+	(vst2_lane_bf16): Likewise.
+	(vst2q_lane_bf16): Likewise.
+	(vst3_lane_bf16): Likewise.
+	(vst3q_lane_bf16): Likewise.
+	(vst4_lane_bf16): Likewise.
+	(vst4q_lane_bf16): Likewise.
+	* config/aarch64/geniterators.sh: Modify iterator regex to
+	match new vector-tuple modes.
+	* config/aarch64/iterators.md (insn_count): Extend mode
+	attribute with vector-tuple type information.
+	(nregs): Likewise.
+	(Vendreg): Likewise.
+	(Vetype): Likewise.
+	(Vtype): Likewise.
+	(VSTRUCT_2D): New mode iterator.
+	(VSTRUCT_2DNX): Likewise.
+	(VSTRUCT_2DX): Likewise.
+	(VSTRUCT_2Q): Likewise.
+	(VSTRUCT_2QD): Likewise.
+	(VSTRUCT_3D): Likewise.
+	(VSTRUCT_3DNX): Likewise.
+	(VSTRUCT_3DX): Likewise.
+	(VSTRUCT_3Q): Likewise.
+	(VSTRUCT_3QD): Likewise.
+	(VSTRUCT_4D): Likewise.
+	(VSTRUCT_4DNX): Likewise.
+	(VSTRUCT_4DX): Likewise.
+	(VSTRUCT_4Q): Likewise.
+	(VSTRUCT_4QD): Likewise.
+	(VSTRUCT_D): Likewise.
+	(VSTRUCT_Q): Likewise.
+	(VSTRUCT_QD): Likewise.
+	(VSTRUCT_ELT): New mode attribute.
+	(vstruct_elt): Likewise.
+	* genmodes.c (VECTOR_MODE): Add default prefix and order
+	parameters.
+	(VECTOR_MODE_WITH_PREFIX): Define.
+	(make_vector_mode): Add mode prefix and order parameters.
+
+2021-11-04  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* expmed.c (extract_bit_field_1): Ensure modes are tieable.
+
+2021-11-04  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* expr.c (emit_group_load_1): Remove historic workaround.
+
+2021-11-04  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/aarch64-builtins.c (aarch64_init_simd_builtins):
+	Factor out main loop to...
+	(aarch64_init_simd_builtin_functions): This new function.
+	(register_tuple_type): Define.
+	(aarch64_scalar_builtin_type_p): Define.
+	(handle_arm_neon_h): Define.
+	* config/aarch64/aarch64-c.c (aarch64_pragma_aarch64): Handle
+	pragma for arm_neon.h.
+	* config/aarch64/aarch64-protos.h (aarch64_advsimd_struct_mode_p):
+	Declare.
+	(handle_arm_neon_h): Likewise.
+	* config/aarch64/aarch64.c (aarch64_advsimd_struct_mode_p):
+	Remove static modifier.
+	* config/aarch64/arm_neon.h (target): Remove Neon vector
+	structure type definitions.
+
+2021-11-04  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/102943
+	* gimple-range-path.cc (path_range_query::range_on_path_entry):
+	Prefer range_of_expr unless there are no statements in the BB.
+
+2021-11-04  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/102943
+	* tree-ssa-threadbackward.c (back_threader::find_paths_to_names):
+	Avoid duplicate calculation of paths.
+
+2021-11-04  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/102943
+	* gimple-range-path.cc (path_range_query::compute_phi_relations):
+	Only compute relations for SSA names in the import list.
+	(path_range_query::compute_outgoing_relations): Same.
+	* gimple-range-path.h (path_range_query::import_p): New.
+
+2021-11-04  Richard Biener  <rguenther@suse.de>
+
+	PR rtl-optimization/103075
+	* simplify-rtx.c (exact_int_to_float_conversion_p): Return
+	false for a VOIDmode operand.
+
+2021-11-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_vector_costs): Make member
+	variables private and add "m_" to their names.  Remove is_loop.
+	(aarch64_record_potential_advsimd_unrolling): Replace with...
+	(aarch64_vector_costs::record_potential_advsimd_unrolling): ...this.
+	(aarch64_analyze_loop_vinfo): Replace with...
+	(aarch64_vector_costs::analyze_loop_vinfo): ...this.
+	Move initialization of (m_)vec_flags to add_stmt_cost.
+	(aarch64_analyze_bb_vinfo): Delete.
+	(aarch64_count_ops): Replace with...
+	(aarch64_vector_costs::count_ops): ...this.
+	(aarch64_vector_costs::add_stmt_cost): Set m_vec_flags,
+	using m_costing_for_scalar to test whether we're costing
+	scalar or vector code.
+	(aarch64_adjust_body_cost_sve): Replace with...
+	(aarch64_vector_costs::adjust_body_cost_sve): ...this.
+	(aarch64_adjust_body_cost): Replace with...
+	(aarch64_vector_costs::adjust_body_cost): ...this.
+	(aarch64_vector_costs::finish_cost): Use m_vinfo instead of is_loop.
+
+2021-11-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* target.def (targetm.vectorize.init_cost): Replace with...
+	(targetm.vectorize.create_costs): ...this.
+	(targetm.vectorize.add_stmt_cost): Delete.
+	(targetm.vectorize.finish_cost): Likewise.
+	(targetm.vectorize.destroy_cost_data): Likewise.
+	* doc/tm.texi.in (TARGET_VECTORIZE_INIT_COST): Replace with...
+	(TARGET_VECTORIZE_CREATE_COSTS): ...this.
+	(TARGET_VECTORIZE_ADD_STMT_COST): Delete.
+	(TARGET_VECTORIZE_FINISH_COST): Likewise.
+	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
+	* doc/tm.texi: Regenerate.
+	* tree-vectorizer.h (vec_info::vec_info): Remove target_cost_data
+	parameter.
+	(vec_info::target_cost_data): Change from a void * to a vector_costs *.
+	(vector_costs): New class.
+	(init_cost): Take a vec_info and return a vector_costs.
+	(dump_stmt_cost): Remove data parameter.
+	(add_stmt_cost): Replace vinfo and data parameters with a vector_costs.
+	(add_stmt_costs): Likewise.
+	(finish_cost): Replace data parameter with a vector_costs.
+	(destroy_cost_data): Delete.
+	* tree-vectorizer.c (dump_stmt_cost): Remove data argument and
+	don't print it.
+	(vec_info::vec_info): Remove the target_cost_data parameter and
+	initialize the member variable to null instead.
+	(vec_info::~vec_info): Delete target_cost_data instead of calling
+	destroy_cost_data.
+	(vector_costs::add_stmt_cost): New function.
+	(vector_costs::finish_cost): Likewise.
+	(vector_costs::record_stmt_cost): Likewise.
+	(vector_costs::adjust_cost_for_freq): Likewise.
+	* tree-vect-loop.c (_loop_vec_info::_loop_vec_info): Update
+	call to vec_info::vec_info.
+	(vect_compute_single_scalar_iteration_cost): Update after above
+	changes to costing interface.
+	(vect_analyze_loop_operations): Likewise.
+	(vect_estimate_min_profitable_iters): Likewise.
+	(vect_analyze_loop_2): Initialize LOOP_VINFO_TARGET_COST_DATA
+	at the start_over point, where it needs to be recreated after
+	trying without slp.  Update retry code accordingly.
+	* tree-vect-slp.c (_bb_vec_info::_bb_vec_info): Update call
+	to vec_info::vec_info.
+	(vect_slp_analyze_operation): Update after above changes to costing
+	interface.
+	(vect_bb_vectorization_profitable_p): Likewise.
+	* targhooks.h (default_init_cost): Replace with...
+	(default_vectorize_create_costs): ...this.
+	(default_add_stmt_cost): Delete.
+	(default_finish_cost, default_destroy_cost_data): Likewise.
+	* targhooks.c (default_init_cost): Replace with...
+	(default_vectorize_create_costs): ...this.
+	(default_add_stmt_cost): Delete, moving logic to vector_costs instead.
+	(default_finish_cost, default_destroy_cost_data): Delete.
+	* config/aarch64/aarch64.c (aarch64_vector_costs): Inherit from
+	vector_costs.  Add a constructor.
+	(aarch64_init_cost): Replace with...
+	(aarch64_vectorize_create_costs): ...this.
+	(aarch64_add_stmt_cost): Replace with...
+	(aarch64_vector_costs::add_stmt_cost): ...this.  Use record_stmt_cost
+	to adjust the cost for inner loops.
+	(aarch64_finish_cost): Replace with...
+	(aarch64_vector_costs::finish_cost): ...this.
+	(aarch64_destroy_cost_data): Delete.
+	(TARGET_VECTORIZE_INIT_COST): Replace with...
+	(TARGET_VECTORIZE_CREATE_COSTS): ...this.
+	(TARGET_VECTORIZE_ADD_STMT_COST): Delete.
+	(TARGET_VECTORIZE_FINISH_COST): Likewise.
+	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
+	* config/i386/i386.c (ix86_vector_costs): New structure.
+	(ix86_init_cost): Replace with...
+	(ix86_vectorize_create_costs): ...this.
+	(ix86_add_stmt_cost): Replace with...
+	(ix86_vector_costs::add_stmt_cost): ...this.  Use adjust_cost_for_freq
+	to adjust the cost for inner loops.
+	(ix86_finish_cost, ix86_destroy_cost_data): Delete.
+	(TARGET_VECTORIZE_INIT_COST): Replace with...
+	(TARGET_VECTORIZE_CREATE_COSTS): ...this.
+	(TARGET_VECTORIZE_ADD_STMT_COST): Delete.
+	(TARGET_VECTORIZE_FINISH_COST): Likewise.
+	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
+	* config/rs6000/rs6000.c (TARGET_VECTORIZE_INIT_COST): Replace with...
+	(TARGET_VECTORIZE_CREATE_COSTS): ...this.
+	(TARGET_VECTORIZE_ADD_STMT_COST): Delete.
+	(TARGET_VECTORIZE_FINISH_COST): Likewise.
+	(TARGET_VECTORIZE_DESTROY_COST_DATA): Likewise.
+	(rs6000_cost_data): Inherit from vector_costs.
+	Add a constructor.  Drop loop_info, cost and costing_for_scalar
+	in favor of the corresponding vector_costs member variables.
+	Add "m_" to the names of the remaining member variables and
+	initialize them.
+	(rs6000_density_test): Replace with...
+	(rs6000_cost_data::density_test): ...this.
+	(rs6000_init_cost): Replace with...
+	(rs6000_vectorize_create_costs): ...this.
+	(rs6000_update_target_cost_per_stmt): Replace with...
+	(rs6000_cost_data::update_target_cost_per_stmt): ...this.
+	(rs6000_add_stmt_cost): Replace with...
+	(rs6000_cost_data::add_stmt_cost): ...this.  Use adjust_cost_for_freq
+	to adjust the cost for inner loops.
+	(rs6000_adjust_vect_cost_per_loop): Replace with...
+	(rs6000_cost_data::adjust_vect_cost_per_loop): ...this.
+	(rs6000_finish_cost): Replace with...
+	(rs6000_cost_data::finish_cost): ...this.  Group loop code
+	into a single if statement and pass the loop_vinfo down to
+	subroutines.
+	(rs6000_destroy_cost_data): Delete.
+
+2021-11-04  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/103062
+	PR tree-optimization/103062
+	* value-pointer-equiv.cc (ssa_equiv_stack::ssa_equiv_stack):
+	Increase size of allocation by 1.
+	(ssa_equiv_stack::push_replacement): Grow as needed.
+	(ssa_equiv_stack::get_replacement): Same.
+	(pointer_equiv_analyzer::pointer_equiv_analyzer): Same.
+	(pointer_equiv_analyzer::~pointer_equiv_analyzer): Remove delete.
+	(pointer_equiv_analyzer::set_global_equiv): Grow as needed.
+	(pointer_equiv_analyzer::get_equiv): Same.
+	(pointer_equiv_analyzer::get_equiv_expr): Remove const.
+	* value-pointer-equiv.h (class pointer_equiv_analyzer): Remove
+	const markers.  Use auto_vec instead of tree *.
+
+2021-11-04  Richard Biener  <rguenther@suse.de>
+
+	* tree-ssa-sccvn.c (vn_nary_op_insert_into): Remove always
+	true parameter and inline valueization.
+	(vn_nary_op_lookup_1): Inline valueization from ...
+	(vn_nary_op_compute_hash): ... here and remove it here.
+	* tree-ssa-pre.c (phi_translate_1): Do not valueize
+	before vn_nary_lookup_pieces.
+	(get_representative_for): Mark created SSA representatives
+	as visited.
+
+2021-11-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* simplify-rtx.c (simplify_context::simplify_gen_vec_select): Assert
+	that the operand has a vector mode.  Use subreg_lowpart_offset
+	to test whether an index corresponds to the low part.
+
+2021-11-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* read-rtl.c: Remove dead !GENERATOR_FILE block.
+	* read-rtl-function.c (function_reader::consolidate_singletons):
+	Generate canonical CONST_VECTORs.
+
+2021-11-04  liuhongt  <hongtao.liu@intel.com>
+
+	PR target/101989
+	* config/i386/predicates.md (reg_or_notreg_operand): Rename to ..
+	(regmem_or_bitnot_regmem_operand): .. and extend to handle
+	memory_operand.
+	* config/i386/sse.md (*<avx512>_vpternlog<mode>_1): Force_reg
+	the operands which are required to be register_operand.
+	(*<avx512>_vpternlog<mode>_2): Ditto.
+	(*<avx512>_vpternlog<mode>_3): Ditto.
+	(*<avx512>_vternlog<mode>_all): Disallow embeded broadcast for
+	vector HFmodes since it's not a real AVX512FP16 instruction.
+
+2021-11-04  liuhongt  <hongtao.liu@intel.com>
+
+	PR target/102464
+	* match.pd: simplify (trunc)copysign((extend)a, (extend)b) to
+	.COPYSIGN (a,b) when a and b are same type as the truncation
+	type and has less precision than extend type.
+
+2021-11-04  Richard Biener  <rguenther@suse.de>
+
+	* doc/generic.texi: Update TARGET_MEM_REF and MEM_REF
+	documentation.
+
+2021-11-04  Hongyu Wang  <hongyu.wang@intel.com>
+
+	* config/i386/sse.md (VI2_AVX512VNNIBW): New mode iterator.
+	(VI1_AVX512VNNI): Likewise.
+	(SDOT_VPDP_SUF): New mode_attr.
+	(VI1SI): Likewise.
+	(vi1si): Likewise.
+	(sdot_prod<mode>): Use VI2_AVX512F iterator, expand to
+	vpdpwssd when VNNI targets available.
+	(usdot_prod<mode>): New expander for vector QImode.
+
+2021-11-04  Hongyu Wang  <hongyu.wang@intel.com>
+
+	* config/i386/amxtileintrin.h (_tile_loadd_internal): Add
+	parentheses to base and stride.
+	(_tile_stream_loadd_internal): Likewise.
+	(_tile_stored_internal): Likewise.
+
+2021-11-03  Maciej W. Rozycki  <macro@embecosm.com>
+
+	* config/riscv/riscv.c (riscv_class_max_nregs): Swap the
+	arguments to `reg_class_subset_p'.
+
+2021-11-03  Joseph Myers  <joseph@codesourcery.com>
+
+	PR c/103031
+	* fold-const.c (fold_init): New function.
+	* fold-const.h (fold_init): New prototype.
+
+2021-11-03  Andrew MacLeod  <amacleod@redhat.com>
+
+	* params.opt (param_vrp2_mode): Make ranger the default for VRP2.
+
+2021-11-03  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gimple-range-fold.cc (fold_using_range::range_of_phi): Don't import
+	a range from edge if arg == phidef.
+
+2021-11-03  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gimple-range-fold.cc (fold_using_range::range_of_builtin_call): Test
+	for constant before any other processing.
+
+2021-11-03  Andrew MacLeod  <amacleod@redhat.com>
+
+	* flag-types.h (RANGER_DEBUG_ALL): Fix values.
+
+2021-11-03  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gimple-range.cc (gimple_ranger::gimple_ranger): Initialize current_bb.
+	(gimple_ranger::range_of_expr): Pick up range_on_entry when there is
+	no explcit context and current_bb is set.
+	(gimple_ranger::fold_stmt): New.
+	* gimple-range.h (current_bb, fold_stmt): New.
+	* tree-vrp.c (rvrp_folder::fold_stmt): Call ranger's fold_stmt.
+
+2021-11-03  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/102970
+	* tree-ssa-pre.c (phi_translate_1): Drop clique and base
+	when translating a MEM_REF over a backedge.
+
+2021-11-03  Philipp Tomsich  <philipp.tomsich@vrull.eu>
+
+	* config/aarch64/aarch64-cores.def (AARCH64_CORE): New Ampere-1	core.
+	* config/aarch64/aarch64-tune.md: Regenerate.
+	* config/aarch64/aarch64-cost-tables.h: Add extra costs for Ampere-1.
+	* config/aarch64/aarch64.c: Add tuning structures for Ampere-1.
+	* doc/invoke.texi: Add documentation for Ampere-1 core.
+
+2021-11-03  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	* config/aarch64/aarch64.md (movsi): Add alternative for GOT accesses.
+	(movdi): Likewise.
+	(ldr_got_small_<mode>): Remove pattern.
+	(ldr_got_small_sidi): Likewise.
+	* config/aarch64/aarch64.c (aarch64_load_symref_appropriately): Keep
+	GOT accesses as moves.
+	(aarch64_print_operand): Correctly print got_lo12 in L specifier.
+	(aarch64_mov_operand_p): Make GOT accesses valid move operands.
+	* config/aarch64/constraints.md: Add new constraint Usw for GOT access.
+
+2021-11-03  Martin Liska  <mliska@suse.cz>
+
+	* gcov.c (read_line): Remove dead variable.
+
+2021-11-03  Martin Liska  <mliska@suse.cz>
+
+	PR bootstrap/102828
+	* ipa-fnsummary.c (edge_predicate_pool): Rename predicate class to ipa_predicate.
+	(ipa_fn_summary::account_size_time): Likewise.
+	(edge_set_predicate): Likewise.
+	(set_hint_predicate): Likewise.
+	(add_freqcounting_predicate): Likewise.
+	(evaluate_conditions_for_known_args): Likewise.
+	(evaluate_properties_for_edge): Likewise.
+	(remap_freqcounting_preds_after_dup): Likewise.
+	(ipa_fn_summary_t::duplicate): Likewise.
+	(set_cond_stmt_execution_predicate): Likewise.
+	(set_switch_stmt_execution_predicate): Likewise.
+	(compute_bb_predicates): Likewise.
+	(will_be_nonconstant_expr_predicate): Likewise.
+	(will_be_nonconstant_predicate): Likewise.
+	(phi_result_unknown_predicate): Likewise.
+	(predicate_for_phi_result): Likewise.
+	(analyze_function_body): Likewise.
+	(compute_fn_summary): Likewise.
+	(summarize_calls_size_and_time): Likewise.
+	(estimate_calls_size_and_time): Likewise.
+	(ipa_call_context::estimate_size_and_time): Likewise.
+	(remap_edge_summaries): Likewise.
+	(remap_freqcounting_predicate): Likewise.
+	(ipa_merge_fn_summary_after_inlining): Likewise.
+	(ipa_update_overall_fn_summary): Likewise.
+	(read_ipa_call_summary): Likewise.
+	(inline_read_section): Likewise.
+	* ipa-fnsummary.h (struct ipa_freqcounting_predicate): Likewise.
+	* ipa-predicate.c (predicate::add_clause): Likewise.
+	(ipa_predicate::add_clause): Likewise.
+	(predicate::or_with): Likewise.
+	(ipa_predicate::or_with): Likewise.
+	(predicate::evaluate): Likewise.
+	(ipa_predicate::evaluate): Likewise.
+	(predicate::probability): Likewise.
+	(ipa_predicate::probability): Likewise.
+	(dump_condition): Likewise.
+	(dump_clause): Likewise.
+	(predicate::dump): Likewise.
+	(ipa_predicate::dump): Likewise.
+	(predicate::debug): Likewise.
+	(ipa_predicate::debug): Likewise.
+	(predicate::remap_after_duplication): Likewise.
+	(ipa_predicate::remap_after_duplication): Likewise.
+	(predicate::remap_after_inlining): Likewise.
+	(ipa_predicate::remap_after_inlining): Likewise.
+	(predicate::stream_in): Likewise.
+	(ipa_predicate::stream_in): Likewise.
+	(predicate::stream_out): Likewise.
+	(ipa_predicate::stream_out): Likewise.
+	(add_condition): Likewise.
+	* ipa-predicate.h (class predicate): Likewise.
+	(class ipa_predicate): Likewise.
+	(add_condition): Likewise.
+
+2021-11-03  Richard Biener  <rguenther@suse.de>
+
+	* bitmap.h (bitmap_bit_p): Change the return type to bool.
+	* bitmap.c (bitmap_bit_p): Likewise.
+	* sbitmap.h (bitmap_bit_p): Likewise.
+	(bitmap_set_bit): Return whether the bit changed.
+	(bitmap_clear_bit): Likewise.
+	* tree-ssa.c (verify_vssa): Make use of the changed state
+	from bitmap_set_bit.
+
+2021-11-03  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/103033
+	* internal-fn.c (expand_DEFERRED_INIT): Elide the
+	native_interpret_expr path in favor of folding the
+	VIEW_CONVERT_EXPR generated when punning the RHS.
+
+2021-11-03  Stefan Schulze Frielinghaus  <stefansf@linux.ibm.com>
+
+	* config/s390/s390.c (s390_loop_unroll_adjust): In case of early
+	exit free bbs.
+
+2021-11-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	PR ipa/103040
+	* ipa-modref.c (callee_to_caller_flags): New function.
+	(modref_eaf_analysis::analyze_ssa_name): Use it.
+	(ipa_merge_modref_summary_after_inlining): Fix whitespace.
+
+2021-11-02  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref.c (modref_eaf_analysis::analyze_ssa_name): Revert
+	accidental commit.
+
+2021-11-02  Roger Sayle  <roger@nextmovesoftware.com>
+	    Uroš Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/i386.md (<any_rotate>ti3): Provide expansion for
+	rotations by non-constant amounts.
+
+2021-11-02  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-modref.c: Fix anonymous namespace placement.
+	(class modref_eaf_analysis): New class.
+	(analyze_ssa_name_flags): Turn to ...
+	(modref_eaf_analysis::analyze_ssa_name): ... this one.
+	(merge_call_lhs_flags): Turn to ...
+	(modref_eaf_analysis::merge_call_lhs_flags): .. this one
+	(modref_eaf_analysis::merge_with_ssa_name): New member function.
+	(record_escape_points): Turn to ...
+	(modref_eaf_analysis::record_escape_points): ... this one.
+	(analyze_parms): Updat
+	(ipa_merge_modref_summary_after_inlining): Move to the end of file.
+
+2021-11-02  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gimple.c (gimple_call_static_chain_flags): New function.
+	* gimple.h (gimple_call_static_chain_flags): Declare
+	* ipa-modref.c (modref_summary::modref_summary): Initialize
+	static_chain_flags.
+	(modref_summary_lto::modref_summary_lto): Likewise.
+	(modref_summary::useful_p): Test static_chain_flags.
+	(modref_summary_lto::useful_p): Likewise.
+	(struct modref_summary_lto): Add static_chain_flags.
+	(modref_summary::dump): Dump static_chain_flags.
+	(modref_summary_lto::dump): Likewise.
+	(struct escape_point): Add static_cahin_arg.
+	(analyze_ssa_name_flags): Use gimple_call_static_chain_flags.
+	(analyze_parms): Handle static chains.
+	(modref_summaries::duplicate): Duplicate static_chain_flags.
+	(modref_summaries_lto::duplicate): Likewise.
+	(modref_write): Stream static_chain_flags.
+	(read_section): Likewise.
+	(modref_merge_call_site_flags): Handle static_chain_flags.
+	* ipa-modref.h (struct modref_summary): Add static_chain_flags.
+	* tree-ssa-structalias.c (handle_rhs_call): Use
+	gimple_static_chain_flags.
+
+2021-11-02  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/103029
+	* tree-vect-loop-manip.c (vect_loop_versioning): Ensure
+	the PHI nodes in the loop maintain their original operand
+	order.
+
+2021-11-02  Jan Hubicka  <hubicka@ucw.cz>
+
+	* tree-core.h (EAF_NOT_RETURNED_DIRECTLY): New flag.
+	(EAF_NOREAD): Renumber.
+	* ipa-modref.c (dump_eaf_flags): Dump EAF_NOT_RETURNED_DIRECTLY.
+	(remove_useless_eaf_flags): Handle EAF_NOT_RETURNED_DIRECTLY
+	(deref_flags): Likewise.
+	(modref_lattice::init): Likewise.
+	(modref_lattice::merge): Likewise.
+	(merge_call_lhs_flags): Likewise.
+	(analyze_ssa_name_flags): Likewise.
+	(modref_merge_call_site_flags): Likewise.
+	* tree-ssa-structalias.c (handle_call_arg): Likewise.
+
+2021-11-02  Maciej W. Rozycki  <macro@embecosm.com>
+
+	* config/riscv/riscv.c (riscv_rtx_costs): Correct a CONST_INT_P
+	check and remove an unused local variable with shNadd/shNadd.uw
+	pattern handling.
+
+2021-11-02  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/103007
+	* tree-vect-slp-patterns.c (complex_fms_pattern::matches): Add elem
+	check.
+
+2021-11-02  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/103038
+	* fold-const.c (native_interpret_expr): Handle OFFSET_TYPE.
+	(can_native_interpret_type_p): Likewise.
+	* internal-fn.c (expand_DEFERRED_INIT): View-convert the
+	RHS if the LHS is an SSA name.
+
+2021-11-02  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* langhooks.h (lang_hooks_for_types::simulate_record_decl): New hook.
+	* langhooks-def.h (lhd_simulate_record_decl): Declare.
+	(LANG_HOOKS_SIMULATE_RECORD_DECL): Define.
+	(LANG_HOOKS_FOR_TYPES_INITIALIZER): Include it.
+	* langhooks.c (lhd_simulate_record_decl): New function.
+
+2021-11-02  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/103020
+	* config/i386/i386.c (ix86_vector_mode_supported_p): Reject vector
+	modes with TImode inner mode if 32-bit.
+
+2021-11-02  liuhongt  <hongtao.liu@intel.com>
+
+	* doc/sourcebuild.texi (vect_slp_v4qi_store_unalign,
+	vect_slp_v2hi_store_unalign, vect_slp_v4hi_store_unalign,
+	vect_slp_v4si_store_unalign): Document efficient target.
+	(vect_slp_v4qi_store_unalign_1, vect_slp_v8qi_store_unalign_1,
+	vect_slp_v16qi_store_unalign_1): Ditto.
+	(vect_slp_v2hi_store_align,vect_slp_v2qi_store_align,
+	vect_slp_v2si_store_align, vect_slp_v4qi_store_align): Ditto.
+	(struct_4char_block_move, struct_8char_block_move,
+	struct_16char_block_move): Ditto.
+
+2021-11-02  Roger Sayle  <roger@nextmovesoftware.com>
+	    Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/102986
+	* config/i386/i386-expand.c (ix86_expand_v1ti_to_ti,
+	ix86_expand_ti_to_v1ti): New helper functions.
+	(ix86_expand_v1ti_shift): Check if the amount operand is an
+	integer constant, and expand as a TImode shift if it isn't.
+	(ix86_expand_v1ti_rotate): Check if the amount operand is an
+	integer constant, and expand as a TImode rotate if it isn't.
+	(ix86_expand_v1ti_ashiftrt): New function to expand arithmetic
+	right shifts of V1TImode quantities.
+	* config/i386/i386-protos.h (ix86_expand_v1ti_ashift): Prototype.
+	* config/i386/sse.md (ashlv1ti3, lshrv1ti3): Change constraints
+	to QImode general_operand, and let the helper functions lower
+	shifts by non-constant operands, as TImode shifts.  Make
+	conditional on TARGET_64BIT.
+	(ashrv1ti3): New expander calling ix86_expand_v1ti_ashiftrt.
+	(rotlv1ti3, rotrv1ti3): Change shift operand to QImode.
+	Make conditional on TARGET_64BIT.
+
+2021-11-02  Stefan Schulze Frielinghaus  <stefansf@linux.ibm.com>
+
+	* config/s390/s390.md ("*cc_to_int", "tabort", "*tabort_1",
+	"*tabort_1_plus"): Remove operands non-null check.
+
+2021-11-01  Martin Liska  <mliska@suse.cz>
+
+	* opt-functions.awk: Add new sanity checking.
+	* optc-gen.awk: Add new argument to integer_range_info.
+	* params.opt: Update 2 params which have negative IntegerRange.
+
+2021-11-01  qing zhao  <qing.zhao@oracle.com>
+
+	* gimplify.c (gimplify_decl_expr): Do not add call to
+	__builtin_clear_padding when a variable is a gimple register
+	or it might not have padding.
+	(gimplify_init_constructor): Likewise.
+
+2021-11-01  Tamar Christina  <tamar.christina@arm.com>
+
+	* config/arm/aarch-common-protos.h (struct vector_cost_table): Add
+	movi, dup and extract costing fields.
+	* config/aarch64/aarch64-cost-tables.h (qdf24xx_extra_costs,
+	thunderx_extra_costs, thunderx2t99_extra_costs,
+	thunderx3t110_extra_costs, tsv110_extra_costs, a64fx_extra_costs): Use
+	them.
+	* config/arm/aarch-cost-tables.h (generic_extra_costs,
+	cortexa53_extra_costs, cortexa57_extra_costs, cortexa76_extra_costs,
+	exynosm1_extra_costs, xgene1_extra_costs): Likewise
+	* config/aarch64/aarch64-simd.md (aarch64_simd_dup<mode>): Add r->w dup.
+	* config/aarch64/aarch64.c (aarch64_rtx_costs): Add extra costs.
+
+2021-11-01  Tamar Christina  <tamar.christina@arm.com>
+
+	* cse.c (add_to_set): New.
+	(find_sets_in_insn): Register constants in sets.
+	(canonicalize_insn): Use auto_vec instead.
+	(cse_insn): Try materializing using vec_dup.
+	* rtl.h (simplify_context::simplify_gen_vec_select,
+	simplify_gen_vec_select): New.
+	* simplify-rtx.c (simplify_context::simplify_gen_vec_select): New.
+
+2021-11-01  David Malcolm  <dmalcolm@redhat.com>
+
+	* common.opt (fdiagnostics-escape-format=): New.
+	(diagnostics_escape_format): New enum.
+	(DIAGNOSTICS_ESCAPE_FORMAT_UNICODE): New enum value.
+	(DIAGNOSTICS_ESCAPE_FORMAT_BYTES): Likewise.
+	* diagnostic-format-json.cc (json_end_diagnostic): Add
+	"escape-source" attribute.
+	* diagnostic-show-locus.c
+	(exploc_with_display_col::exploc_with_display_col): Replace
+	"tabstop" param with a cpp_char_column_policy and add an "aspect"
+	param.  Use these to compute m_display_col accordingly.
+	(struct char_display_policy): New struct.
+	(layout::m_policy): New field.
+	(layout::m_escape_on_output): New field.
+	(def_policy): New function.
+	(make_range): Update for changes to exploc_with_display_col ctor.
+	(default_print_decoded_ch): New.
+	(width_per_escaped_byte): New.
+	(escape_as_bytes_width): New.
+	(escape_as_bytes_print): New.
+	(escape_as_unicode_width): New.
+	(escape_as_unicode_print): New.
+	(make_policy): New.
+	(layout::layout): Initialize new fields.  Update m_exploc ctor
+	call for above change to ctor.
+	(layout::maybe_add_location_range): Update for changes to
+	exploc_with_display_col ctor.
+	(layout::calculate_x_offset_display): Update for change to
+	cpp_display_width.
+	(layout::print_source_line): Pass policy
+	to cpp_display_width_computation. Capture cpp_decoded_char when
+	calling process_next_codepoint.  Move printing of source code to
+	m_policy.m_print_cb.
+	(line_label::line_label): Pass in policy rather than context.
+	(layout::print_any_labels): Update for change to line_label ctor.
+	(get_affected_range): Pass in policy rather than context, updating
+	calls to location_compute_display_column accordingly.
+	(get_printed_columns): Likewise, also for cpp_display_width.
+	(correction::correction): Pass in policy rather than tabstop.
+	(correction::compute_display_cols): Pass m_policy rather than
+	m_tabstop to cpp_display_width.
+	(correction::m_tabstop): Replace with...
+	(correction::m_policy): ...this.
+	(line_corrections::line_corrections): Pass in policy rather than
+	context.
+	(line_corrections::m_context): Replace with...
+	(line_corrections::m_policy): ...this.
+	(line_corrections::add_hint): Update to use m_policy rather than
+	m_context.
+	(line_corrections::add_hint): Likewise.
+	(layout::print_trailing_fixits): Likewise.
+	(selftest::test_display_widths): New.
+	(selftest::test_layout_x_offset_display_utf8): Update to use
+	policy rather than tabstop.
+	(selftest::test_one_liner_labels_utf8): Add test of escaping
+	source lines.
+	(selftest::test_diagnostic_show_locus_one_liner_utf8): Update to
+	use policy rather than tabstop.
+	(selftest::test_overlapped_fixit_printing): Likewise.
+	(selftest::test_overlapped_fixit_printing_utf8): Likewise.
+	(selftest::test_overlapped_fixit_printing_2): Likewise.
+	(selftest::test_tab_expansion): Likewise.
+	(selftest::test_escaping_bytes_1): New.
+	(selftest::test_escaping_bytes_2): New.
+	(selftest::diagnostic_show_locus_c_tests): Call the new tests.
+	* diagnostic.c (diagnostic_initialize): Initialize
+	context->escape_format.
+	(convert_column_unit): Update to use default character width policy.
+	(selftest::test_diagnostic_get_location_text): Likewise.
+	* diagnostic.h (enum diagnostics_escape_format): New enum.
+	(diagnostic_context::escape_format): New field.
+	* doc/invoke.texi (-fdiagnostics-escape-format=): New option.
+	(-fdiagnostics-format=): Add "escape-source" attribute to examples
+	of JSON output, and document it.
+	* input.c (location_compute_display_column): Pass in "policy"
+	rather than "tabstop", passing to
+	cpp_byte_column_to_display_column.
+	(selftest::test_cpp_utf8): Update to use cpp_char_column_policy.
+	* input.h (class cpp_char_column_policy): New forward decl.
+	(location_compute_display_column): Pass in "policy" rather than
+	"tabstop".
+	* opts.c (common_handle_option): Handle
+	OPT_fdiagnostics_escape_format_.
+	* selftest.c (temp_source_file::temp_source_file): New ctor
+	overload taking a size_t.
+	* selftest.h (temp_source_file::temp_source_file): Likewise.
+
+2021-11-01  Aldy Hernandez  <aldyh@redhat.com>
+
+	* dbgcnt.def: Add debug counter for back_thread[12] and
+	back_threadfull[12].
+	* passes.def: Pass "first" argument to each back threading pass.
+	* tree-ssa-threadbackward.c (back_threader::back_threader): Add
+	first argument.
+	(back_threader::debug_counter): New.
+	(back_threader::maybe_register_path): Call debug_counter.
+
+2021-11-01  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c (BT_NONE): New.
+	(BT_SPEED): New.
+	(BT_RESOLVE): New.
+	(back_threader::back_threader): Add flags.
+	Move loop initialization here.
+	(back_threader::~back_threader): New.
+	(back_threader::find_taken_edge_switch): Change solver and ranger
+	to pointers.
+	(back_threader::find_taken_edge_cond): Same.
+	(back_threader::find_paths_to_names): Same.
+	(back_threader::find_paths): Same.
+	(back_threader::dump): Same.
+	(try_thread_blocks): Merge into thread_blocks.
+	(back_threader::thread_blocks): New.
+	(do_early_thread_jumps): Merge into thread_blocks.
+	(do_thread_jumps): Merge into thread_blocks.
+	(back_threader::thread_through_all_blocks): Remove.
+
+2021-11-01  Andrew MacLeod  <amacleod@redhat.com>
+
+	PR tree-optimization/103003
+	* value-relation.cc (dom_oracle::register_relation): If the 2
+	ssa names are the same, don't register any relation.
+
+2021-11-01  Dan Li  <ashimida@linux.alibaba.com>
+
+	* config/aarch64/aarch64.c (aarch64_expand_epilogue): Remove
+	redundant check for calls_eh_return.
+	* config/aarch64/aarch64.md (*do_return): Likewise.
+
+2021-11-01  Xionghu Luo  <luoxhu@linux.ibm.com>
+
+	* cfghooks.c (cfg_hook_duplicate_loop_to_header_edge): Rename
+	duplicate_loop_to_header_edge to
+	duplicate_loop_body_to_header_edge.
+	(cfg_hook_duplicate_loop_body_to_header_edge): Likewise.
+	* cfghooks.h (struct cfg_hooks): Likewise.
+	(cfg_hook_duplicate_loop_body_to_header_edge): Likewise.
+	* cfgloopmanip.c (duplicate_loop_body_to_header_edge): Likewise.
+	(clone_loop_to_header_edge): Likewise.
+	* cfgloopmanip.h (duplicate_loop_body_to_header_edge): Likewise.
+	* cfgrtl.c (struct cfg_hooks): Likewise.
+	* doc/loop.texi: Likewise.
+	* loop-unroll.c (unroll_loop_constant_iterations): Likewise.
+	(unroll_loop_runtime_iterations): Likewise.
+	(unroll_loop_stupid): Likewise.
+	(apply_opt_in_copies): Likewise.
+	* tree-cfg.c (struct cfg_hooks): Likewise.
+	* tree-ssa-loop-ivcanon.c (try_unroll_loop_completely): Likewise.
+	(try_peel_loop): Likewise.
+	* tree-ssa-loop-manip.c (copy_phi_node_args): Likewise.
+	(gimple_duplicate_loop_body_to_header_edge): Likewise.
+	(tree_transform_and_unroll_loop): Likewise.
+	* tree-ssa-loop-manip.h (gimple_duplicate_loop_body_to_header_edge):
+	Likewise.
+
+2021-11-01  Xionghu Luo  <luoxhu@linux.ibm.com>
+
+	* cfgloopmanip.c (loop_version): Refactor loopify to
+	loop_version.  Move condition generation after loopify.
+	(loopify): Delete.
+	* cfgloopmanip.h (loopify): Delete.
+
+2021-10-31  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-fnsummary.c: Include tree-dfa.h.
+	(points_to_local_or_readonly_memory_p): Return true on return
+	slot writes.
+	* ipa-modref.c (analyze_ssa_name_flags): Fix handling of copy
+	statement.
+
+2021-10-30  Tobias Burnus  <tobias@codesourcery.com>
+
+	PR middle-end/102972
+	* omp-low.c (omp_runtime_api_call): Use DECL_ASSEMBLER_NAME to get
+	internal Fortran name; new permit_num_teams arg to permit
+	omp_get_num_teams and omp_get_team_num.
+	(scan_omp_1_stmt): Update call to it, add missing call for
+	reverse offload, and check for strictly nested API calls in teams.
+
+2021-10-30  Jakub Jelinek  <jakub@redhat.com>
+
+	* gimplify.c (gimplify_omp_for): Diagnose threadprivate iterators.
+
+2021-10-29  David Malcolm  <dmalcolm@redhat.com>
+
+	* selftest.c (assert_streq): Add newlines when emitting non-equal
+	non-NULL strings.
+
+2021-10-29  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in: Fix syntax for reference to LIBDEPS in
+	gengtype link rule.
+
+2021-10-29  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* doc/install.texi: Bump required minimum DejaGnu version.
+
+2021-10-29  Aldy Hernandez  <aldyh@redhat.com>
+	    Andrew MacLeod  <amacleod@redhat.com>
+
+	* value-relation.cc (path_oracle::killing_def): Add a
+	self-equivalence so we don't look to the root oracle.
+
+2021-10-29  Aldy Hernandez  <aldyh@redhat.com>
+
+	* passes.def: Replace the pass_thread_jumps before VRP* with
+	pass_thread_jumps_full.  Remove all pass_vrp_threader instances.
+	* tree-ssa-threadbackward.c (pass_data_thread_jumps_full):
+	Remove hyphen from "thread-full" name.
+
+2021-10-29  Andrew MacLeod  <amacleod@redhat.com>
+
+	PR tree-optimization/102983
+	* gimple-range-cache.h (propagate_updated_value): Make public.
+	* gimple-range.cc (gimple_ranger::range_of_stmt): Propagate exports
+	when processing gcond stmts.
+
+2021-10-29  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gimple.c (gimple_call_retslot_flags): New function.
+	* gimple.h (gimple_call_retslot_flags): Declare.
+	* ipa-modref.c: Include tree-cfg.h.
+	(struct escape_entry): Turn parm_index to signed.
+	(modref_summary_lto::modref_summary_lto): Add retslot_flags.
+	(modref_summary::modref_summary): Initialize retslot_flags.
+	(struct modref_summary_lto): Likewise.
+	(modref_summary::useful_p): Check retslot_flags.
+	(modref_summary_lto::useful_p): Likewise.
+	(modref_summary::dump): Dump retslot_flags.
+	(modref_summary_lto::dump): Likewise.
+	(struct escape_point): Add hidden_args enum.
+	(analyze_ssa_name_flags): Ignore return slot return;
+	use gimple_call_retslot_flags.
+	(record_escape_points): Break out from ...
+	(analyze_parms): ... here; handle retslot_flags.
+	(modref_summaries::duplicate): Duplicate retslot_flags.
+	(modref_summaries_lto::duplicate): Likewise.
+	(modref_write_escape_summary): Stream parm_index as signed.
+	(modref_read_escape_summary): Likewise.
+	(modref_write): Stream retslot_flags.
+	(read_section): Likewise.
+	(struct escape_map): Fix typo in comment.
+	(update_escape_summary_1): Fix whitespace.
+	(ipa_merge_modref_summary_after_inlining): Drop retslot_flags.
+	(modref_merge_call_site_flags): Merge retslot_flags.
+	* ipa-modref.h (struct modref_summary): Add retslot_flags.
+	* tree-ssa-structalias.c (handle_rhs_call): Handle retslot_flags.
+
+2021-10-29  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/102977
+	* tree-vect-slp-patterns.c (vect_match_call_p): Remove.
+	(vect_detect_pair_op): Add crosslane check.
+	(vect_match_call_complex_mla): Remove.
+	(class complex_mul_pattern): Update comment.
+	(complex_mul_pattern::matches): Update detection.
+	(class complex_fma_pattern): Remove.
+	(complex_fma_pattern::matches): Remove.
+	(complex_fma_pattern::recognize): Remove.
+	(complex_fma_pattern::build): Remove.
+	(class complex_fms_pattern):  Update comment.
+	(complex_fms_pattern::matches): Remove.
+	(complex_operations_pattern::recognize): Remove complex_fma_pattern
+
+2021-10-29  Jakub Jelinek  <jakub@redhat.com>
+
+	* gimple-fold.c (gimple_fold_builtin_memset): Copy over location from
+	call to store.
+
+2021-10-29  Xionghu Luo  <luoxhu@linux.ibm.com>
+
+	PR target/102868
+	* config/rs6000/rs6000.c (altivec_expand_vec_perm_const): Add
+	patterns match and emit for VSX xxpermdi.
+
+2021-10-29  liuhongt  <hongtao.liu@intel.com>
+
+	PR target/102464
+	* config/i386/i386-builtin-types.def (V8HF_FTYPE_V8HF): New
+	function type.
+	(V16HF_FTYPE_V16HF): Ditto.
+	(V32HF_FTYPE_V32HF): Ditto.
+	(V8HF_FTYPE_V8HF_ROUND): Ditto.
+	(V16HF_FTYPE_V16HF_ROUND): Ditto.
+	(V32HF_FTYPE_V32HF_ROUND): Ditto.
+	* config/i386/i386-builtin.def ( IX86_BUILTIN_FLOORPH,
+	IX86_BUILTIN_CEILPH, IX86_BUILTIN_TRUNCPH,
+	IX86_BUILTIN_FLOORPH256, IX86_BUILTIN_CEILPH256,
+	IX86_BUILTIN_TRUNCPH256, IX86_BUILTIN_FLOORPH512,
+	IX86_BUILTIN_CEILPH512, IX86_BUILTIN_TRUNCPH512): New builtin.
+	* config/i386/i386-builtins.c
+	(ix86_builtin_vectorized_function): Enable vectorization for
+	HFmode FLOOR/CEIL/TRUNC operation.
+	* config/i386/i386-expand.c (ix86_expand_args_builtin): Handle
+	new builtins.
+	* config/i386/sse.md (rint<mode>2, nearbyint<mode>2): Extend
+	to vector HFmodes.
+
+2021-10-28  Aldy Hernandez  <aldyh@redhat.com>
+	    Andrew MacLeod  <amacleod@redhat.com>
+
+	* value-relation.cc (path_oracle::killing_def): Walk the
+	equivalency list and remove SSA from any equivalencies.
+
+2021-10-28  Stafford Horne  <shorne@gmail.com>
+
+	* config/or1k/or1k.h (PROFILE_HOOK): Add return address argument
+	to _mcount.
+
+2021-10-28  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/102951
+	* fold-const.h (address_compare): Declare.
+	* fold-const.c (address_compare): New function.
+	* match.pd (cmp (convert1?@2 addr@0) (convert2? addr@1)): Use
+	address_compare helper.
+	(minmax cmp (convert1?@2 addr@0) (convert2?@3 addr@1)): New
+	simplification.
+
+2021-10-28  Andrew MacLeod  <amacleod@redhat.com>
+
+	* vr-values.c (simplify_using_ranges::fold_cond): Change fold message.
+
+2021-10-28  Andrew MacLeod  <amacleod@redhat.com>
+
+	PR tree-optimization/102940
+	* tree-vrp.c (execute_ranger_vrp): Reset scev.
+
+2021-10-28  Richard Purdie  <richard.purdie@linuxfoundation.org>
+
+	* config/nios2/linux.h (MUSL_DYNAMIC_LINKER): Add musl linker
+
+2021-10-28  Richard Purdie  <richard.purdie@linuxfoundation.org>
+
+	* configure: Regenerate.
+	* configure.ac: Use CPPFLAGS_FOR_BUILD for GMPINC
+
+2021-10-28  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* doc/invoke.texi (%X): Remove obsolete reference to -Wl.
+
+2021-10-28  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/84407
+	* fold-const.c (fold_convert_const): Avoid int to float
+	constant folding with -frounding-math and inexact result.
+	* simplify-rtx.c (simplify_const_unary_operation): Likewise
+	for both float and unsigned_float.
+
+2021-10-28  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c
+	(back_threader::find_taken_edge_switch): Use find_case_label_range
+	instead of find_taken_edge.
+
+2021-10-28  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c (class back_threader_registry):
+	Inherit from back_jt_path_registry.
+	(back_threader_registry::thread_through_all_blocks): Remove.
+	(back_threader_registry::register_path): Remove
+	m_lowlevel_registry prefix.
+
+2021-10-28  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/57245
+	* fold-const.c (fold_convert_const_real_from_real): Honor
+	-frounding-math if the conversion is not exact.
+	* simplify-rtx.c (simplify_const_unary_operation): Do not
+	simplify FLOAT_TRUNCATE with sign dependent rounding.
+
+2021-10-28  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/102949
+	* tree-vect-stmts.c (ensure_base_align): Look at the
+	dr_info of a group leader and assert we are looking at
+	one with analyzed alignment.
+
+2021-10-28  Kewen Lin  <linkw@linux.ibm.com>
+
+	PR target/102767
+	* config/rs6000/rs6000.c (rs6000_builtin_vectorization_cost): Consider
+	V1T1 mode for unaligned load and store.
+
+2021-10-28  Kito Cheng  <kito.cheng@sifive.com>
+
+	* config/riscv/riscv.md (zero_extendsidi2_internal): Allow ZBB
+	use this pattern.
+
+2021-10-28  Kito Cheng  <kito.cheng@sifive.com>
+
+	* config/riscv/arch-canonicalize (CANONICAL_ORDER): Add `i` to
+	CANONICAL_ORDER.
+
+2021-10-28  Alexandre Oliva  <oliva@adacore.com>
+
+	* common.opt (fharden-compares): New.
+	(fharden-conditional-branches): New.
+	* doc/invoke.texi: Document new options.
+	* gimple-harden-conditionals.cc: New.
+	* Makefile.in (OBJS): Build it.
+	* passes.def: Add new passes.
+	* tree-pass.h (make_pass_harden_compares): Declare.
+	(make_pass_harden_conditional_branches): Declare.
+
+2021-10-28  Xionghu Luo  <luoxhu@linux.ibm.com>
+
+	PR target/94613
+	* config/rs6000/altivec.md: Add vsx register constraints.
+	* config/rs6000/vsx.md (vsx_xxsel<mode>): Delete.
+	(vsx_xxsel<mode>2): Likewise.
+	(vsx_xxsel<mode>3): Likewise.
+	(vsx_xxsel<mode>4): Likewise.
+
+2021-10-28  Xionghu Luo  <luoxhu@linux.ibm.com>
+
+	PR target/94613
+	* config/rs6000/altivec.md (*altivec_vsel<mode>): Change to ...
+	(altivec_vsel<mode>): ... this and update define.
+	(*altivec_vsel<mode>_uns): Delete.
+	(altivec_vsel<mode>2): New define_insn.
+	(altivec_vsel<mode>3): Likewise.
+	(altivec_vsel<mode>4): Likewise.
+	* config/rs6000/rs6000-call.c (altivec_expand_vec_sel_builtin): New.
+	(altivec_expand_builtin): Call altivec_expand_vec_sel_builtin to expand
+	vel_sel.
+	* config/rs6000/rs6000.c (rs6000_emit_vector_cond_expr): Use bit-wise
+	selection instead of per element.
+	* config/rs6000/vector.md:
+	* config/rs6000/vsx.md (*vsx_xxsel<mode>): Change to ...
+	(vsx_xxsel<mode>): ... this and update define.
+	(*vsx_xxsel<mode>_uns): Delete.
+	(vsx_xxsel<mode>2): New define_insn.
+	(vsx_xxsel<mode>3): Likewise.
+	(vsx_xxsel<mode>4): Likewise.
+
+2021-10-28  Hongyu Wang  <hongyu.wang@intel.com>
+
+	* config/i386/i386.c (use_rsqrt_p): Add mode parameter, enable
+	  HFmode rsqrt without TARGET_SSE_MATH.
+	(ix86_optab_supported_p): Refactor rint, adjust floor, ceil,
+	btrunc condition to be restricted by -ftrapping-math, adjust
+	use_rsqrt_p function call.
+	* config/i386/i386.md (rcphf2): New define_insn.
+	(rsqrthf2): Likewise.
+	* config/i386/sse.md (div<mode>3): Change VF2H to VF2.
+	(div<mode>3): New expander for HF mode.
+	(rsqrt<mode>2): Likewise.
+	(*avx512fp16_vmrcpv8hf2): New define_insn for rpad pass.
+	(*avx512fp16_vmrsqrtv8hf2): Likewise.
+
+2021-10-27  Saagar Jha  <saagar@saagarjha.com>
+
+	* config.gcc: Adjust for Darwin21.
+	* config/darwin-c.c (macosx_version_as_macro): Likewise.
+	* config/darwin-driver.c (validate_macosx_version_min):
+	Likewise.
+	(darwin_find_version_from_kernel): Likewise.
+
+2021-10-27  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gimple-range-path.cc
+	(path_range_query::range_defined_in_block): Call killing_def.
+
+2021-10-27  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gimple-range-fold.cc (fold_using_range::range_of_range_op): Dump
+	operands as well as relation.
+	* gimple-range-path.cc
+	(path_range_query::compute_ranges_in_block): Compute PHI relations
+	first.  Compute outgoing relations at the end.
+	(path_range_query::compute_ranges): Remove call to compute_relations.
+	(path_range_query::compute_relations): Remove.
+	(path_range_query::maybe_register_phi_relation): New.
+	(path_range_query::compute_phi_relations): Abstract out
+	registering one PHI relation to...
+	(path_range_query::compute_outgoing_relations): ...here.
+	* gimple-range-path.h (class path_range_query): Remove
+	compute_relations.
+	Add maybe_register_phi_relation.
+
+2021-10-27  Aldy Hernandez  <aldyh@redhat.com>
+	    Andrew MacLeod  <amacleod@redhat.com>
+
+	* value-relation.cc (path_oracle::killing_def): Kill second
+	order relations.
+
+2021-10-27  Martin Jambor  <mjambor@suse.cz>
+
+	* ipa-cp.c (good_cloning_opportunity_p): Decide whether to use
+	profile feedback depending on their local availability.
+
+2021-10-27  Martin Jambor  <mjambor@suse.cz>
+
+	* params.opt (param_ipa_cp_profile_count_base): New parameter.
+	* doc/invoke.texi (Optimize Options): Add entry for
+	ipa-cp-profile-count-base.
+	* ipa-cp.c (max_count): Replace with base_count, replace all
+	occurrences too, unless otherwise stated.
+	(ipcp_cloning_candidate_p): identify mostly-directly called
+	functions based on their counts, not max_count.
+	(compare_edge_profile_counts): New function.
+	(ipcp_propagate_stage): Instead of setting max_count, find the
+	appropriate edge count in a sorted vector of counts of eligible
+	edges and make it the base_count.
+
+2021-10-27  Martin Jambor  <mjambor@suse.cz>
+
+	* ipa-cp.c (struct caller_statistics): New fields rec_count_sum,
+	n_nonrec_calls and itself, document all fields.
+	(init_caller_stats): Initialize the above new fields.
+	(gather_caller_stats): Gather self-recursive counts and calls number.
+	(get_info_about_necessary_edges): Gather counts of self-recursive and
+	other edges bringing in the requested value separately.
+	(dump_profile_updates): Rework to dump info about a single node only.
+	(lenient_count_portion_handling): New function.
+	(struct gather_other_count_struct): New type.
+	(gather_count_of_non_rec_edges): New function.
+	(struct desc_incoming_count_struct): New type.
+	(analyze_clone_icoming_counts): New function.
+	(adjust_clone_incoming_counts): Likewise.
+	(update_counts_for_self_gen_clones): Likewise.
+	(update_profiling_info): Rewritten.
+	(update_specialized_profile): Adjust call to dump_profile_updates.
+	(create_specialized_node): Do not update profiling info.
+	(decide_about_value): New parameter self_gen_clones, either push new
+	clones into it or updat their profile counts.  For self-recursively
+	generated values, use a portion of the node count instead of count
+	from self-recursive edges to estimate goodness.
+	(decide_whether_version_node): Gather clones for self-generated values
+	in a new vector, update their profiles at once at the end.
+
+2021-10-27  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.c (vect_transform_loops): New function,
+	split out from ...
+	(try_vectorize_loop_1): ... here.  Simplify as epilogues
+	are now fully handled in the split part.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	* omp-expand.c (expand_omp_for_init_counts): Handle non-rectangular
+	iterators with pointer types.
+	(expand_omp_for_init_vars, extract_omp_for_update_vars): Likewise.
+
+2021-10-26  Martin Sebor  <msebor@redhat.com>
+
+	PR tree-optimization/102238
+	PR tree-optimization/102919
+	* gimple-ssa-sprintf.c (get_string_length): Add an argument.
+	(array_elt_at_offset): Move to pointer-query.
+	(set_aggregate_size_and_offset): New function.
+	(field_at_offset):  Move to pointer-query.
+	(get_origin_and_offset): Rename...
+	(get_origin_and_offset_r): this.  Add an argument.  Make aggregate
+	handling more robust.
+	(get_origin_and_offset): New.
+	(alias_offset): Add an argument.
+	(format_string): Use subobject size determined by get_origin_and_offset.
+	* pointer-query.cc (field_at_offset): Move from gimple-ssa-sprintf.c.
+	Improve/correct handling of aggregates.
+	(array_elt_at_offset): Same.
+	* pointer-query.h (field_at_offset): Declare.
+	(array_elt_at_offset): Declare.
+
+2021-10-26  Martin Sebor  <msebor@redhat.com>
+
+	* builtins.c (check_strncat_sizes): Pass access_data ctor additional
+	arguments.
+	(expand_builtin_memcmp): Move code to gimple-ssa-warn-access.cc.
+	(expand_builtin_fork_or_exec): Same.
+	* gimple-array-bounds.cc (array_bounds_checker::check_mem_ref): Pass
+	compute_objsize additional arguments.
+	(inbounds_memaccess_p): Same.
+	(array_bounds_checker::check_array_bounds): Add an assert.  Stash
+	statement in a member.
+	(check_array_bounds_dom_walker::before_dom_children): Same.
+	* gimple-array-bounds.h (array_bounds_checker::m_stmt): New member.
+	* gimple-ssa-sprintf.c (get_destination_size): Add an argument.
+	(handle_printf_call): Pass a new argument.
+	* gimple-ssa-warn-access.cc (get_size_range): Add an argument.
+	(check_access): Add an argument and pass it along to callees.
+	(check_read_access): Make a member function.
+	(pass_waccess::check_strcat): Pass access_data ctor additional
+	arguments.
+	(pass_waccess::check_strncat): Same.
+	(pass_waccess::check_stxcpy): Same.
+	(pass_waccess::check_stxncpy): Same.
+	(pass_waccess::check_strncmp): Same.
+	(pass_waccess::check_read_access): Same.
+	(pass_waccess::check_builtin): Same.
+	(pass_waccess::maybe_check_access_sizes): Same.
+	(pass_waccess::maybe_check_dealloc_call): Same.
+	* gimple-ssa-warn-access.h (check_read_access): Declare a new
+	member function.
+	* pointer-query.cc (compute_objsize_r): Add an argument.
+	(gimple_call_return_array): Same.
+	(gimple_call_alloc_size): Same.
+	(access_ref::access_ref): Same.
+	(access_ref::get_ref): Same.
+	(pointer_query::get_ref): Same.
+	(handle_min_max_size): Pass an arguments to callees.
+	(handle_array_ref): Add an argument.
+	(handle_mem_ref): Same.
+	(compute_objsize): Same.
+	* pointer-query.h (struct access_ref): Adjust signatures.
+	(struct access_data): Same.
+	(gimple_call_alloc_size): Add an argument.
+	(gimple_parm_array_size): Same.
+	(compute_objsize): Same.
+	* tree-ssa-strlen.c (strlen_pass::adjust_last_stmt): Pass an additional
+	argument to compute_objsize.
+	(strlen_pass::maybe_warn_overflow): Same.
+	(maybe_diag_stxncpy_trunc): Same.
+
+2021-10-26  Martin Sebor  <msebor@redhat.com>
+
+	PR middle-end/102453
+	* gimple-ssa-warn-access.cc (pass_waccess::check_atomic_builtin): New.
+	(pass_waccess::check_atomic_builtin): Call it.
+
+2021-10-26  Vladimir N. Makarov  <vmakarov@redhat.com>
+
+	PR rtl-optimization/102842
+	* lra-constraints.c (match_reload): Ignore out in checking values
+	of outs.
+	(curr_insn_transform): Collect outputs before doing reloads of operands.
+
+2021-10-26  Paul A. Clarke  <pc@us.ibm.com>
+
+	PR target/102719
+	* config/rs6000/x86intrin.h: Move some included headers to new
+	headers.  Include new immintrin.h instead of those headers.
+	* config/rs6000/immintrin.h: New.
+	* config/rs6000/x86gprintrin.h: New.
+	* config.gcc (powerpc*-*-*): Add new headers to extra_headers.
+
+2021-10-26  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_create_addr_base_for_vector_ref):
+	Remove byte_offset parameter.
+	(vect_create_data_ref_ptr): Likewise.
+	* tree-vect-data-refs.c (vect_create_addr_base_for_vector_ref):
+	Likewise.
+	(vect_create_data_ref_ptr): Likewise.
+	* tree-vect-stmts.c (vectorizable_store): Adjust.
+	(vectorizable_load): Likewise.
+
+2021-10-26  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/96109
+	* tree-vectorizer.h (dr_misalignment): Add optional offset
+	parameter.
+	* tree-vect-data-refs.c (dr_misalignment): Likewise.  Remove
+	offset applied for negative stride accesses.
+	(vect_enhance_data_refs_alignment): Compute negative stride
+	access offset and pass it to dr_misalignment.
+	* tree-vect-stmts.c (get_negative_load_store_type): Pass
+	negative offset to dr_misalignment.
+	(get_group_load_store_type): Likewise.
+	(get_load_store_type): Likewise.
+	(vectorizable_store): Remove asserts about alignment.
+	(vectorizable_load): Likewise.
+
+2021-10-26  Kewen Lin  <linkw@linux.ibm.com>
+
+	PR tree-optimization/102897
+	* tree-ssa-forwprop.c (simplify_permutation): Remove a wrong assertion.
+
+2021-10-26  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-data-refs.c (vect_create_addr_base_for_vector_ref):
+	Take offset in bytes.
+	(vect_create_data_ref_ptr): Likewise.
+	* tree-vect-loop-manip.c (get_misalign_in_elems): Multiply
+	offset by element size.
+	(vect_create_cond_for_align_checks): Likewise.
+	* tree-vect-stmts.c (get_negative_load_store_type): Likewise.
+	(vectorizable_load): Remove duplicate leftover from merge
+	conflict.
+
+2021-10-26  Roger Sayle  <roger@nextmovesoftware.com>
+
+	* config/i386/i386-expand.c (ix86_expand_v1ti_shift): New helper
+	function to expand V1TI mode logical shifts by integer constants.
+	(ix86_expand_v1ti_rotate): New helper function to expand V1TI
+	mode rotations by integer constants.
+	* config/i386/i386-protos.h (ix86_expand_v1ti_shift,
+	ix86_expand_v1ti_rotate): Prototype new functions here.
+	* config/i386/sse.md (ashlv1ti3, lshrv1ti3, rotlv1ti3, rotrv1ti3):
+	New TARGET_SSE2 expanders to implement V1TI shifts and rotations.
+
+2021-10-26  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c (back_threader::maybe_register_path):
+	Avoid threading circular paths.
+
+2021-10-26  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c (back_threader::resolve_phi):
+	Attempt to resolve all incoming paths to a PHI.
+	(back_threader::resolve_def): Always return true for PHIs.
+
+2021-10-26  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c (back_threader::find_paths_to_names):
+	Always try to resolve path without looking back.
+	* tree-ssa-threadupdate.c (dump_jump_thread): Indidicate whether
+	edge is a back edge.
+
+2021-10-26  Kewen Lin  <linkw@linux.ibm.com>
+
+	PR tree-optimization/102789
+	* tree-vect-loop-manip.c (vect_update_inits_of_drs): Do not
+	update inits of simd_lane_access.
+
+2021-10-25  Andrew MacLeod  <amacleod@redhat.com>
+
+	* tree-vrp.c (vrp_simplify_cond_using_ranges): Add return type and
+	move to vr-values.c.
+	(simplify_casted_conds): Move to vrp_folder class.
+	(execute_vrp): Call via vrp_folder now.
+	* vr-values.c (simplify_cond_using_ranges_1): Call simplify_casted_cond.
+	(simplify_using_ranges::simplify_casted_cond): Relocate from tree-vrp.c.
+	* vr-values.h (simplify_casted_cond): Add prototype.
+
+2021-10-25  Andrew MacLeod  <amacleod@redhat.com>
+
+	* tree-vrp.c (rvrp_folder::fold_stmt): If simplification fails, try
+	to fold anyway.
+
+2021-10-25  Paul A. Clarke  <pc@us.ibm.com>
+
+	* config/rs6000/smmintrin.h (_mm_testz_si128): Add "extern" to
+	function signature.
+	(_mm_testc_si128): Likewise.
+	(_mm_testnzc_si128): Likewise.
+	(_mm_blend_ps): Likewise.
+	(_mm_blendv_ps): Likewise.
+	(_mm_blend_pd): Likewise.
+	(_mm_blendv_pd): Likewise.
+	(_mm_ceil_pd): Likewise.
+	(_mm_ceil_sd): Likewise.
+	(_mm_ceil_ps): Likewise.
+	(_mm_ceil_ss): Likewise.
+	(_mm_floor_pd): Likewise.
+	(_mm_floor_sd): Likewise.
+	(_mm_floor_ps): Likewise.
+	(_mm_floor_ss): Likewise.
+	(_mm_minpos_epu16): Likewise.
+	(_mm_mul_epi32): Likewise.
+	(_mm_cvtepi8_epi16): Likewise.
+	(_mm_packus_epi32): Likewise.
+	(_mm_cmpgt_epi64): Likewise.
+
+2021-10-25  Roger Sayle  <roger@nextmovesoftware.com>
+
+	* simplify-rtx.c (simplify_binary_operation_1) [SS_ASHIFT]: Simplify
+	shifts of the mode's smin_value and smax_value when the bit count
+	operand doesn't have side-effects.
+	[US_ASHIFT]: Likewise, simplify shifts of the mode's umax_value
+	when the bit count operand doesn't have side-effects.
+	(simplify_const_binary_operation) [SS_ASHIFT, US_ASHIFT]: Perform
+	compile-time evaluation of saturating left shifts with constant
+	arguments.
+
+2021-10-25  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gimple-range.cc (gimple_ranger::export_global_ranges): Remove check
+	for TDF_DETAILS.
+
+2021-10-25  Andrew MacLeod  <amacleod@redhat.com>
+
+	* flag-types.h (enum ranger_debug): Adjust values.
+	* params.opt (ranger_debug): Ditto.
+
+2021-10-25  Martin Jambor  <mjambor@suse.cz>
+
+	PR tree-optimization/102886
+	* tree-sra.c (totally_scalarize_subtree): Fix the out of
+	access-condition.
+
+2021-10-25  Andrew Pinski  <apinski@marvell.com>
+
+	* tree-ssa-dce.c (simple_dce_from_worklist):
+	Check stmt_unremovable_because_of_non_call_eh_p also
+	before removing the statement.
+
+2021-10-25  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/102905
+	* tree-vect-data-refs.c (vect_enhance_data_refs_alignment):
+	Use vect_supportable_dr_alignment again to determine whether
+	an access is supported when not aligned.
+
+2021-10-25  Kito Cheng  <kito.cheng@sifive.com>
+
+	* config/riscv/riscv.c (riscv_rtx_costs): Handle cost model
+	for zbs extension.
+
+2021-10-25  Jim Wilson  <jimw@sifive.com>
+	    Kito Cheng  <kito.cheng@sifive.com>
+	    Jia-Wei Chen  <jiawei@iscas.ac.cn>
+	    Shi-Hua Liao  <shihua@iscas.ac.cn>
+
+	* config/riscv/bitmanip.md (shiftm1): New.
+	(*bset<mode>): Ditto.
+	(*bset<mode>_mask): Ditto.
+	(*bset<mode>_1): Ditto.
+	(*bset<mode>_1_mask): Ditto.
+	(*bseti<mode>): Ditto.
+	(*bclr<mode>): Ditto.
+	(*bclri<mode>): Ditto.
+	(*binv<mode>): Ditto.
+	(*binvi<mode>): Ditto.
+	(*bext<mode>): Ditto.
+	(*bexti): Ditto.
+	* config/riscv/predicates.md (splittable_const_int_operand):
+	Handle bseti.
+	(single_bit_mask_operand): New.
+	(not_single_bit_mask_operand): Ditto.
+	(const31_operand): Ditto.
+	(const63_operand): Ditto.
+	* config/riscv/riscv.c (riscv_build_integer_1): Handle bseti.
+	(riscv_output_move): Ditto.
+	(riscv_print_operand): Handle new operand type: T and S.
+	* config/riscv/riscv.h (SINGLE_BIT_MASK_OPERAND): New.
+
+2021-10-25  Jim Wilson  <jimw@sifive.com>
+
+	* config/riscv/riscv.c (riscv_build_integer_1): Build integer
+	with rotate.
+
+2021-10-25  Kito Cheng  <kito.cheng@sifive.com>
+
+	* config/riscv/riscv.c (riscv_extend_cost): Handle cost model
+	for zbb extension.
+	(riscv_rtx_costs): Ditto.
+
+2021-10-25  Jim Wilson  <jimw@sifive.com>
+	    Kito Cheng  <kito.cheng@sifive.com>
+	    Jia-Wei Chen  <jiawei@iscas.ac.cn>
+
+	* config/riscv/bitmanip.md (bitmanip_bitwise): New.
+	(bitmanip_minmax): New.
+	(clz_ctz_pcnt): New.
+	(bitmanip_optab): New.
+	(bitmanip_insn): New.
+	(*<optab>_not<mode>): New.
+	(*xor_not<mode>): New.
+	(<bitmanip_optab>si2): New.
+	(*<bitmanip_optab>disi2): New.
+	(<bitmanip_optab>di2): New.
+	(*zero_extendhi<GPR:mode>2_bitmanip): New.
+	(*extend<SHORT:mode><SUPERQI:mode>2_zbb): New.
+	(*zero_extendhi<GPR:mode>2_zbb): New.
+	(rotrsi3): New.
+	(rotrdi3): New.
+	(rotrsi3_sext): New.
+	(rotlsi3): New.
+	(rotldi3): New.
+	(rotlsi3_sext): New.
+	(bswap<mode>2): New.
+	(<bitmanip_optab><mode>3): New.
+	* config/riscv/riscv.md (type): Add rotate.
+	(zero_extendhi<GPR:mode>2): Change to define_expand pattern.
+	(*zero_extendhi<GPR:mode>2): New.
+	(extend<SHORT:mode><SUPERQI:mode>2): Change to define_expand pattern.
+	(*extend<SHORT:mode><SUPERQI:mode>2): New.
+
+2021-10-25  Kito Cheng  <kito.cheng@sifive.com>
+
+	* config/riscv/riscv.c (riscv_extend_cost): Handle cost model
+	for zba extension.
+	(riscv_rtx_costs): Ditto.
+
+2021-10-25  Jim Wilson  <jimw@sifive.com>
+	    Kito Cheng  <kito.cheng@sifive.com>
+	    Jia-Wei Chen  <jiawei@iscas.ac.cn>
+
+	* config/riscv/bitmanip.md (*zero_extendsidi2_bitmanip): New.
+	(*shNadd): Ditto.
+	(*shNadduw): Ditto.
+	(*add.uw): Ditto.
+	(*slliuw): Ditto.
+	(riscv_rtx_costs): Ditto.
+	* config/riscv/riscv.md: Include bitmanip.md
+	(type): Add bitmanip bype.
+	(zero_extendsidi2): Change to define_expand pattern.
+	(*zero_extendsidi2_internal): New.
+	(zero_extendsidi2_shifted): Disable for ZBA.
+
+2021-10-25  Kito Cheng  <kito.cheng@sifive.com>
+
+	* common/config/riscv/riscv-common.c (riscv_ext_version_table):
+	Add zba, zbb, zbc and zbs.
+	(riscv_ext_flag_table): Ditto.
+	* config/riscv/riscv-opts.h (MASK_ZBA): New.
+	(MASK_ZBB): Ditto.
+	(MASK_ZBC): Ditto.
+	(MASK_ZBS): Ditto.
+	(TARGET_ZBA): Ditto.
+	(TARGET_ZBB): Ditto.
+	(TARGET_ZBC): Ditto.
+	(TARGET_ZBS): Ditto.
+	* config/riscv/riscv.opt (riscv_zb_subext): New.
+
+2021-10-25  liuhongt  <hongtao.liu@intel.com>
+
+	PR target/102464
+	* match.pd: Simplify (_Float16) sqrtf((float) a) to .SQRT(a)
+	when direct_internal_fn_supported_p, similar for sqrt/sqrtl.
+
+2021-10-25  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/102920
+	* tree-ssa-sccvn.h (expressions_equal_p): Add argument
+	controlling VN_TOP matching behavior.
+	* tree-ssa-sccvn.c (expressions_equal_p): Likewise.
+	(vn_phi_eq): Do not optimistically match VN_TOP.
+
+2021-10-25  konglin1  <lingling.kong@intel.com>
+
+	* config/i386/sse.md (fma_<mode>_fadd_fmul): Add new
+	define_insn_and_split.
+	(fma_<mode>_fadd_fcmul):Likewise
+	(fma_<complexopname>_<mode>_fma_zero):Likewise
+
+2021-10-24  John David Anglin  <danglin@gcc.gnu.org>
+
+	* config/pa/pa-d.c (pa_d_handle_target_float_abi): Don't check
+	TARGET_DISABLE_FPREGS.
+	* config/pa/pa.c (fix_range): Use MASK_SOFT_FLOAT instead of
+	MASK_DISABLE_FPREGS.
+	(hppa_rtx_costs): Don't check TARGET_DISABLE_FPREGS.  Adjust
+	cost of hardware integer multiplication.
+	(pa_conditional_register_usage): Don't check TARGET_DISABLE_FPREGS.
+	* config/pa/pa.h (INT14_OK_STRICT): Likewise.
+	* config/pa/pa.md: Don't check TARGET_DISABLE_FPREGS. Check
+	TARGET_SOFT_FLOAT in patterns that use xmpyu instruction.
+	* config/pa/pa.opt (mdisable-fpregs): Change target mask to
+	SOFT_FLOAT.  Revise comment.
+	(msoft-float): New option.
+
+2021-10-24  John David Anglin  <danglin@gcc.gnu.org>
+
+	* config/pa/pa.md: Don't use 'G' constraint in integer move patterns.
+
+2021-10-24  Gerald Pfeifer  <gerald@pfeifer.com>
+
+	* doc/install.texi (Specific): Remove obsolete details
+	around GNU/Linux on Itanium.
+	(Specific): Remove reference to Windows for Itanium.
+
+2021-10-23  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* config/i386/x86-tune-sched-bd.c (dispatch_group): Commentary
+	typo fix.
+
+2021-10-23  Jan Hubicka  <hubicka@ucw.cz>
+
+	* tree-ssa-structalias.c (compute_points_to_sets): Cleanup.
+
+2021-10-23  Roger Sayle  <roger@nextmovesoftware.com>
+
+	* config/i386/sse.md (<any_logic>v1ti3): New define_insn to
+	implement V1TImode AND, IOR and XOR on TARGET_SSE2 (and above).
+	(one_cmplv1ti2): New define expand.
+
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Makefile.in: Handle dvidir and install-dvi target.
+	* configure: Regenerate.
+	* configure.ac: Add install-dvi to target_list.
+
+2021-10-22  Gerald Pfeifer  <gerald@pfeifer.com>
+
+	* doc/install.texi (Binaries): Convert mingw-w64.org to https.
+	(Specific): Ditto.
+
+2021-10-22  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/102893
+	* tree-ssa-dce.c (find_obviously_necessary_stmts): Fix the
+	test for an exit edge.
+
+2021-10-22  Aldy Hernandez  <aldyh@redhat.com>
+	    Andrew MacLeod  <amacleod@redhat.com>
+
+	* gimple-range-path.cc (path_range_query::compute_phi_relations):
+	Kill any global relations we may know before registering a new
+	one.
+	* value-relation.cc (path_oracle::killing_def): New.
+	* value-relation.h (path_oracle::killing_def): New.
+
+2021-10-22  Richard Biener  <rguenther@suse.de>
+
+	PR bootstrap/102681
+	* tree-ssa-sccvn.c (vn_phi_insert): For undefined SSA args
+	record VN_TOP.
+	(vn_phi_lookup): Likewise.
+
+2021-10-21  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/98667
+	* doc/invoke.texi: Document -fcf-protection requires i686 or
+	new.
+
+2021-10-21  Eric Botcazou  <ebotcazou@adacore.com>
+
+	PR middle-end/102764
+	* cfgexpand.c (expand_gimple_basic_block): Robustify latest change.
+
+2021-10-21  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* config/aarch64/arm_neon.h (__STRUCTN): Delete function
+	macro and all invocations.
+
+2021-10-21  Andrew MacLeod  <amacleod@redhat.com>
+
+	* doc/invoke.texi (ranger-debug): Document.
+	* flag-types.h (enum ranger_debug): New.
+	(enum evrp_mode): Remove debug values.
+	* gimple-range-cache.cc (DEBUG_RANGE_CACHE): Use new debug flag.
+	* gimple-range-gori.cc (gori_compute::gori_compute): Ditto.
+	* gimple-range.cc (gimple_ranger::gimple_ranger): Ditto.
+	* gimple-ssa-evrp.c (hybrid_folder::choose_value): Ditto.
+	(execute_early_vrp): Use evrp-mode directly.
+	* params.opt (enum evrp_mode): Remove debug values.
+	(ranger-debug): New.
+	(ranger-logical-depth): Relocate to be in alphabetical order.
+
+2021-10-21  Andrew MacLeod  <amacleod@redhat.com>
+
+	* doc/invoke.texi: (vrp1-mode, vrp2-mode): Document.
+	* flag-types.h: (enum vrp_mode): New.
+	* params.opt: (vrp1-mode, vrp2-mode): New.
+	* tree-vrp.c (vrp_pass_num): New.
+	(pass_vrp::pass_vrp): Set pass number.
+	(pass_vrp::execute): Choose which VRP mode to execute.
+
+2021-10-21  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gimple-ssa-evrp.c (class rvrp_folder): Move to tree-vrp.c.
+	(execute_early_vrp): For ranger only mode, invoke ranger_vrp.
+	* tree-vrp.c (class rvrp_folder): Relocate here.
+	(execute_ranger_vrp): New.
+	* tree-vrp.h (execute_ranger_vrp): Export.
+
+2021-10-21  Martin Liska  <mliska@suse.cz>
+
+	PR debug/102585
+	PR bootstrap/102766
+	* opts.c (finish_options): Process flag_var_tracking* options
+	here as they can be adjusted by optimize attribute.
+	Process also flag_syntax_only and flag_gtoggle.
+	* toplev.c (process_options): Remove it here.
+	* common.opt: Make debug_nonbind_markers_p as PerFunction
+	attribute as it depends on optimization level.
+
+2021-10-21  Martin Jambor  <mjambor@suse.cz>
+
+	PR tree-optimization/102505
+	* tree-sra.c (totally_scalarize_subtree): Check that the
+	encountered field fits within the acces we would like to put it
+	in.
+
+2021-10-21  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c
+	(back_threader::maybe_register_path): Remove circular paths check.
+
+2021-10-21  Richard Biener  <rguenther@suse.de>
+
+	* toplev.c (process_options): Move the initial debug_hooks
+	setting ...
+	(toplev::main): ... before the call of the post_options
+	langhook.
+
+2021-10-21  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/102847
+	* tree-vect-stmts.c (vect_model_load_cost): Add the scalar
+	load cost in the prologue for VMAT_INVARIANT.
+
+2021-10-21  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/102847
+	* tree-vect-stmts.c (vect_model_load_cost): Explicitely
+	handle VMAT_INVARIANT as a splat in the prologue.
+
+2021-10-21  Hongyu Wang  <hongyu.wang@intel.com>
+
+	PR target/102812
+	* config/i386/i386.c (ix86_get_ssemov): Adjust HFmode vector
+	move to use the same logic as HImode.
+
+2021-10-21  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-slp.c (vect_build_slp_tree_1): Remove
+	superfluous gimple_call_nothrow_p check.
+
+2021-10-21  Andrew Pinski  <apinski@marvell.com>
+
+	* tree-cfg.c (maybe_remove_writeonly_store): Add dce_ssa_names argument.
+	Mark the ssa-name of the rhs as one to be removed.
+	(execute_fixup_cfg): Update call to maybe_remove_writeonly_store.
+	Call simple_dce_from_worklist at the end to a simple dce.
+
+2021-10-21  Andrew Pinski  <apinski@marvell.com>
+
+	* tree-cfg.c (maybe_remove_writeonly_store): New function
+	factored out from ...
+	(execute_fixup_cfg): Here. Call maybe_remove_writeonly_store.
+
+2021-10-21  Andrew Pinski  <apinski@marvell.com>
+
+	* tree-cfg.c (execute_fixup_cfg): Remove comment
+	about standalone pass.
+
+2021-10-21  Andrew Pinski  <apinski@marvell.com>
+
+	* tree-cfg.c (execute_fixup_cfg): Output when the statement
+	is removed when it is a write only var.
+
+2021-10-21  Aldy Hernandez  <aldyh@redhat.com>
+
+	* tree-ssa-threadbackward.c (back_threader::maybe_register_path):
+	Avoid threading circular paths.
+
 2021-10-20  Alex Coplan  <alex.coplan@arm.com>
 
 	* calls.c (initialize_argument_information): Remove some dead
diff --git a/gcc/DATESTAMP b/gcc/DATESTAMP
index 9b3296b..7090fb4 100644
--- a/gcc/DATESTAMP
+++ b/gcc/DATESTAMP
@@ -1 +1 @@
-20211021
+20211116
diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index f36ffa4..571e9c2 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -713,6 +713,8 @@ tmpdir = /tmp
 
 datarootdir = @datarootdir@
 docdir = @docdir@
+# Directory in which to put DVIs
+dvidir = @dvidir@
 # Directory in which to build HTML
 build_htmldir = $(objdir)/HTML/gcc-$(version)
 # Directory in which to put HTML
@@ -1389,6 +1391,7 @@ OBJS = \
 	gimple-if-to-switch.o \
 	gimple-iterator.o \
 	gimple-fold.o \
+	gimple-harden-conditionals.o \
 	gimple-laddress.o \
 	gimple-loop-interchange.o \
 	gimple-loop-jam.o \
@@ -2962,7 +2965,7 @@ build/gengtype$(build_exeext) : build/gengtype-lex.o build/gengtype-parse.o \
 gengtype$(exeext) : gengtype.o gengtype-lex.o gengtype-parse.o \
               gengtype-state.o errors.o $(LIBDEPS)
 	+$(LINKER) $(ALL_LINKERFLAGS) $(LDFLAGS) -o $@ \
-	    $(filter-out ($LIBDEPS), $^) $(LIBS)
+	    $(filter-out $(LIBDEPS), $^) $(LIBS)
 
 # Rule for the generator programs:
 $(genprog:%=build/gen%$(build_exeext)): build/gen%$(build_exeext): build/gen%.o $(BUILD_LIBDEPS)
@@ -3386,8 +3389,10 @@ $(build_htmldir)/gcc/index.html: $(TEXI_GCC_FILES)
 $(build_htmldir)/gccint/index.html: $(TEXI_GCCINT_FILES)
 $(build_htmldir)/cppinternals/index.html: $(TEXI_CPPINT_FILES)
 
-dvi:: doc/gcc.dvi doc/gccint.dvi doc/gccinstall.dvi doc/cpp.dvi \
-      doc/cppinternals.dvi lang.dvi
+DVIFILES = doc/gcc.dvi doc/gccint.dvi doc/gccinstall.dvi doc/cpp.dvi \
+           doc/cppinternals.dvi
+
+dvi:: $(DVIFILES) lang.dvi
 
 doc/%.dvi: %.texi
 	$(TEXI2DVI) -I . -I $(abs_docdir) -I $(abs_docdir)/include -o $@ $<
@@ -3797,6 +3802,18 @@ $(DESTDIR)$(infodir)/%.info: doc/%.info installdirs
 	  else true; fi; \
 	else true; fi;
 
+dvi__strip_dir = `echo $$p | sed -e 's|^.*/||'`;
+
+install-dvi: $(DVIFILES) lang.install-dvi
+	@$(NORMAL_INSTALL)
+	test -z "$(dvidir)/gcc" || $(mkinstalldirs) "$(DESTDIR)$(dvidir)/gcc"
+	@list='$(DVIFILES)'; for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  f=$(dvi__strip_dir) \
+	  echo " $(INSTALL_DATA) '$$d$$p' '$(DESTDIR)$(dvidir)/gcc/$$f'"; \
+	  $(INSTALL_DATA) "$$d$$p" "$(DESTDIR)$(dvidir)/gcc/$$f"; \
+	done
+
 pdf__strip_dir = `echo $$p | sed -e 's|^.*/||'`;
 
 install-pdf: $(PDFFILES) lang.install-pdf
diff --git a/gcc/ada/ChangeLog b/gcc/ada/ChangeLog
index 13f349c..887949b 100644
--- a/gcc/ada/ChangeLog
+++ b/gcc/ada/ChangeLog
@@ -1,3 +1,703 @@
+2021-11-15  Iain Sandoe  <iain@sandoe.co.uk>
+
+	* gcc-interface/Makefile.in: Use DSYMUTIL_FOR_TARGET in
+	libgnat/libgnarl recipies.
+
+2021-11-10  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/ada-tree.h (DECL_STUBBED_P): Delete.
+	* gcc-interface/decl.c (gnat_to_gnu_entity): Do not set it.
+	* gcc-interface/trans.c (Call_to_gnu): Use GNAT_NAME local variable
+	and adjust accordingly.  Replace test on DECL_STUBBED_P with direct
+	test on Convention and move it down in the processing.
+
+2021-11-10  Bob Duff  <duff@adacore.com>
+
+	* scng.adb (Check_Bidi): New procedure to give warning. Note
+	that this is called only for non-ASCII characters, so should not
+	be an efficiency issue.
+	(Slit): Call Check_Bidi for wide characters in string_literals.
+	(Minus_Case): Call Check_Bidi for wide characters in comments.
+	(Char_Literal_Case): Call Check_Bidi for wide characters in
+	character_literals.  Move Accumulate_Checksum down, because
+	otherwise, if Err is True, the Code is uninitialized.
+	* errout.ads: Make the obsolete nature of "Insertion character
+	?" more prominent; one should not have to read several
+	paragraphs before finding out that it's obsolete.
+
+2021-11-10  Bob Duff  <duff@adacore.com>
+
+	* repinfo.adb (List_Component_Layout): Initialize Sbit.
+
+2021-11-10  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch4.adb (Expand_Array_Equality): Fix inconsistent casing
+	in comment about the template for expansion of array equality;
+	now we use lower case for true/false/boolean.
+	(Handle_One_Dimension): Fix comment about the template for
+	expansion of array equality.
+
+2021-11-10  Bob Duff  <duff@adacore.com>
+
+	* repinfo.adb (List_Common_Type_Info, List_Object_Info): Add
+	check for In_Generic_Scope.
+	(List_Component_Layout): Check for known static values.
+	* sem_ch13.adb (Check_Record_Representation_Clause): Add check
+	for In_Generic_Scope.
+
+2021-11-10  Etienne Servais  <servais@adacore.com>
+
+	* aspects.adb, aspects.ads (Is_Aspect_Id): New function.
+	* namet-sp.ads, namet-sp.adb (Aspect_Spell_Check,
+	Attribute_Spell_Check): New Functions.
+	* par-ch13.adb (Possible_Misspelled_Aspect): Removed.
+	(With_Present): Use Aspect_Spell_Check, use Is_Aspect_Id.
+	(Get_Aspect_Specifications): Use Aspect_Spell_Check,
+	Is_Aspect_Id, Bad_Aspect.
+	* par-sync.adb (Resync_Past_Malformed_Aspect): Use Is_Aspect_Id.
+	* sem_ch13.adb (Check_One_Attr): Use Is_Aspect_Id.
+	* sem_prag.adb (Process_Restrictions_Or_Restriction_Warnings):
+	Introduce the Process_No_Specification_Of_Aspect, emit a warning
+	instead of an error on unknown aspect, hint for typos.
+	Introduce Process_No_Use_Of_Attribute to add spell check for
+	attributes too.
+	(Set_Error_Msg_To_Profile_Name): Use Is_Aspect_Id.
+	* sem_util.adb (Bad_Attribute): Use Attribute_Spell_Check.
+	(Bad_Aspect): New function.
+	* sem_util.ads (Bad_Aspect): New function.
+
+2021-11-10  Patrick Bernardi  <bernardi@adacore.com>
+
+	* libgnarl/s-taskin.adb (Initialize_ATCB): Initialize
+	T.Common.Current_Priority to Priority'First.
+	* libgnarl/s-taskin.ads (Unspecified_Priority): Redefined as -1.
+	* libgnat/system-rtems.ads: Start priority range from 1, as 0 is
+	reserved by the operating system.
+
+2021-11-10  Pierre-Alexandre Bazin  <bazin@adacore.com>
+
+	* libgnat/a-nbnbig.ads: Mark the unit as Pure.
+	* libgnat/s-aridou.adb: Add contracts and ghost code for proof.
+	(Scaled_Divide): Reorder operations and use of temporaries in
+	two places to facilitate proof.
+	* libgnat/s-aridou.ads: Add full functional contracts.
+	* libgnat/s-arit64.adb: Mark in SPARK.
+	* libgnat/s-arit64.ads: Add contracts similar to those from
+	s-aridou.ads.
+	* rtsfind.ads: Document the limitation that runtime units
+	loading does not work for private with-clauses.
+
+2021-11-10  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch3.adb (Make_Eq_Body): Adapt call to
+	Expand_Record_Equality.
+	* exp_ch4.ads, exp_ch4.adb (Expand_Composite_Equality): Remove
+	Bodies parameter; adapt comment; fix style in body; adapt calls
+	to Expand_Record_Equality.
+	(Expand_Array_Equality): Adapt calls to
+	Expand_Composite_Equality.
+	(Expand_Record_Equality): Remove Bodies parameter; adapt
+	comment; adapt call to Expand_Composite_Equality.
+	* exp_ch8.adb (Build_Body_For_Renaming): Adapt call to
+	Expand_Record_Equality.
+
+2021-11-10  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch4.adb (Expand_Composite_Equality): Handle arrays inside
+	records just like scalars; only records inside records need
+	dedicated handling.
+
+2021-11-10  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* sem_type.ads (Has_Compatible_Type): Add For_Comparison parameter.
+	* sem_type.adb (Has_Compatible_Type): Put back the reversed calls
+	to Covers guarded with For_Comparison.
+	* sem_ch4.adb (Analyze_Membership_Op) <Try_One_Interp>: Remove new
+	reversed call to Covers and set For_Comparison to true instead.
+	(Find_Comparison_Types) <Try_One_Interp>: Likewise
+	(Find_Equality_Types) <Try_One_Interp>: Likewise.
+
+2021-11-10  Yannick Moy  <moy@adacore.com>
+
+	* Makefile.rtl: Add unit.
+	* libgnat/a-nbnbin__ghost.adb: Move...
+	* libgnat/a-nbnbig.adb: ... here. Mark ghost as ignored.
+	* libgnat/a-nbnbin__ghost.ads: Move...
+	* libgnat/a-nbnbig.ads: ... here.  Add comment for purpose of
+	this unit. Mark ghost as ignored.
+	* libgnat/s-widthu.adb: Use new unit.
+	* sem_aux.adb (First_Subtype): Adapt to the case of a ghost type
+	whose freeze node is rewritten to a null statement.
+
+2021-11-10  Etienne Servais  <servais@adacore.com>
+
+	* libgnat/s-regexp.adb (Check_Well_Formed_Pattern): Fix
+	Constraint_Error on missing close bracket.
+
+2021-11-10  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch4.adb (Expand_Array_Equality): Remove check of the array
+	bound being an N_Range node; use Type_High_Bound/Type_Low_Bound,
+	which handle all kinds of array bounds.
+
+2021-11-10  Etienne Servais  <servais@adacore.com>
+
+	* sem_ch3.adb (Derived_Type_Declaration): Introduce a subprogram
+	for tree transformation. If a tree transformation is performed,
+	then warn that it would be better to reorder the interfaces.
+
+2021-11-10  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* sem_ch4.adb (Find_Non_Universal_Interpretations): Add guard.
+
+2021-11-10  Yannick Moy  <moy@adacore.com>
+
+	* par-ch4.adb (P_Primary): Adapt test for getting error message
+	on missing parentheses.
+
+2021-11-09  Ghjuvan Lacambre  <lacambre@adacore.com>
+
+	* freeze.adb (Check_Inherited_Conditions): Initialize
+	Ifaces_List.
+
+2021-11-09  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* sem_ch4.adb (Analyze_Membership_Op) <Find_Interpretation>: Handle
+	both overloaded and non-overloaded cases.
+	<Try_One_Interp>: Do a reversed call to Covers if the outcome of the
+	call to Has_Compatible_Type is false.
+	Simplify implementation after change to Find_Interpretation.
+	(Analyze_User_Defined_Binary_Op): Be prepared for previous errors.
+	(Find_Comparison_Types) <Try_One_Interp>: Do a reversed call to
+	Covers if the outcome of the call to Has_Compatible_Type is false.
+	(Find_Equality_Types) <Try_One_Interp>: Likewise.
+	* sem_type.adb (Has_Compatible_Type): Remove the reversed calls to
+	Covers.  Add explicit return on all paths.
+
+2021-11-09  Justin Squirek  <squirek@adacore.com>
+
+	* gnatls.adb (Initialize_Default_Project_Path): Remove early
+	abort on invalid path.
+
+2021-11-09  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* sprint.adb (Sprint_Node_Actual) <N_Allocator>: Also print the
+	Procedure_To_Call field if it is present.
+	<N_Extended_Return_Statement>: Also print the Storage_Pool and
+	Procedure_To_Call fields if they are present.
+	<N_Free_Statement>: Likewise.
+	<N_Simple_Return_Statement>: Likewise.
+
+2021-11-09  Patrick Bernardi  <bernardi@adacore.com>
+
+	* libgnarl/s-tasini.adb (Check_Abort_Status): Don't return 1 if
+	the task is aborting.
+	* libgnarl/s-tasini.ads (Check_Abort_Status): Update comment.
+
+2021-11-09  Patrick Bernardi  <bernardi@adacore.com>
+
+	* Makefile.rtl: Add ATOMICS_TARGET_PAIRS and
+	ATOMICS_BUILTINS_TARGET_PAIRS to LIBGNAT_TARGET_PAIRS for
+	aarch64-rtems.
+
+2021-11-09  Justin Squirek  <squirek@adacore.com>
+
+	* exp_util.adb (Ancestor_Primitive): Prevent return of an
+	subprogram alias when the ancestor primitive of the alias is the
+	same as Subp.
+
+2021-11-09  Justin Squirek  <squirek@adacore.com>
+
+	* exp_ch4.adb (Tagged_Membership): Use corresponding record type
+	when expanding a tagged membership test on protected types.
+
+2021-11-09  Alexandre Oliva  <oliva@adacore.com>
+
+	* strub.adb, strub.ads: New files.
+	* exp_attr.adb (Access_Cases): Copy strub mode to subprogram type.
+	* exp_disp.adb (Expand_Dispatching_Call): Likewise.
+	* freeze.adb (Check_Inherited_Conditions): Check that strub modes
+	match overridden subprograms and interfaces.
+	(Freeze_All): Renaming declarations too.
+	* sem_attr.adb (Resolve_Attribute): Reject 'Access to
+	strub-annotated data object.
+	* sem_ch3.adb (Derive_Subprogram): Copy strub mode to
+	inherited subprogram.
+	* sem_prag.adb (Analyze_Pragma): Propagate Strub Machine_Attribute
+	from access-to-subprogram to subprogram type when required,
+	but not from access-to-data to data type.  Mark the entity that
+	got the pragma as having a gigi rep item.
+	* sem_res.adb (Resolve): Reject implicit conversions that
+	would change strub modes.
+	(Resolve_Type_Conversions): Reject checked conversions
+	between incompatible strub modes.
+	* doc/gnat_rm/security_hardening_features.rst: Update.
+	* gnat_rm.texi: Regenerate.
+	* libgnat/a-except.ads (Raise_Exception): Revert strub-callable
+	annotation in public subprogram.
+	* libgnat/s-arit128.ads (Multiply_With_Ovflo_Check128): Likewise.
+	* libgnat/s-arit64.ads (Multiply_With_Ovflo_Check64): Likewise.
+	* libgnat/s-secsta.ads (SS_Allocate): Likewise.
+	(SS_Mark, SS_Release): Likewise.
+	* gcc-interface/Make-lang.in (GNAT_ADA_OBJS): Add ada/strub.o.
+
+2021-11-09  Etienne Servais  <servais@adacore.com>
+
+	* sinput.ads: Initialize Current_Source_Unit to No_Unit.
+
+2021-11-09  Piotr Trojanek  <trojanek@adacore.com>
+
+	* Makefile.rtl (ARM and Aarch64 VxWorks): Use atomic variants of
+	runtime units.
+	* libgnat/a-strunb__shared.ads: Mention AARCH64 and ARM as
+	supported.
+	* libgnat/s-atocou.ads: Likewise.
+
+2021-11-09  Ghjuvan Lacambre  <lacambre@adacore.com>
+
+	* sem_util.adb (Accessibility_Level): Use init_proc_level_formal
+	instead of computing a new accessibility level from Scope_Depth
+	(E).
+
+2021-11-09  Johannes Kliemann  <kliemann@adacore.com>
+
+	* vxworks7-cert-rtp-link.spec: Replace the definition of
+	__wrs_rtp_base with the base_link spec.
+	* vxworks7-cert-rtp-base-link.spec: Add base_link spec with
+	__wrs_rtp_base definition for all architectures.
+	* vxworks7-cert-rtp-base-link__ppc64.spec: Add base_link spec
+	with __wrs_rtp_base definition for ppc64.
+	* vxworks7-cert-rtp-base-link__x86.spec: Add base_link spec with
+	__wrs_rtp_base definition for x86.
+	* vxworks7-cert-rtp-base-link__x86_64.spec: Add base_link spec
+	with __wrs_rtp_base definition for x86_64.
+
+2021-11-09  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_util.adb (Copy_Parameter_List): Refactor to remove
+	repeated calls to First_Formal.
+
+2021-11-09  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch8.adb (Build_Body_For_Renaming): Remove unnecessary
+	calls to Sloc; set Handled_Statement_Sequence when building
+	subprogram body; whitespace cleanup.
+
+2021-11-09  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch4.adb (Handle_One_Dimension): Parameter N must be always
+	positive, because it is translated into expression of 'First and
+	'Last.
+
+2021-11-09  Piotr Trojanek  <trojanek@adacore.com>
+
+	* tbuild.adb (New_Occurrence_Of): Simplify by reusing
+	Make_Identifier.
+
+2021-11-09  Etienne Servais  <servais@adacore.com>
+
+	* scng.adb (Set_Start_Column): Return Column_Number'Last when
+	handling Constraint_Error. Plus reformat a strangely 7 level if
+	loop.
+
+2021-11-09  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch4.adb (Test_Empty_Arrays): Simplify with Evolve_Or_Else;
+	remove unnecessary call to Relocate_Node.
+	(Test_Lengths_Correspond): Likewise.
+
+2021-11-09  Piotr Trojanek  <trojanek@adacore.com>
+
+	* libgnat/a-strunb.adb (Deallocate): Rename Reference_Copy to
+	Old, to make the code similar to other routines in this package.
+	(Realloc_For_Chunk): Use a temporary, deallocate the previous
+	string using a null-allowing copy of the string reference.
+
+2021-11-09  Ghjuvan Lacambre  <lacambre@adacore.com>
+
+	* sem_util.adb (Accessibility_Level): Fix typos.
+
+2021-11-09  Tom Tromey  <tromey@adacore.com>
+
+	* doc/gnat_ugn/platform_specific_information.rst: Document the
+	SuSE kernel bug.
+	* gnat_ugn.texi: Regenerate.
+
+2021-11-09  Gary Dismukes  <dismukes@adacore.com>
+
+	* sem_ch13.adb (Freeze_Entity_Checks): Analyze the expression of
+	a pragma Predicate associated with an aspect at the freeze point
+	of the type, to ensure that references to globals get saved when
+	the aspect occurs within a generic body. Also, add
+	Aspect_Static_Predicate to the choices of the membership test of
+	the enclosing guard.
+
+2021-11-09  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch4.adb (Arr_Attr): Refine type of the parameter from Int
+	to Pos; refine name of the parameter from Num to Dim; fix
+	reference to "Expr" in comment.
+
+2021-11-09  Steve Baird  <baird@adacore.com>
+
+	* libgnat/s-regexp.adb (Compile.Check_Well_Formed_Patern): When
+	a "|" operator is encountered in a pattern, check that it is not
+	the last character of the pattern.
+
+2021-11-09  Yannick Moy  <moy@adacore.com>
+
+	* sem_ch3.adb (Analyze_Subtype_Declaration,
+	Derived_Type_Declaration): Initialize list of primitive
+	operations when Extensions_Allowed.
+
+2021-11-09  Piotr Trojanek  <trojanek@adacore.com>
+
+	* checks.adb (Apply_Constraint_Check): Guard against calling
+	Choices when the first association in an array aggregate is a
+	N_Iterated_Component_Association node.
+
+2021-11-09  Yannick Moy  <moy@adacore.com>
+
+	* sem_ch3.adb (Analyze_Incomplete_Type_Decl): Add the missing
+	initialization.
+
+2021-11-09  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_prag.adb (Check_Usage): Guard against calling Usage_Error
+	with illegal Item_Id. The intention to do this was already
+	described in the comment but not implemented.
+
+2021-11-09  Etienne Servais  <servais@adacore.com>
+
+	* exp_ch6.adb (Expand_Actuals): Add a condition to check for the
+	possibility of task.
+
+2021-11-05  Iain Sandoe  <iain@sandoe.co.uk>
+
+	* gcc-interface/Make-lang.in: Use iOS signal trampoline code
+	for hosted Ada tools.
+	* sigtramp-ios.c: Wrap the declarations in extern "C" when
+	the code is built by a C++ compiler.
+
+2021-10-30  Alexandre Oliva  <oliva@adacore.com>
+
+	* doc/gnat_rm/security_hardening_features.rst: Mention
+	optimization to operations with implied compares.
+
+2021-10-28  Alexandre Oliva  <oliva@adacore.com>
+
+	* doc/gnat_rm/security_hardening_features.rst
+	(Hardened Conditionals): New.
+
+2021-10-25  Arnaud Charlet  <charlet@adacore.com>
+
+	* gcc-interface/Make-lang.in, gcc-interface/Makefile.in: Remove
+	gnatfind and gnatxref.
+
+2021-10-25  Ed Schonberg  <schonberg@adacore.com>
+
+	* sem_ch4.adb (Has_Possible_Literal_Aspects): If analysis of an
+	operator node fails to find  a possible interpretation, and one
+	of its operands is a literal or a named number, assign to the
+	node the corresponding class type (Any_Integer, Any_String,
+	etc).
+	(Operator_Check): Call it before emitting a type error.
+	* sem_res.adb (Has_Applicable_User_Defined_Literal): Given a
+	literal and a type, determine whether the type has a
+	user_defined aspect that can apply to the literal, and rewrite
+	the node as call to the corresponding function. Most of the code
+	was previously in procedure Resolve.
+	(Try_User_Defined_Literal): Check operands of a predefined
+	operator that fails to resolve, and apply
+	Has_Applicable_User_Defined_Literal to literal operands if any,
+	to find if a conversion will allow the operator to resolve
+	properly.
+	(Resolve): Call the above when a literal or an operator with a
+	literal operand fails to resolve.
+
+2021-10-25  Bob Duff  <duff@adacore.com>
+
+	* freeze.adb (Freeze_Fixed_Point_Type): Remove
+	previously-inserted test for Uint_0; no longer needed.
+	* gen_il-gen.ads: Improve comments.
+	* repinfo.adb (Rep_Value): Use Ubool type for B.
+	* repinfo.ads (Node_Ref): Use Unegative type.
+	(Node_Ref_Or_Val): Document that values of this type can be
+	No_Uint.
+	* exp_disp.adb (Make_Disp_Requeue_Body): Minor comment fix.
+	* sem_ch3.adb: Likewise.
+	* sem_ch8.adb: Likewise.
+	* sinfo-utils.adb (End_Location): End_Span can never be No_Uint,
+	so remove the "if No (L)" test.
+	* uintp.adb (Image_String): Use "for ... of" loop.
+	* uintp.ads (Unegative): New type for negative integers.  We
+	give it a long name (unlike Unat and Upos) because it is rarely
+	used.
+
+2021-10-25  Etienne Servais  <servais@adacore.com>
+
+	* errout.adb (Skip_Msg_Insertion_Warning): Adapt and format as
+	Erroutc.Prescan_Message.Parse_Message_Class.
+	(Warn_Insertion): Adapt to new format.
+	* errout.ads: Update documentation.
+	* erroutc.adb (Get_Warning_Tag): Adapt to new format.
+	(Prescan_Message): Introduce Parse_Message_Class function.
+	(Validate_Specific_Warnings): Update ?W? to ?.w?.
+	* erroutc.ads: Update type and documentation.
+	* checks.adb (Validity_Check_Range): Update ?X? to ?.x?.
+	* exp_ch11.adb (Possible_Local_Raise): Update ?X? to ?.x?.
+	(Warn_If_No_Local_Raise): Likewise.
+	(Warn_If_No_Propagation): Likewise.
+	(Warn_No_Exception_Propagation_Active): Likewise.
+	* exp_ch4.adb (Expand_N_Allocator): Attach warning message to
+	-gnatw_a.
+	* exp_prag.adb (Expand_Pragma_Check): Update ?A? to ?.a?.
+	* exp_util.adb (Activate_Atomic_Synchronization): Update ?N? to
+	?.n?.
+	(Add_Invariant_Check): Update ?L? to ?.l?.
+	* freeze.adb (Check_Suspicious_Modulus): Update ?M? to ?.m?.
+	(Freeze_Entity): Update ?T? to ?.t?, ?Z? to ?.z?.
+	* par-util.adb (Warn_If_Standard_Redefinition): Update ?K? to
+	?.k?.
+	* sem_attr.adb (Min_Max): Update ?U? to ?.u?.
+	* sem_ch13.adb (Adjust_Record_For_Reverse_Bit_Order): Update ?V?
+	to ?.v?.
+	(Adjust_Record_For_Reverse_Bit_Order_Ada_95): Update ?V? to ?.v?.
+	(Component_Size_Case): Update ?S? to ?.s?.
+	(Analyze_Record_Representation_Clause): Update ?S? to ?.s? and
+	?C? to ?.c?.
+	(Add_Call): Update ?L? to ?.l?.
+	(Component_Order_Check): Attach warning message to -gnatw_r.
+	(Check_Component_List): Update ?H? to ?.h?.
+	(Set_Biased): Update ?B? to ?.b?.
+	* sem_ch3.adb (Modular_Type_Declaration): Update ?M? to ?.m?.
+	* sem_ch4.adb (Analyze_Mod): Update ?M? to ?.m?.
+	(Analyze_Quantified_Expression): Update ?T? to ?.t?.
+	* sem_ch6.adb (Check_Conformance): Attach warning message to
+	-gnatw_p.
+	(List_Inherited_Pre_Post_Aspects): Update ?L? to ?.l?.
+	* sem_ch7.adb (Unit_Requires_Body_Info): Update ?Y? to ?.y?.
+	* sem_ch8.adb (Analyze_Object_Renaming): Update ?R? to ?.r?.
+	* sem_prag.adb (Validate_Compile_Time_Warning_Or_Error): Attach
+	warning message to -gnatw_c.
+	* sem_res.adb (Check_Argument_Order): Update ?P? to ?.p?.
+	(Resolve_Comparison_Op): Update ?U? to ?.u?.
+	(Resolve_Range): Update ?U? to ?.u?.
+	(Resolve_Short_Circuit): Update ?A? to ?.a?.
+	(Resolve_Unary_Op): Update ?M? to ?.m?.
+	* sem_util.adb (Check_Result_And_Post_State): Update ?T? to ?.t?.
+	* sem_warn.adb (Output_Unused_Warnings_Off_Warnings): Update ?W?
+	to ?.w?.
+	* warnsw.ads: Update documentation for -gnatw_c.
+
+2021-10-25  Bob Duff  <duff@adacore.com>
+
+	* inline.adb (Establish_Actual_Mapping_For_Inlined_Call): Fix
+	comment.
+
+2021-10-25  Bob Duff  <duff@adacore.com>
+
+	* gen_il-gen.adb (Put_Seinfo): Generate type
+	Seinfo.Type_Only_Enum based on type
+	Gen_IL.Internals.Type_Only_Enum. Automatically generating a copy
+	of the type will help keep them in sync.  (Note that there are
+	no Ada compiler packages imported into Gen_IL.)  Add a Type_Only
+	field to Field_Descriptor, so this information is available in
+	the Ada compiler (as opposed to just in the Gen_IL "compiler").
+	(One_Comp): Add initialization of the Type_Only field of
+	Field_Descriptor.
+	* gen_il-internals.ads (Image): Image function for
+	Type_Only_Enum.
+	* atree.ads (Node_To_Fetch_From): New function to compute which
+	node to fetch from, based on the Type_Only aspect.
+	* atree.adb (Get_Field_Value): Call Node_To_Fetch_From.
+	* treepr.adb (Print_Entity_Field): Call Node_To_Fetch_From.
+	(Print_Node_Field): Assert.
+	* sinfo-utils.adb (Walk_Sinfo_Fields,
+	Walk_Sinfo_Fields_Pairwise): Asserts.
+
+2021-10-25  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch4.adb (Expand_Composite_Equality): Fix style.
+	(Element_To_Compare): Simplify loop.
+	(Expand_Record_Equality): Adapt calls to Element_To_Compare.
+
+2021-10-25  Steve Baird  <baird@adacore.com>
+
+	* sem_case.adb (Composite_Case_Ops.Box_Value_Required): A new
+	function which takes a component type and returns a Boolean.
+	Returns True for the cases which were formerly forbidden as
+	components (these checks were formerly performed in the
+	now-deleted procedure
+	Check_Composite_Case_Selector.Check_Component_Subtype).
+	(Composite_Case_Ops.Normalized_Case_Expr_Type): Hoist this
+	function out of the Array_Case_Ops package because it has been
+	generalized to also do the analogous thing in the case of a
+	discriminated type.
+	(Composite_Case_Ops.Scalar_Part_Count): Return 0 if
+	Box_Value_Required returns True for the given type/subtype.
+	(Composite_Case_Ops.Choice_Analysis.Choice_Analysis.Component_Bounds_Info.
+	Traverse_Discrete_Parts): Return without doing anything if
+	Box_Value_Required returns True for the given type/subtype.
+	(Composite_Case_Ops.Choice_Analysis.Parse_Choice.Traverse_Choice):
+	If Box_Value_Required yields True for a given component type,
+	then check that the value of that component in a choice
+	expression is indeed a box (in which case the component is
+	ignored).
+	* doc/gnat_rm/implementation_defined_pragmas.rst: Update
+	documentation.
+	* gnat_rm.texi: Regenerate.
+
+2021-10-25  Piotr Trojanek  <trojanek@adacore.com>
+
+	* libgnat/a-nbnbin__gmp.adb (From_String): Fix predicate
+	mismatch between subprogram declaration and body.
+
+2021-10-25  Bob Duff  <duff@adacore.com>
+
+	* einfo-utils.adb (Declaration_Node): Avoid returning the
+	following node kinds: N_Assignment_Statement, N_Integer_Literal,
+	N_Procedure_Call_Statement, N_Subtype_Indication, and
+	N_Type_Conversion.  Assert that the result is in N_Is_Decl or
+	empty.
+	* gen_il-gen-gen_nodes.adb (N_Is_Decl): Modify to match the
+	things that Declaration_Node can return.
+
+2021-10-25  Piotr Trojanek  <trojanek@adacore.com>
+
+	* libgnat/a-strsup.ads (Super_Length, Super_Element,
+	Super_Slice): Add Global contracts.
+
+2021-10-25  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch3.adb (Predefined_Primitive_Bodies): Simplify detection
+	of existing equality operator.
+
+2021-10-25  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch3.adb (Predefined_Primitive_Bodies): Remove redundant
+	conditions related to interface types.
+
+2021-10-25  Piotr Trojanek  <trojanek@adacore.com>
+
+	* adaint.c (__gnat_portable_spawn): Do not expect execv to
+	return 0.
+	(__gnat_portable_no_block_spawn): Likewise.
+
+2021-10-25  Ghjuvan Lacambre  <lacambre@adacore.com>
+
+	* sem_ch8.adb (Analyze_Subprogram_Renaming): Set New_S to Empty.
+
+2021-10-25  Piotr Trojanek  <trojanek@adacore.com>
+
+	* libgnat/a-strunb.ads (Unbounded_String): Reference is never
+	null.
+	* libgnat/a-strunb.adb (Finalize): Copy reference while it needs
+	to be deallocated.
+
+2021-10-25  Piotr Trojanek  <trojanek@adacore.com>
+
+	* lib-xref.adb (Get_Through_Renamings): Exit loop when an
+	enumeration literal is found.
+
+2021-10-25  Arnaud Charlet  <charlet@adacore.com>
+
+	* libgnat/s-widthu.adb: Add pragma Annotate.
+
+2021-10-25  Javier Miranda  <miranda@adacore.com>
+
+	* sem_ch8.adb (Build_Class_Wide_Wrapper): Previous version split
+	in two subprograms to factorize its functionality:
+	Find_Suitable_Candidate, and Build_Class_Wide_Wrapper. These
+	routines are also placed in the new subprogram
+	Handle_Instance_With_Class_Wide_Type.
+	(Handle_Instance_With_Class_Wide_Type): New subprogram that
+	encapsulates all the code that handles instantiations with
+	class-wide types.
+	(Analyze_Subprogram_Renaming): Adjust code to invoke the new
+	nested subprogram Handle_Instance_With_Class_Wide_Type; adjust
+	documentation.
+
+2021-10-25  Bob Duff  <duff@adacore.com>
+
+	* einfo-utils.ads, einfo-utils.adb (Alias, Set_Alias,
+	Renamed_Entity, Set_Renamed_Entity, Renamed_Object,
+	Set_Renamed_Object): Add assertions that reflect how these are
+	supposed to be used and what they are supposed to return.
+	(Renamed_Entity_Or_Object): New getter.
+	(Set_Renamed_Object_Of_Possibly_Void): Setter that allows N to
+	be E_Void.
+	* checks.adb (Ensure_Valid): Use Renamed_Entity_Or_Object
+	because this is called for both cases.
+	* exp_dbug.adb (Debug_Renaming_Declaration): Use
+	Renamed_Entity_Or_Object because this is called for both cases.
+	Add assertions.
+	* exp_util.adb (Possible_Bit_Aligned_Component): Likewise.
+	* freeze.adb (Freeze_All_Ent): Likewise.
+	* sem_ch5.adb (Within_Function): Likewise.
+	* exp_attr.adb (Calculate_Header_Size): Call Renamed_Entity
+	instead of Renamed_Object.
+	* exp_ch11.adb (Expand_N_Raise_Statement): Likewise.
+	* repinfo.adb (Find_Declaration): Likewise.
+	* sem_ch10.adb (Same_Unit, Process_Spec_Clauses,
+	Analyze_With_Clause, Install_Parents): Likewise.
+	* sem_ch12.adb (Build_Local_Package, Needs_Body_Instantiated,
+	Build_Subprogram_Renaming, Check_Formal_Package_Instance,
+	Check_Generic_Actuals, In_Enclosing_Instance,
+	Denotes_Formal_Package, Process_Nested_Formal,
+	Check_Initialized_Types, Map_Formal_Package_Entities,
+	Restore_Nested_Formal): Likewise.
+	* sem_ch6.adb (Report_Conflict): Likewise.
+	* sem_ch8.adb (Analyze_Exception_Renaming,
+	Analyze_Generic_Renaming, Analyze_Package_Renaming,
+	Is_Primitive_Operator_In_Use, Declared_In_Actual,
+	Note_Redundant_Use): Likewise.
+	* sem_warn.adb (Find_Package_Renaming): Likewise.
+	* sem_elab.adb (Ultimate_Variable): Call Renamed_Object instead
+	of Renamed_Entity.
+	* exp_ch6.adb (Get_Function_Id): Call
+	Set_Renamed_Object_Of_Possibly_Void, because the defining
+	identifer is still E_Void at this point.
+	* sem_util.adb (Function_Call_Or_Allocator_Level): Likewise.
+	Remove redundant (unreachable) code.
+	(Is_Object_Renaming, Is_Valid_Renaming): Call Renamed_Object
+	instead of Renamed_Entity.
+	(Get_Fullest_View): Call Renamed_Entity instead of
+	Renamed_Object.
+	(Copy_Node_With_Replacement): Call
+	Set_Renamed_Object_Of_Possibly_Void because the defining entity
+	is sometimes E_Void.
+	* exp_ch5.adb (Expand_N_Assignment_Statement): Protect a call to
+	Renamed_Object with Is_Object to avoid assertion failure.
+	* einfo.ads: Minor comment fixes.
+	* inline.adb: Minor comment fixes.
+	* tbuild.ads: Minor comment fixes.
+
+2021-10-25  Arnaud Charlet  <charlet@adacore.com>
+
+	* sem_ch13.adb (Build_Discrete_Static_Predicate): Remove use of
+	exception propagation since this code is exercised during the
+	bootstrap.
+
+2021-10-25  Yannick Moy  <moy@adacore.com>
+
+	* sem_ch13.adb (Freeze_Entity_Checks): Perform same check on
+	predicate expression inside pragma as inside aspect.
+	* sem_util.adb (Is_Current_Instance): Recognize possible
+	occurrence of subtype as current instance inside the pragma
+	Predicate.
+
+2021-10-25  Ghjuvan Lacambre  <lacambre@adacore.com>
+
+	* erroutc.adb (Count_Compile_Time_Pragma_Warnings): Don't count
+	deleted warnings.
+
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* gcc-interface/Make-lang.in: Allow dvi-formatted
+	documentation to be installed.
+
 2021-10-20  Bob Duff  <duff@adacore.com>
 
 	* gcc-interface/decl.c (gnat_to_gnu_entity): Remove unnecessary
diff --git a/gcc/ada/Makefile.rtl b/gcc/ada/Makefile.rtl
index ebfbe95..282b25c 100644
--- a/gcc/ada/Makefile.rtl
+++ b/gcc/ada/Makefile.rtl
@@ -211,6 +211,7 @@ GNATRTL_NONTASKING_OBJS= \
   a-lllwti$(objext) \
   a-lllzti$(objext) \
   a-locale$(objext) \
+  a-nbnbig$(objext) \
   a-nbnbin$(objext) \
   a-nbnbre$(objext) \
   a-ncelfu$(objext) \
@@ -1362,7 +1363,9 @@ ifeq ($(strip $(filter-out aarch64 arm% coff wrs vx%,$(target_cpu) $(target_vend
   g-socthi.ads<libgnat/g-socthi__vxworks.ads \
   g-socthi.adb<libgnat/g-socthi__vxworks.adb \
   g-sopowa.adb<libgnat/g-sopowa__posix.adb \
-  g-stsifd.adb<libgnat/g-stsifd__sockets.adb
+  g-stsifd.adb<libgnat/g-stsifd__sockets.adb \
+  $(ATOMICS_TARGET_PAIRS) \
+  $(ATOMICS_BUILTINS_TARGET_PAIRS)
 
   ifeq ($(strip $(filter-out aarch64, $(target_cpu))),)
     ARCH_STR=aarch64
@@ -2070,6 +2073,13 @@ ifeq ($(strip $(filter-out rtems%,$(target_os))),)
   ifeq ($(strip $(filter-out aarch64% riscv%,$(target_cpu))),)
     LIBGNAT_TARGET_PAIRS += a-nallfl.ads<libgnat/a-nallfl__wraplf.ads
   endif
+
+  # Target CPUs that support atomic primitives
+  ifeq ($(strip $(filter-out aarch64%,$(target_cpu))),)
+     LIBGNAT_TARGET_PAIRS += \
+       $(ATOMICS_TARGET_PAIRS) \
+       $(ATOMICS_BUILTINS_TARGET_PAIRS)
+  endif
 endif
 
 # PikeOS
diff --git a/gcc/ada/adaint.c b/gcc/ada/adaint.c
index 60cfa93..6781728 100644
--- a/gcc/ada/adaint.c
+++ b/gcc/ada/adaint.c
@@ -2424,8 +2424,10 @@ __gnat_portable_spawn (char *args[] ATTRIBUTE_UNUSED)
   if (pid == 0)
     {
       /* The child. */
-      if (execv (args[0], MAYBE_TO_PTR32 (args)) != 0)
-	_exit (1);
+      execv (args[0], MAYBE_TO_PTR32 (args));
+
+      /* execv() returns only on error */
+      _exit (1);
     }
 
   /* The parent.  */
@@ -2822,8 +2824,10 @@ __gnat_portable_no_block_spawn (char *args[] ATTRIBUTE_UNUSED)
   if (pid == 0)
     {
       /* The child.  */
-      if (execv (args[0], MAYBE_TO_PTR32 (args)) != 0)
-	_exit (1);
+      execv (args[0], MAYBE_TO_PTR32 (args));
+
+      /* execv() returns only on error */
+      _exit (1);
     }
 
   return pid;
diff --git a/gcc/ada/aspects.adb b/gcc/ada/aspects.adb
index a6e4f28..bf661b9 100644
--- a/gcc/ada/aspects.adb
+++ b/gcc/ada/aspects.adb
@@ -324,6 +324,16 @@ package body Aspects is
    end Has_Aspect;
 
    ------------------
+   -- Is_Aspect_Id --
+   ------------------
+
+   function Is_Aspect_Id (Aspect : Name_Id) return Boolean is
+     (Get_Aspect_Id (Aspect) /= No_Aspect);
+
+   function Is_Aspect_Id (Aspect : Node_Id) return Boolean is
+     (Get_Aspect_Id (Aspect) /= No_Aspect);
+
+   ------------------
    -- Move_Aspects --
    ------------------
 
diff --git a/gcc/ada/aspects.ads b/gcc/ada/aspects.ads
index ab11bfd..4bb28ce 100644
--- a/gcc/ada/aspects.ads
+++ b/gcc/ada/aspects.ads
@@ -773,6 +773,14 @@ package Aspects is
    --  Given an aspect specification, return the corresponding aspect_id value.
    --  If the name does not match any aspect, return No_Aspect.
 
+   function Is_Aspect_Id (Aspect : Name_Id) return Boolean;
+   pragma Inline (Is_Aspect_Id);
+   --  Return True if a corresponding aspect id exists
+
+   function Is_Aspect_Id (Aspect : Node_Id) return Boolean;
+   pragma Inline (Is_Aspect_Id);
+   --  Return True if a corresponding aspect id exists
+
    ------------------------------------
    -- Delaying Evaluation of Aspects --
    ------------------------------------
diff --git a/gcc/ada/atree.adb b/gcc/ada/atree.adb
index 98614e8..88d766a 100644
--- a/gcc/ada/atree.adb
+++ b/gcc/ada/atree.adb
@@ -854,14 +854,15 @@ package body Atree is
      (N : Node_Id; Field : Node_Or_Entity_Field) return Field_Size_32_Bit
    is
       Desc : Field_Descriptor renames Field_Descriptors (Field);
+      NN : constant Node_Or_Entity_Id := Node_To_Fetch_From (N, Field);
 
    begin
       case Field_Size (Desc.Kind) is
-         when 1 => return Field_Size_32_Bit (Get_1_Bit_Val (N, Desc.Offset));
-         when 2 => return Field_Size_32_Bit (Get_2_Bit_Val (N, Desc.Offset));
-         when 4 => return Field_Size_32_Bit (Get_4_Bit_Val (N, Desc.Offset));
-         when 8 => return Field_Size_32_Bit (Get_8_Bit_Val (N, Desc.Offset));
-         when others => return Get_32_Bit_Val (N, Desc.Offset);  -- 32
+         when 1 => return Field_Size_32_Bit (Get_1_Bit_Val (NN, Desc.Offset));
+         when 2 => return Field_Size_32_Bit (Get_2_Bit_Val (NN, Desc.Offset));
+         when 4 => return Field_Size_32_Bit (Get_4_Bit_Val (NN, Desc.Offset));
+         when 8 => return Field_Size_32_Bit (Get_8_Bit_Val (NN, Desc.Offset));
+         when others => return Get_32_Bit_Val (NN, Desc.Offset);  -- 32
       end case;
    end Get_Field_Value;
 
diff --git a/gcc/ada/atree.ads b/gcc/ada/atree.ads
index 4861236..c239507 100644
--- a/gcc/ada/atree.ads
+++ b/gcc/ada/atree.ads
@@ -47,6 +47,7 @@
 with Alloc;
 with Sinfo.Nodes;    use Sinfo.Nodes;
 with Einfo.Entities; use Einfo.Entities;
+with Einfo.Utils;    use Einfo.Utils;
 with Types;          use Types;
 with Seinfo;         use Seinfo;
 with System;         use System;
@@ -616,6 +617,20 @@ package Atree is
    --  always the same; for example we change from E_Void, to E_Variable, to
    --  E_Void, to E_Constant.
 
+   function Node_To_Fetch_From
+     (N : Node_Or_Entity_Id; Field : Node_Or_Entity_Field)
+     return Node_Or_Entity_Id is
+      (case Field_Descriptors (Field).Type_Only is
+         when No_Type_Only => N,
+         when Base_Type_Only => Base_Type (N),
+         when Impl_Base_Type_Only => Implementation_Base_Type (N),
+         when Root_Type_Only => Root_Type (N));
+   --  This is analogous to the same-named function in Gen_IL.Gen. Normally,
+   --  Type_Only is No_Type_Only, and we fetch the field from the node N. But
+   --  if Type_Only = Base_Type_Only, we need to go to the Base_Type, and
+   --  similarly for the other two cases. This can return something other
+   --  than N only if N is an Entity.
+
    -----------------------------
    -- Private Part Subpackage --
    -----------------------------
diff --git a/gcc/ada/checks.adb b/gcc/ada/checks.adb
index c85cba9a5..6fb33b4 100644
--- a/gcc/ada/checks.adb
+++ b/gcc/ada/checks.adb
@@ -847,7 +847,7 @@ package body Checks is
             else
                Error_Msg_N
                  ("\address value may be incompatible with alignment of "
-                  & "object?X?", AC);
+                  & "object?.x?", AC);
             end if;
          end if;
 
@@ -1388,6 +1388,8 @@ package body Checks is
 
          if Nkind (N) = N_Aggregate
            and then No (Expressions (N))
+           and then Nkind (First (Component_Associations (N))) =
+             N_Component_Association
            and then Nkind
             (First (Choices (First (Component_Associations (N)))))
               = N_Others_Choice
@@ -6676,8 +6678,9 @@ package body Checks is
       elsif not Comes_From_Source (Expr)
         and then not
           (Nkind (Expr) = N_Identifier
-            and then Present (Renamed_Object (Entity (Expr)))
-            and then Comes_From_Source (Renamed_Object (Entity (Expr))))
+            and then Present (Renamed_Entity_Or_Object (Entity (Expr)))
+            and then
+              Comes_From_Source (Renamed_Entity_Or_Object (Entity (Expr))))
         and then not Force_Validity_Checks
         and then (Nkind (Expr) /= N_Unchecked_Type_Conversion
                     or else Kill_Range_Check (Expr))
diff --git a/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst b/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst
index 0375982..768dd668 100644
--- a/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst
+++ b/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst
@@ -2268,9 +2268,24 @@ of GNAT specific extensions are recognized as follows:
   set shall be a proper subset of the second (and the later alternative
   will not be executed if the earlier alternative "matches"). All possible
   values of the composite type shall be covered. The composite type of the
-  selector shall be a nonlimited untagged (but possibly discriminated)
-  record type, all of whose subcomponent subtypes are either static discrete
-  subtypes or record types that meet the same restrictions.
+  selector shall be an array or record type that is neither limited
+  class-wide.
+
+  If a subcomponent's subtype does not meet certain restrictions, then
+  the only value that can be specified for that subcomponent in a case
+  choice expression is a "box" component association (which matches all
+  possible values for the subcomponent). This restriction applies if
+
+  - the component subtype is not a record, array, or discrete type; or
+
+  - the component subtype is subject to a non-static constraint or
+    has a predicate; or
+
+  - the component type is an enumeration type that is subject to an
+    enumeration representation clause; or
+
+  - the component type is a multidimensional array type or an
+    array type with a nonstatic index subtype.
 
   Support for casing on arrays (and on records that contain arrays) is
   currently subject to some restrictions. Non-positional
diff --git a/gcc/ada/doc/gnat_rm/security_hardening_features.rst b/gcc/ada/doc/gnat_rm/security_hardening_features.rst
index 1c46e3a4..bdcfd99 100644
--- a/gcc/ada/doc/gnat_rm/security_hardening_features.rst
+++ b/gcc/ada/doc/gnat_rm/security_hardening_features.rst
@@ -73,17 +73,90 @@ Note that Ada secondary stacks are not scrubbed.  The restriction
 ``No_Secondary_Stack`` avoids their use, and thus their accidental
 preservation of data that should be scrubbed.
 
-Also note that the machine attribute is not integrated in the Ada type
-system.  Though it may modify subprogram and variable interfaces, it
-is not fully reflected in Ada types, ``Access`` attributes, renaming
-and overriding.  Every access type, renaming, and overriding and
-overridden dispatching operations that may refer to an entity with an
-attribute-modified interface must be annotated with the same
-interface-modifying attribute, or with an interface-compatible one.
-
-Even then, the pragma is currently only functional when applied to
-subprograms and scalar variables; other uses, such as directly on
-types and subtypes, may be silently ignored.  Specifically, it is not
-currently recommended to rely on any effects this pragma might be
-expected to have when calling subprograms through access-to-subprogram
-variables.
+Attributes ``Access`` and ``Unconstrained_Access`` of variables and
+constants with ``strub`` enabled require types with ``strub`` enabled;
+there is no way to express an access-to-strub type otherwise.
+``Unchecked_Access`` bypasses this constraint, but the resulting
+access type designates a non-strub type.
+
+.. code-block:: ada
+
+     VI : Integer;
+     XsVI : access Integer := VI'Access; -- Error.
+     UXsVI : access Integer := VI'Unchecked_Access; -- OK,
+     -- UXsVI.all does not enable strub in the enclosing subprogram.
+
+     type Strub_Int is new Integer;
+     pragma Machine_Attribute (Strub_Int, "strub");
+     VSI : Strub_Int;
+     XsVSI : access Strub_Int := VSI'Access; -- OK.
+     -- XsVSI.all enables strub in the enclosing subprogram.
+
+
+Every access-to-subprogram type, renaming, and overriding and
+overridden dispatching operations that may refer to a subprogram with
+an attribute-modified interface must be annotated with the same
+interface-modifying attribute.  Access-to-subprogram types can be
+explicitly converted to different strub modes, as long as they are
+interface-compatible (i.e., adding or removing ``at-calls`` is not
+allowed).  For example, a ``strub``-``disabled`` subprogram can be
+turned ``callable`` through such an explicit conversion:
+
+.. code-block:: ada
+
+     type TBar is access procedure;
+
+     type TBar_Callable is access procedure;
+     pragma Machine_Attribute (TBar_Callable, "strub", "callable");
+
+     Bar_Callable_Ptr : constant TBar_Callable
+		:= TBar_Callable (TBar'(Bar'Access));
+
+     procedure Bar_Callable renames Bar_Callable_Ptr.all;
+     pragma Machine_Attribute (Bar_Callable, "strub", "callable");
+
+Note that the renaming declaration is expanded to a full subprogram
+body, it won't be just an alias.  Only if it is inlined will it be as
+efficient as a call by dereferencing the access-to-subprogram constant
+Bar_Callable_Ptr.
+
+
+.. Hardened Conditionals:
+
+Hardened Conditionals
+=====================
+
+GNAT can harden conditionals to protect against control flow attacks.
+
+This is accomplished by two complementary transformations, each
+activated by a separate command-line option.
+
+The option *-fharden-compares* enables hardening of compares that
+compute results stored in variables, adding verification that the
+reversed compare yields the opposite result.
+
+The option *-fharden-conditional-branches* enables hardening of
+compares that guard conditional branches, adding verification of the
+reversed compare to both execution paths.
+
+These transformations are introduced late in the compilation pipeline,
+long after boolean expressions are decomposed into separate compares,
+each one turned into either a conditional branch or a compare whose
+result is stored in a boolean variable or temporary.  Compiler
+optimizations, if enabled, may also turn conditional branches into
+stored compares, and vice-versa, or into operations with implied
+conditionals (e.g. MIN and MAX).  Conditionals may also be optimized
+out entirely, if their value can be determined at compile time, and
+occasionally multiple compares can be combined into one.
+
+It is thus difficult to predict which of these two options will affect
+a specific compare operation expressed in source code.  Using both
+options ensures that every compare that is neither optimized out nor
+optimized into implied conditionals will be hardened.
+
+The addition of reversed compares can be observed by enabling the dump
+files of the corresponding passes, through command line options
+*-fdump-tree-hardcmp* and *-fdump-tree-hardcbr*, respectively.
+
+They are separate options, however, because of the significantly
+different performance impact of the hardening transformations.
diff --git a/gcc/ada/doc/gnat_ugn/platform_specific_information.rst b/gcc/ada/doc/gnat_ugn/platform_specific_information.rst
index 4f68d25..fd2c6bd 100644
--- a/gcc/ada/doc/gnat_ugn/platform_specific_information.rst
+++ b/gcc/ada/doc/gnat_ugn/platform_specific_information.rst
@@ -233,6 +233,7 @@ GNU/Linux Topics
 This section describes topics that are specific to GNU/Linux platforms.
 
 .. _Required_packages_on_GNU_Linux:
+.. _A_GNU_Linux_debug_quirk:
 
 Required Packages on GNU/Linux
 ------------------------------
@@ -252,6 +253,22 @@ you'll need the 32-bit version of the following packages:
 Other GNU/Linux distributions might be choosing a different name
 for those packages.
 
+A GNU/Linux Debug Quirk
+-----------------------
+
+On SuSE 15, some kernels have a defect causing issues when debugging
+programs using threads or Ada tasks. Due to the lack of documentation
+found regarding this kernel issue, we can only provide limited
+information about which kernels are impacted: kernel version 5.3.18 is
+known to be impacted, and kernels in the 5.14 range or newer are
+believed to fix this problem.
+
+The bug affects the debugging of 32-bit processes on a 64-bit system.
+Symptoms can vary: Unexpected ``SIGABRT`` signals being received by
+the program, "The futex facility returned an unexpected error code"
+error message, and inferior programs hanging indefinitely range among
+the symptoms most commonly observed.
+
 .. index:: Windows
 
 .. _Microsoft_Windows_Topics:
diff --git a/gcc/ada/einfo-utils.adb b/gcc/ada/einfo-utils.adb
index 0c89c82..763b646 100644
--- a/gcc/ada/einfo-utils.adb
+++ b/gcc/ada/einfo-utils.adb
@@ -43,24 +43,87 @@ package body Einfo.Utils is
    --  Determine whether abstract state State_Id has particular option denoted
    --  by the name Option_Nam.
 
-   -----------------------------------
-   -- Renamings of Renamed_Or_Alias --
-   -----------------------------------
+   -------------------------------------------
+   -- Aliases/Renamings of Renamed_Or_Alias --
+   -------------------------------------------
 
    function Alias (N : Entity_Id) return Node_Id is
    begin
-      pragma Assert
-        (Is_Overloadable (N) or else Ekind (N) = E_Subprogram_Type);
-      return Renamed_Or_Alias (N);
+      return Val : constant Node_Id := Renamed_Or_Alias (N) do
+         pragma Assert
+           (Is_Overloadable (N) or else Ekind (N) = E_Subprogram_Type);
+         pragma Assert (Val in N_Entity_Id | N_Empty_Id);
+      end return;
    end Alias;
 
    procedure Set_Alias (N : Entity_Id; Val : Node_Id) is
    begin
       pragma Assert
         (Is_Overloadable (N) or else Ekind (N) = E_Subprogram_Type);
+      pragma Assert (Val in N_Entity_Id | N_Empty_Id);
+
       Set_Renamed_Or_Alias (N, Val);
    end Set_Alias;
 
+   function Renamed_Entity (N : Entity_Id) return Node_Id is
+   begin
+      return Val : constant Node_Id := Renamed_Or_Alias (N) do
+         pragma Assert (not Is_Object (N) or else Etype (N) = Any_Type);
+         pragma Assert (Val in N_Entity_Id | N_Empty_Id);
+      end return;
+   end Renamed_Entity;
+
+   procedure Set_Renamed_Entity (N : Entity_Id; Val : Node_Id) is
+   begin
+      pragma Assert (not Is_Object (N));
+      pragma Assert (Val in N_Entity_Id);
+
+      Set_Renamed_Or_Alias (N, Val);
+   end Set_Renamed_Entity;
+
+   function Renamed_Object (N : Entity_Id) return Node_Id is
+   begin
+      return Val : constant Node_Id := Renamed_Or_Alias (N) do
+         --  Formal_Kind uses the entity, not a name of it. This happens
+         --  in front-end inlining, which also sets to Empty. Also in
+         --  Exp_Ch9, where formals are renamed for the benefit of gdb.
+
+         if Ekind (N) not in Formal_Kind then
+            pragma Assert (Is_Object (N));
+            pragma Assert (Val in N_Subexpr_Id | N_Empty_Id);
+            null;
+         end if;
+      end return;
+   end Renamed_Object;
+
+   procedure Set_Renamed_Object (N : Entity_Id; Val : Node_Id) is
+   begin
+      if Ekind (N) not in Formal_Kind then
+         pragma Assert (Is_Object (N));
+         pragma Assert (Val in N_Subexpr_Id | N_Empty_Id);
+         null;
+      end if;
+
+      Set_Renamed_Or_Alias (N, Val);
+   end Set_Renamed_Object;
+
+   function Renamed_Entity_Or_Object (N : Entity_Id) return Node_Id is
+   begin
+      if Is_Object (N) then
+         return Renamed_Object (N);
+      else
+         return Renamed_Entity (N);
+      end if;
+   end Renamed_Entity_Or_Object;
+
+   procedure Set_Renamed_Object_Of_Possibly_Void
+     (N : Entity_Id; Val : Node_Id)
+   is
+   begin
+      pragma Assert (Val in N_Subexpr_Id);
+      Set_Renamed_Or_Alias (N, Val);
+   end Set_Renamed_Object_Of_Possibly_Void;
+
    ----------------
    -- Has_Option --
    ----------------
@@ -635,6 +698,30 @@ package body Einfo.Utils is
          P := Empty;
       end if;
 
+      --  Declarations are sometimes removed by replacing them with other
+      --  irrelevant nodes. For example, a declare expression can be turned
+      --  into a literal by constant folding. In these cases we want to
+      --  return Empty.
+
+      if Nkind (P) in
+          N_Assignment_Statement
+        | N_Integer_Literal
+        | N_Procedure_Call_Statement
+        | N_Subtype_Indication
+        | N_Type_Conversion
+      then
+         P := Empty;
+      end if;
+
+      --  The following Assert indicates what kinds of nodes can be returned;
+      --  they are not all "declarations".
+
+      if Serious_Errors_Detected = 0 then
+         pragma Assert
+           (Nkind (P) in N_Is_Decl | N_Empty,
+            "Declaration_Node incorrect kind: " & Node_Kind'Image (Nkind (P)));
+      end if;
+
       return P;
    end Declaration_Node;
 
diff --git a/gcc/ada/einfo-utils.ads b/gcc/ada/einfo-utils.ads
index 0e6c8cd..c82b469 100644
--- a/gcc/ada/einfo-utils.ads
+++ b/gcc/ada/einfo-utils.ads
@@ -27,25 +27,37 @@ with Einfo.Entities; use Einfo.Entities;
 
 package Einfo.Utils is
 
-   -----------------------------------
-   -- Renamings of Renamed_Or_Alias --
-   -----------------------------------
+   -------------------------------------------
+   -- Aliases/Renamings of Renamed_Or_Alias --
+   -------------------------------------------
 
    --  See the comment in einfo.ads, "Renaming and Aliasing", which is somewhat
-   --  incorrect. In fact, the compiler uses Alias, Renamed_Entity, and
-   --  Renamed_Object more-or-less interchangeably, so we rename them here.
-   --  Alias isn't really renamed, because we want an assertion in the body.
+   --  incorrect. Each of the following calls [Set_]Renamed_Or_Alias. Alias and
+   --  Renamed_Entity are fields of nonobject Entity_Ids, and the value of the
+   --  field is Entity_Id. Alias is only for callable entities and subprogram
+   --  types. We sometimes call Set_Renamed_Entity and then expect Alias to
+   --  return the value set. Renamed_Object is a field of Entity_Ids that are
+   --  objects, and it returns an expression, because you can rename things
+   --  like "X.all(J).Y". Renamings of entries and subprograms can also be
+   --  expressions, but those use different mechanisms; the fields here are not
+   --  used.
 
    function Alias (N : Entity_Id) return Node_Id;
    procedure Set_Alias (N : Entity_Id; Val : Node_Id);
-   function Renamed_Entity
-     (N : Entity_Id) return Node_Id renames Renamed_Or_Alias;
-   procedure Set_Renamed_Entity
-     (N : Entity_Id; Val : Node_Id) renames Set_Renamed_Or_Alias;
-   function Renamed_Object
-     (N : Entity_Id) return Node_Id renames Renamed_Or_Alias;
-   procedure Set_Renamed_Object
-     (N : Entity_Id; Val : Node_Id) renames Set_Renamed_Or_Alias;
+   function Renamed_Entity (N : Entity_Id) return Node_Id;
+   procedure Set_Renamed_Entity (N : Entity_Id; Val : Node_Id);
+   function Renamed_Object (N : Entity_Id) return Node_Id;
+   procedure Set_Renamed_Object (N : Entity_Id; Val : Node_Id);
+
+   function Renamed_Entity_Or_Object (N : Entity_Id) return Node_Id;
+   --  This getter is used when we don't know statically whether we want to
+   --  call Renamed_Entity or Renamed_Object.
+
+   procedure Set_Renamed_Object_Of_Possibly_Void
+     (N : Entity_Id; Val : Node_Id);
+   --  Set_Renamed_Object doesn't allow Void; this is used in the rare cases
+   --  where we set the field of an entity that might be Void. It might be a
+   --  good idea to get rid of calls to this.
 
    pragma Inline (Alias);
    pragma Inline (Set_Alias);
@@ -53,6 +65,8 @@ package Einfo.Utils is
    pragma Inline (Set_Renamed_Entity);
    pragma Inline (Renamed_Object);
    pragma Inline (Set_Renamed_Object);
+   pragma Inline (Renamed_Entity_Or_Object);
+   pragma Inline (Set_Renamed_Object_Of_Possibly_Void);
 
    -------------------
    -- Type Synonyms --
diff --git a/gcc/ada/einfo.ads b/gcc/ada/einfo.ads
index 13440ce..51cb014 100644
--- a/gcc/ada/einfo.ads
+++ b/gcc/ada/einfo.ads
@@ -4762,7 +4762,7 @@ package Einfo is
 
 --  Several entity attributes relate to renaming constructs, and to the use of
 --  different names to refer to the same entity. The following is a summary of
---  these constructs and their prefered uses.
+--  these constructs and their preferred uses.
 
 --  There are three related attributes:
 
@@ -4775,7 +4775,7 @@ package Einfo is
 
 --  a) Renamed_Entity applies to entities in renaming declarations that rename
 --  an entity, so the value of the attribute IS an entity. This applies to
---  generic renamings, package renamings, exception renamings, and subprograms
+--  generic renamings, package renamings, exception renamings, and subprogram
 --  renamings that rename a subprogram (rather than an attribute, an entry, a
 --  protected operation, etc).
 
@@ -4797,9 +4797,9 @@ package Einfo is
 --  in a few cases we need to use a loop to trace a chain of object renamings
 --  where all of them happen to be entities. So:
 
---    X : integer;
---    Y : integer renames X;   -- renamed object is the identifier X
---    Z : integer renames Y;   -- renamed object is the identifier Y
+--    X : Integer;
+--    Y : Integer renames X;   -- renamed object is the identifier X
+--    Z : Integer renames Y;   -- renamed object is the identifier Y
 
 --  The front-end does not store explicitly the fact that Z renames X.
 
diff --git a/gcc/ada/errout.adb b/gcc/ada/errout.adb
index 05a8266..76a8268 100644
--- a/gcc/ada/errout.adb
+++ b/gcc/ada/errout.adb
@@ -211,12 +211,9 @@ package body Errout is
    --  This is called for warning messages only (so Warning_Msg_Char is set)
    --  and returns a corresponding string to use at the beginning of generated
    --  auxiliary messages, such as "in instantiation at ...".
-   --    'a' .. 'z'   returns "?x?"
-   --    'A' .. 'Z'   returns "?X?"
-   --    '*'          returns "?*?"
-   --    '$'          returns "?$?info: "
-   --    ' '          returns " "
-   --  No other settings are valid
+   --    "?"     returns "??"
+   --    " "     returns "?"
+   --    other   trimmed, prefixed and suffixed with "?".
 
    -----------------------
    -- Change_Error_Text --
@@ -1177,7 +1174,7 @@ package body Errout is
                   Errors.Table (Cur_Msg).Warn := True;
                   Errors.Table (Cur_Msg).Warn_Chr := Warning_Msg_Char;
 
-               elsif Warning_Msg_Char /= ' ' then
+               elsif Warning_Msg_Char /= "  " then
                   Errors.Table (Cur_Msg).Warn_Chr := Warning_Msg_Char;
                end if;
             end if;
@@ -3927,12 +3924,15 @@ package body Errout is
       P : Natural;     -- Current index;
 
       procedure Skip_Msg_Insertion_Warning (C : Character);
-      --  Deal with ? ?? ?x? ?X? ?*? ?$? insertion sequences (and the same
+      --  Skip the ? ?? ?x? ?*? ?$? insertion sequences (and the same
       --  sequences using < instead of ?). The caller has already bumped
       --  the pointer past the initial ? or < and C is set to this initial
       --  character (? or <). This procedure skips past the rest of the
       --  sequence. We do not need to set Msg_Insertion_Char, since this
       --  was already done during the message prescan.
+      --  No validity check is performed as the insertion sequence is
+      --  supposed to be sane. See Prescan_Message.Parse_Message_Class in
+      --  erroutc.adb for the validity checks.
 
       --------------------------------
       -- Skip_Msg_Insertion_Warning --
@@ -3943,17 +3943,16 @@ package body Errout is
          if P <= Text'Last and then Text (P) = C then
             P := P + 1;
 
-         elsif P + 1 <= Text'Last
-           and then (Text (P) in 'a' .. 'z'
-                       or else
-                     Text (P) in 'A' .. 'Z'
-                       or else
-                     Text (P) = '*'
-                       or else
-                     Text (P) = '$')
-           and then Text (P + 1) = C
+         elsif P < Text'Last and then Text (P + 1) = C
+           and then Text (P) in 'a' .. 'z' | '*' | '$'
          then
             P := P + 2;
+
+         elsif P + 1 < Text'Last and then Text (P + 2) = C
+           and then Text (P) in '.' | '_'
+           and then Text (P + 1) in 'a' .. 'z'
+         then
+            P := P + 3;
          end if;
       end Skip_Msg_Insertion_Warning;
 
@@ -4404,19 +4403,15 @@ package body Errout is
 
    function Warn_Insertion return String is
    begin
-      case Warning_Msg_Char is
-         when '?' =>
-            return "??";
-
-         when 'a' .. 'z' | 'A' .. 'Z' | '*' | '$' =>
-            return '?' & Warning_Msg_Char & '?';
-
-         when ' ' =>
-            return "?";
-
-         when others =>
-            raise Program_Error;
-      end case;
+      if Warning_Msg_Char = "? " then
+         return "??";
+      elsif Warning_Msg_Char = "  " then
+         return "?";
+      elsif Warning_Msg_Char (2) = ' ' then
+         return '?' & Warning_Msg_Char (1) & '?';
+      else
+         return '?' & Warning_Msg_Char & '?';
+      end if;
    end Warn_Insertion;
 
 end Errout;
diff --git a/gcc/ada/errout.ads b/gcc/ada/errout.ads
index 9b2e08d..d6b8910 100644
--- a/gcc/ada/errout.ads
+++ b/gcc/ada/errout.ads
@@ -60,13 +60,13 @@ package Errout is
    --  Exception raised if Raise_Exception_On_Error is true
 
    Warning_Doc_Switch : Boolean renames Err_Vars.Warning_Doc_Switch;
-   --  If this is set True, then the ??/?*?/?$?/?x?/?X? insertion sequences in
-   --  error messages generate appropriate tags for the output error messages.
-   --  If this switch is False, then these sequences are still recognized (for
-   --  the purposes of implementing the pattern matching in pragmas Warnings
-   --  (Off,..) and Warning_As_Pragma(...) but do not result in adding the
-   --  error message tag. The -gnatw.d switch sets this flag True, -gnatw.D
-   --  sets this flag False.
+   --  If this is set True, then the ??/?*?/?$?/?x?/?.x?/?_x? insertion
+   --  sequences in error messages generate appropriate tags for the output
+   --  error messages. If this switch is False, then these sequences are still
+   --  recognized (for the purposes of implementing the pattern matching in
+   --  pragmas Warnings (Off,..) and Warning_As_Pragma(...) but do not result
+   --  in adding the error message tag. The -gnatw.d switch sets this flag
+   --  True, -gnatw.D sets this flag False.
 
    Current_Node : Node_Id := Empty;
    --  Used by Error_Msg as a default Node_Id.
@@ -275,7 +275,7 @@ package Errout is
    --      contain subprograms to be inlined in the main program. It is also
    --      used by the Compiler_Unit_Warning pragma for similar reasons.
 
-   --    Insertion character ? (Question: warning message)
+   --    Insertion character ? (Question: warning message -- OBSOLETE)
    --      The character ? appearing anywhere in a message makes the message
    --      warning instead of a normal error message, and the text of the
    --      message will be preceded by "warning:" in the normal case. The
@@ -302,28 +302,23 @@ package Errout is
    --      clear that the continuation is part of a warning message, but it is
    --      not necessary to go through any computational effort to include it.
    --
-   --      Note: this usage is obsolete, use ?? ?*? ?$? ?x? ?X? to specify
-   --      the string to be added when Warn_Doc_Switch is set to True. If this
-   --      switch is True, then for simple ? messages it has no effect. This
-   --      simple form is to ease transition and may be removed later except
-   --      for GNATprove-specific messages (info and warnings) which are not
-   --      subject to the same GNAT warning switches.
+   --      Note: this usage is obsolete; use ?? ?*? ?$? ?x? ?.x? ?_x? to
+   --      specify the string to be added when Warn_Doc_Switch is set to True.
+   --      If this switch is True, then for simple ? messages it has no effect.
+   --      This simple form is to ease transition and may be removed later
+   --      except for GNATprove-specific messages (info and warnings) which are
+   --      not subject to the same GNAT warning switches.
 
    --    Insertion character ?? (Two question marks: default warning)
    --      Like ?, but if the flag Warn_Doc_Switch is True, adds the string
    --      "[enabled by default]" at the end of the warning message. For
    --      continuations, use this in each continuation message.
 
-   --    Insertion character ?x? (warning with switch)
+   --    Insertion character ?x? ?.x? ?_x? (warning with switch)
    --      Like ?, but if the flag Warn_Doc_Switch is True, adds the string
-   --      "[-gnatwx]" at the end of the warning message. x is a lower case
-   --      letter. For continuations, use this on each continuation message.
-
-   --    Insertion character ?X? (warning with dot switch)
-   --      Like ?, but if the flag Warn_Doc_Switch is True, adds the string
-   --      "[-gnatw.x]" at the end of the warning message. X is an upper case
-   --      letter corresponding to the lower case letter x in the message.
-   --      For continuations, use this on each continuation message.
+   --      "[-gnatwx]", "[-gnatw.x]", or "[-gnatw_x]", at the end of the
+   --      warning message. x must be lower case. For continuations, use this
+   --      on each continuation message.
 
    --    Insertion character ?*? (restriction warning)
    --      Like ?, but if the flag Warn_Doc_Switch is True, adds the string
@@ -339,8 +334,8 @@ package Errout is
    --    Insertion character < (Less Than: conditional warning message)
    --      The character < appearing anywhere in a message is used for a
    --      conditional error message. If Error_Msg_Warn is True, then the
-   --      effect is the same as ? described above, and in particular << <X<
-   --      <x< <$< <*< have the effect of ?? ?X? ?x? ?$? ?*? respectively. If
+   --      effect is the same as ? described above, and in particular << <x<
+   --      <$< <*< have the effect of ?? ?x? ?$? ?*? respectively. If
    --      Error_Msg_Warn is False, then the < << or <X< sequence is ignored
    --      and the message is treated as a error rather than a warning.
 
diff --git a/gcc/ada/erroutc.adb b/gcc/ada/erroutc.adb
index 9e67b92..8225fd4 100644
--- a/gcc/ada/erroutc.adb
+++ b/gcc/ada/erroutc.adb
@@ -277,7 +277,9 @@ package body Erroutc is
    begin
       for J in 1 .. Errors.Last loop
          begin
-            if Errors.Table (J).Warn and Errors.Table (J).Compile_Time_Pragma
+            if Errors.Table (J).Warn
+               and then Errors.Table (J).Compile_Time_Pragma
+               and then not Errors.Table (J).Deleted
             then
                Result := Result + 1;
             end if;
@@ -362,20 +364,20 @@ package body Erroutc is
    ---------------------
 
    function Get_Warning_Tag (Id : Error_Msg_Id) return String is
-      Warn     : constant Boolean    := Errors.Table (Id).Warn;
-      Warn_Chr : constant Character  := Errors.Table (Id).Warn_Chr;
+      Warn     : constant Boolean         := Errors.Table (Id).Warn;
+      Warn_Chr : constant String (1 .. 2) := Errors.Table (Id).Warn_Chr;
    begin
-      if Warn and then Warn_Chr /= ' ' then
-         if Warn_Chr = '?' then
+      if Warn and then Warn_Chr /= "  " then
+         if Warn_Chr = "? " then
             return "[enabled by default]";
-         elsif Warn_Chr = '*' then
+         elsif Warn_Chr = "* " then
             return "[restriction warning]";
-         elsif Warn_Chr = '$' then
+         elsif Warn_Chr = "$ " then
             return "[-gnatel]";
-         elsif Warn_Chr in 'a' .. 'z' then
+         elsif Warn_Chr (2) = ' ' then
+            return "[-gnatw" & Warn_Chr (1) & ']';
+         else
             return "[-gnatw" & Warn_Chr & ']';
-         else pragma Assert (Warn_Chr in 'A' .. 'Z');
-            return "[-gnatw." & Fold_Lower (Warn_Chr) & ']';
          end if;
       else
          return "";
@@ -839,6 +841,51 @@ package body Erroutc is
    procedure Prescan_Message (Msg : String) is
       J : Natural;
 
+      function Parse_Message_Class return String;
+      --  Convert the warning insertion sequence to a warning class represented
+      --  as a length-two string padded, if necessary, with spaces.
+      --  Return the Message class and set the iterator J to the character
+      --  following the sequence.
+      --  Raise a Program_Error if the insertion sequence is not valid.
+
+      -------------------------
+      -- Parse_Message_Class --
+      -------------------------
+
+      function Parse_Message_Class return String is
+         C : constant Character := Msg (J - 1);
+         Message_Class : String (1 .. 2) := "  ";
+      begin
+         if J <= Msg'Last and then Msg (J) = C then
+            Message_Class := "? ";
+            J := J + 1;
+
+         elsif J < Msg'Last and then Msg (J + 1) = C
+           and then Msg (J) in 'a' .. 'z' | '*' | '$'
+         then
+            Message_Class := Msg (J) & " ";
+            J := J + 2;
+
+         elsif J + 1 < Msg'Last and then Msg (J + 2) = C
+           and then Msg (J) in '.' | '_'
+           and then Msg (J + 1) in 'a' .. 'z'
+         then
+            Message_Class := Msg (J .. J + 1);
+            J := J + 3;
+         elsif (J < Msg'Last and then Msg (J + 1) = C) or else
+            (J + 1 < Msg'Last and then Msg (J + 2) = C)
+         then
+            raise Program_Error;
+         end if;
+
+         --  In any other cases, this is not a warning insertion sequence
+         --  and the default "  " value is returned.
+
+         return Message_Class;
+      end Parse_Message_Class;
+
+   --  Start of processing for Prescan_Message
+
    begin
       --  Nothing to do for continuation line, unless -gnatdF is set
 
@@ -846,7 +893,7 @@ package body Erroutc is
          return;
 
       --  Some global variables are not set for continuation messages, as they
-      --  only make sense for the initial mesage.
+      --  only make sense for the initial message.
 
       elsif Msg (Msg'First) /= '\' then
 
@@ -898,29 +945,10 @@ package body Erroutc is
 
          elsif Msg (J) = '?' or else Msg (J) = '<' then
             Is_Warning_Msg := Msg (J) = '?' or else Error_Msg_Warn;
-            Warning_Msg_Char := ' ';
             J := J + 1;
 
             if Is_Warning_Msg then
-               declare
-                  C : constant Character := Msg (J - 1);
-               begin
-                  if J <= Msg'Last then
-                     if Msg (J) = C then
-                        Warning_Msg_Char := '?';
-                        J := J + 1;
-
-                     elsif J < Msg'Last and then Msg (J + 1) = C
-                       and then (Msg (J) in 'a' .. 'z' or else
-                                 Msg (J) in 'A' .. 'Z' or else
-                                 Msg (J) = '*'         or else
-                                 Msg (J) = '$')
-                     then
-                        Warning_Msg_Char := Msg (J);
-                        J := J + 2;
-                     end if;
-                  end if;
-               end;
+               Warning_Msg_Char := Parse_Message_Class;
             end if;
 
             --  Bomb if untagged warning message. This code can be uncommented
@@ -1685,7 +1713,7 @@ package body Erroutc is
 
                if SWE.Open then
                   Eproc.all
-                    ("?W?pragma Warnings Off with no matching Warnings On",
+                    ("?.w?pragma Warnings Off with no matching Warnings On",
                      SWE.Start);
 
                --  Warn for ineffective Warnings (Off, ..)
@@ -1700,7 +1728,7 @@ package body Erroutc is
                    (SWE.Msg'Length > 3 and then SWE.Msg (2 .. 3) = "-W")
                then
                   Eproc.all
-                    ("?W?no warning suppressed by this pragma", SWE.Start);
+                    ("?.w?no warning suppressed by this pragma", SWE.Start);
                end if;
             end if;
          end;
diff --git a/gcc/ada/erroutc.ads b/gcc/ada/erroutc.ads
index 891391c..0c194e8 100644
--- a/gcc/ada/erroutc.ads
+++ b/gcc/ada/erroutc.ads
@@ -80,14 +80,14 @@ package Erroutc is
    --  Set True to indicate that the current message starts with one of
    --  "high: ", "medium: ", "low: " and is to be treated as a check message.
 
-   Warning_Msg_Char : Character;
-   --  Warning character, valid only if Is_Warning_Msg is True
-   --    ' '      -- ?   or <   appeared on its own in message
-   --    '?'      -- ??  or <<  appeared in message
-   --    'x'      -- ?x? or <x< appeared in message (x = a .. z)
-   --    'X'      -- ?X? or <X< appeared in message (X = A .. Z)
-   --    '*'      -- ?*? or <*< appeared in message
-   --    '$'      -- ?$? or <$< appeared in message
+   Warning_Msg_Char : String (1 .. 2);
+   --  Warning switch, valid only if Is_Warning_Msg is True
+   --    "  "      -- ?   or <   appeared on its own in message
+   --    "? "      -- ??  or <<  appeared in message
+   --    "x "      -- ?x? or <x< appeared in message
+   --              -- (x = a .. z | A .. Z | * | $)
+   --    ".x"      -- ?.x? appeared in message (x = a .. z | A .. Z)
+   --    "_x"      -- ?_x? appeared in message (x = a .. z | A .. Z)
    --  In the case of the < sequences, this is set only if the message is
    --  actually a warning, i.e. if Error_Msg_Warn is True
 
@@ -239,16 +239,8 @@ package Erroutc is
       --  True if this is a warning message which is to be treated as an error
       --  as a result of a match with a Warning_As_Error pragma.
 
-      Warn_Chr : Character;
-      --  Warning character (note: set even if Warning_Doc_Switch is False)
-      --    ' '      -- ?   or <   appeared on its own in message
-      --    '?'      -- ??  or <<  appeared in message
-      --    'x'      -- ?x? or <x< appeared in message (x = a .. z)
-      --    'X'      -- ?X? or <X< appeared in message (X = A .. Z)
-      --    '*'      -- ?*? or <*< appeared in message
-      --    '$'      -- ?$? or <$< appeared in message
-      --  In the case of the < sequences, this is set only if the message is
-      --  actually a warning, i.e. if Error_Msg_Warn is True
+      Warn_Chr : String (1 .. 2);
+      --  See Warning_Msg_Char
 
       Style : Boolean;
       --  True if style message (starts with "(style)")
diff --git a/gcc/ada/exp_attr.adb b/gcc/ada/exp_attr.adb
index 49f5c94..33ac8bd 100644
--- a/gcc/ada/exp_attr.adb
+++ b/gcc/ada/exp_attr.adb
@@ -67,6 +67,7 @@ with Sinfo.Utils;    use Sinfo.Utils;
 with Snames;         use Snames;
 with Stand;          use Stand;
 with Stringt;        use Stringt;
+with Strub;          use Strub;
 with Tbuild;         use Tbuild;
 with Ttypes;         use Ttypes;
 with Uintp;          use Uintp;
@@ -2162,6 +2163,7 @@ package body Exp_Attr is
 
                   begin
                      Subp_Typ := Create_Itype (E_Subprogram_Type, N);
+                     Copy_Strub_Mode (Subp_Typ, Subp);
                      Set_Etype (Subp_Typ, Etype (Subp));
                      Set_Returns_By_Ref (Subp_Typ, Returns_By_Ref (Subp));
 
@@ -3898,8 +3900,8 @@ package body Exp_Attr is
          if Ptyp = Standard_Exception_Type then
             Id_Kind := RTE (RE_Exception_Id);
 
-            if Present (Renamed_Object (Entity (Pref))) then
-               Set_Entity (Pref, Renamed_Object (Entity (Pref)));
+            if Present (Renamed_Entity (Entity (Pref))) then
+               Set_Entity (Pref, Renamed_Entity (Entity (Pref)));
             end if;
 
             Rewrite (N,
diff --git a/gcc/ada/exp_ch11.adb b/gcc/ada/exp_ch11.adb
index 40288e4..b8a9a8d 100644
--- a/gcc/ada/exp_ch11.adb
+++ b/gcc/ada/exp_ch11.adb
@@ -1667,8 +1667,8 @@ package body Exp_Ch11 is
             --  If the exception is a renaming, use the exception that it
             --  renames (which might be a predefined exception, e.g.).
 
-            if Present (Renamed_Object (Id)) then
-               Id := Renamed_Object (Id);
+            if Present (Renamed_Entity (Id)) then
+               Id := Renamed_Entity (Id);
             end if;
 
             --  Build a C-compatible string in case of no exception handlers,
@@ -1861,10 +1861,10 @@ package body Exp_Ch11 is
 
             if Configurable_Run_Time_Mode then
                Error_Msg_NE
-                 ("\?X?& may call Last_Chance_Handler", N, E);
+                 ("\?.x?& may call Last_Chance_Handler", N, E);
             else
                Error_Msg_NE
-                 ("\?X?& may result in unhandled exception", N, E);
+                 ("\?.x?& may result in unhandled exception", N, E);
             end if;
          end if;
       end;
@@ -2163,7 +2163,7 @@ package body Exp_Ch11 is
          Warn_No_Exception_Propagation_Active (N);
 
          Error_Msg_N
-           ("\?X?this handler can never be entered, and has been removed", N);
+           ("\?.x?this handler can never be entered, and has been removed", N);
       end if;
    end Warn_If_No_Local_Raise;
 
@@ -2180,10 +2180,10 @@ package body Exp_Ch11 is
 
          if Configurable_Run_Time_Mode then
             Error_Msg_N
-              ("\?X?Last_Chance_Handler will be called on exception", N);
+              ("\?.x?Last_Chance_Handler will be called on exception", N);
          else
             Error_Msg_N
-              ("\?X?execution may raise unhandled exception", N);
+              ("\?.x?execution may raise unhandled exception", N);
          end if;
       end if;
    end Warn_If_No_Propagation;
@@ -2195,7 +2195,7 @@ package body Exp_Ch11 is
    procedure Warn_No_Exception_Propagation_Active (N : Node_Id) is
    begin
       Error_Msg_N
-        ("?X?pragma Restrictions (No_Exception_Propagation) in effect", N);
+        ("?.x?pragma Restrictions (No_Exception_Propagation) in effect", N);
    end Warn_No_Exception_Propagation_Active;
 
 end Exp_Ch11;
diff --git a/gcc/ada/exp_ch3.adb b/gcc/ada/exp_ch3.adb
index e7eed28..c91008f 100644
--- a/gcc/ada/exp_ch3.adb
+++ b/gcc/ada/exp_ch3.adb
@@ -9864,10 +9864,9 @@ package body Exp_Ch3 is
              Expression =>
                Expand_Record_Equality
                  (Typ,
-                  Typ    => Typ,
-                  Lhs    => Make_Identifier (Loc, Name_X),
-                  Rhs    => Make_Identifier (Loc, Name_Y),
-                  Bodies => Declarations (Decl))));
+                  Typ => Typ,
+                  Lhs => Make_Identifier (Loc, Name_X),
+                  Rhs => Make_Identifier (Loc, Name_Y))));
       end if;
 
       Set_Handled_Statement_Sequence
@@ -10972,16 +10971,13 @@ package body Exp_Ch3 is
          while Present (Prim) loop
             if Chars (Node (Prim)) = Name_Op_Eq
               and then not Is_Internal (Node (Prim))
-              and then Present (First_Entity (Node (Prim)))
 
               --  The predefined equality primitive must have exactly two
-              --  formals whose type is this tagged type
+              --  formals whose type is this tagged type.
 
-              and then Present (Last_Entity (Node (Prim)))
-              and then Next_Entity (First_Entity (Node (Prim)))
-                         = Last_Entity (Node (Prim))
-              and then Etype (First_Entity (Node (Prim))) = Tag_Typ
-              and then Etype (Last_Entity (Node (Prim))) = Tag_Typ
+              and then Number_Formals (Node (Prim)) = 2
+              and then Etype (First_Formal (Node (Prim))) = Tag_Typ
+              and then Etype (Last_Formal (Node (Prim))) = Tag_Typ
             then
                Eq_Needed := False;
                Eq_Name := No_Name;
@@ -11102,7 +11098,6 @@ package body Exp_Ch3 is
       --  they may be ancestors of synchronized interface types).
 
       if Ada_Version >= Ada_2005
-        and then not Is_Interface (Tag_Typ)
         and then
           ((Is_Interface (Etype (Tag_Typ))
              and then Is_Limited_Record (Etype (Tag_Typ)))
@@ -11123,7 +11118,7 @@ package body Exp_Ch3 is
          Append_To (Res, Make_Disp_Timed_Select_Body        (Tag_Typ));
       end if;
 
-      if not Is_Limited_Type (Tag_Typ) and then not Is_Interface (Tag_Typ) then
+      if not Is_Limited_Type (Tag_Typ) then
 
          --  Body for equality
 
diff --git a/gcc/ada/exp_ch4.adb b/gcc/ada/exp_ch4.adb
index 8dcfa85..05124dc 100644
--- a/gcc/ada/exp_ch4.adb
+++ b/gcc/ada/exp_ch4.adb
@@ -146,18 +146,14 @@ package body Exp_Ch4 is
    --  where we allow comparison of "out of range" values.
 
    function Expand_Composite_Equality
-     (Nod    : Node_Id;
-      Typ    : Entity_Id;
-      Lhs    : Node_Id;
-      Rhs    : Node_Id;
-      Bodies : List_Id) return Node_Id;
+     (Nod : Node_Id;
+      Typ : Entity_Id;
+      Lhs : Node_Id;
+      Rhs : Node_Id) return Node_Id;
    --  Local recursive function used to expand equality for nested composite
-   --  types. Used by Expand_Record/Array_Equality, Bodies is a list on which
-   --  to attach bodies of local functions that are created in the process. It
-   --  is the responsibility of the caller to insert those bodies at the right
-   --  place. Nod provides the Sloc value for generated code. Lhs and Rhs are
-   --  the left and right sides for the comparison, and Typ is the type of the
-   --  objects to compare.
+   --  types. Used by Expand_Record/Array_Equality. Nod provides the Sloc value
+   --  for generated code. Lhs and Rhs are the left and right sides for the
+   --  comparison, and Typ is the type of the objects to compare.
 
    procedure Expand_Concatenate (Cnode : Node_Id; Opnds : List_Id);
    --  Routine to expand concatenation of a sequence of two or more operands
@@ -1545,14 +1541,14 @@ package body Exp_Ch4 is
    --          and then
    --        (B'length (1) = 0 or else B'length (2) = 0)
    --     then
-   --        return True;    -- RM 4.5.2(22)
+   --        return true;    -- RM 4.5.2(22)
    --     end if;
 
    --     if A'length (1) /= B'length (1)
    --               or else
    --           A'length (2) /= B'length (2)
    --     then
-   --        return False;   -- RM 4.5.2(23)
+   --        return false;   -- RM 4.5.2(23)
    --     end if;
 
    --     declare
@@ -1625,8 +1621,8 @@ package body Exp_Ch4 is
       function Arr_Attr
         (Arr : Entity_Id;
          Nam : Name_Id;
-         Num : Int) return Node_Id;
-      --  This builds the attribute reference Arr'Nam (Expr)
+         Dim : Pos) return Node_Id;
+      --  This builds the attribute reference Arr'Nam (Dim)
 
       function Component_Equality (Typ : Entity_Id) return Node_Id;
       --  Create one statement to compare corresponding components, designated
@@ -1637,11 +1633,12 @@ package body Exp_Ch4 is
       --  for that argument in the corresponding function formal
 
       function Handle_One_Dimension
-        (N     : Int;
+        (N     : Pos;
          Index : Node_Id) return Node_Id;
       --  This procedure returns the following code
       --
       --    declare
+      --       An : Index_T := A'First (N);
       --       Bn : Index_T := B'First (N);
       --    begin
       --       loop
@@ -1691,14 +1688,14 @@ package body Exp_Ch4 is
       function Arr_Attr
         (Arr : Entity_Id;
          Nam : Name_Id;
-         Num : Int) return Node_Id
+         Dim : Pos) return Node_Id
       is
       begin
          return
            Make_Attribute_Reference (Loc,
              Attribute_Name => Nam,
              Prefix         => New_Occurrence_Of (Arr, Loc),
-             Expressions    => New_List (Make_Integer_Literal (Loc, Num)));
+             Expressions    => New_List (Make_Integer_Literal (Loc, Dim)));
       end Arr_Attr;
 
       ------------------------
@@ -1722,8 +1719,7 @@ package body Exp_Ch4 is
              Prefix      => Make_Identifier (Loc, Chars (B)),
              Expressions => Index_List2);
 
-         Test := Expand_Composite_Equality
-                   (Nod, Component_Type (Typ), L, R, Decls);
+         Test := Expand_Composite_Equality (Nod, Component_Type (Typ), L, R);
 
          --  If some (sub)component is an unchecked_union, the whole operation
          --  will raise program error.
@@ -1786,7 +1782,7 @@ package body Exp_Ch4 is
       ---------------------------
 
       function Handle_One_Dimension
-        (N     : Int;
+        (N     : Pos;
          Index : Node_Id) return Node_Id
       is
          Need_Separate_Indexes : constant Boolean :=
@@ -1902,41 +1898,20 @@ package body Exp_Ch4 is
       -----------------------
 
       function Test_Empty_Arrays return Node_Id is
-         Alist : Node_Id;
-         Blist : Node_Id;
-
-         Atest : Node_Id;
-         Btest : Node_Id;
+         Alist : Node_Id := Empty;
+         Blist : Node_Id := Empty;
 
       begin
-         Alist := Empty;
-         Blist := Empty;
          for J in 1 .. Number_Dimensions (Ltyp) loop
-            Atest :=
+            Evolve_Or_Else (Alist,
               Make_Op_Eq (Loc,
                 Left_Opnd  => Arr_Attr (A, Name_Length, J),
-                Right_Opnd => Make_Integer_Literal (Loc, 0));
+                Right_Opnd => Make_Integer_Literal (Loc, Uint_0)));
 
-            Btest :=
+            Evolve_Or_Else (Blist,
               Make_Op_Eq (Loc,
                 Left_Opnd  => Arr_Attr (B, Name_Length, J),
-                Right_Opnd => Make_Integer_Literal (Loc, 0));
-
-            if No (Alist) then
-               Alist := Atest;
-               Blist := Btest;
-
-            else
-               Alist :=
-                 Make_Or_Else (Loc,
-                   Left_Opnd  => Relocate_Node (Alist),
-                   Right_Opnd => Atest);
-
-               Blist :=
-                 Make_Or_Else (Loc,
-                   Left_Opnd  => Relocate_Node (Blist),
-                   Right_Opnd => Btest);
-            end if;
+                Right_Opnd => Make_Integer_Literal (Loc, Uint_0)));
          end loop;
 
          return
@@ -1950,25 +1925,14 @@ package body Exp_Ch4 is
       -----------------------------
 
       function Test_Lengths_Correspond return Node_Id is
-         Result : Node_Id;
-         Rtest  : Node_Id;
+         Result : Node_Id := Empty;
 
       begin
-         Result := Empty;
          for J in 1 .. Number_Dimensions (Ltyp) loop
-            Rtest :=
+            Evolve_Or_Else (Result,
               Make_Op_Ne (Loc,
                 Left_Opnd  => Arr_Attr (A, Name_Length, J),
-                Right_Opnd => Arr_Attr (B, Name_Length, J));
-
-            if No (Result) then
-               Result := Rtest;
-            else
-               Result :=
-                 Make_Or_Else (Loc,
-                   Left_Opnd  => Relocate_Node (Result),
-                   Right_Opnd => Rtest);
-            end if;
+                Right_Opnd => Arr_Attr (B, Name_Length, J)));
          end loop;
 
          return Result;
@@ -2020,14 +1984,16 @@ package body Exp_Ch4 is
         and then Ltyp = Rtyp
         and then Is_Constrained (Ltyp)
         and then Number_Dimensions (Ltyp) = 1
-        and then Nkind (First_Idx) = N_Range
-        and then Compile_Time_Known_Value (Low_Bound (First_Idx))
-        and then Compile_Time_Known_Value (High_Bound (First_Idx))
-        and then Expr_Value (High_Bound (First_Idx)) =
-                                         Expr_Value (Low_Bound (First_Idx)) + 1
+        and then Compile_Time_Known_Bounds (Ltyp)
+        and then Expr_Value (Type_High_Bound (Etype (First_Idx))) =
+                   Expr_Value (Type_Low_Bound (Etype (First_Idx))) + 1
       then
          declare
             Ctyp         : constant Entity_Id := Component_Type (Ltyp);
+            Low_B        : constant Node_Id :=
+              Type_Low_Bound (Etype (First_Idx));
+            High_B       : constant Node_Id :=
+              Type_High_Bound (Etype (First_Idx));
             L, R         : Node_Id;
             TestL, TestH : Node_Id;
 
@@ -2035,30 +2001,26 @@ package body Exp_Ch4 is
             L :=
               Make_Indexed_Component (Loc,
                 Prefix      => New_Copy_Tree (New_Lhs),
-                Expressions =>
-                  New_List (New_Copy_Tree (Low_Bound (First_Idx))));
+                Expressions => New_List (New_Copy_Tree (Low_B)));
 
             R :=
               Make_Indexed_Component (Loc,
                 Prefix      => New_Copy_Tree (New_Rhs),
-                Expressions =>
-                  New_List (New_Copy_Tree (Low_Bound (First_Idx))));
+                Expressions => New_List (New_Copy_Tree (Low_B)));
 
-            TestL := Expand_Composite_Equality (Nod, Ctyp, L, R, Bodies);
+            TestL := Expand_Composite_Equality (Nod, Ctyp, L, R);
 
             L :=
               Make_Indexed_Component (Loc,
                 Prefix      => New_Lhs,
-                Expressions =>
-                  New_List (New_Copy_Tree (High_Bound (First_Idx))));
+                Expressions => New_List (New_Copy_Tree (High_B)));
 
             R :=
               Make_Indexed_Component (Loc,
                 Prefix      => New_Rhs,
-                Expressions =>
-                  New_List (New_Copy_Tree (High_Bound (First_Idx))));
+                Expressions => New_List (New_Copy_Tree (High_B)));
 
-            TestH := Expand_Composite_Equality (Nod, Ctyp, L, R, Bodies);
+            TestH := Expand_Composite_Equality (Nod, Ctyp, L, R);
 
             return
               Make_And_Then (Loc, Left_Opnd => TestL, Right_Opnd => TestH);
@@ -2471,18 +2433,15 @@ package body Exp_Ch4 is
    --  case because it is not possible to respect normal Ada visibility rules.
 
    function Expand_Composite_Equality
-     (Nod    : Node_Id;
-      Typ    : Entity_Id;
-      Lhs    : Node_Id;
-      Rhs    : Node_Id;
-      Bodies : List_Id) return Node_Id
+     (Nod : Node_Id;
+      Typ : Entity_Id;
+      Lhs : Node_Id;
+      Rhs : Node_Id) return Node_Id
    is
       Loc       : constant Source_Ptr := Sloc (Nod);
       Full_Type : Entity_Id;
       Eq_Op     : Entity_Id;
 
-   --  Start of processing for Expand_Composite_Equality
-
    begin
       if Is_Private_Type (Typ) then
          Full_Type := Underlying_Type (Typ);
@@ -2509,81 +2468,15 @@ package body Exp_Ch4 is
          Full_Type := Underlying_Type (Full_Type);
       end if;
 
-      --  Case of array types
-
-      if Is_Array_Type (Full_Type) then
-
-         --  If the operand is an elementary type other than a floating-point
-         --  type, then we can simply use the built-in block bitwise equality,
-         --  since the predefined equality operators always apply and bitwise
-         --  equality is fine for all these cases.
-
-         if Is_Elementary_Type (Component_Type (Full_Type))
-           and then not Is_Floating_Point_Type (Component_Type (Full_Type))
-         then
-            return Make_Op_Eq (Loc, Left_Opnd => Lhs, Right_Opnd => Rhs);
-
-         --  For composite component types, and floating-point types, use the
-         --  expansion. This deals with tagged component types (where we use
-         --  the applicable equality routine) and floating-point (where we
-         --  need to worry about negative zeroes), and also the case of any
-         --  composite type recursively containing such fields.
-
-         else
-            declare
-               Comp_Typ : Entity_Id;
-               Hi       : Node_Id;
-               Indx     : Node_Id;
-               Ityp     : Entity_Id;
-               Lo       : Node_Id;
-
-            begin
-               --  Do the comparison in the type (or its full view) and not in
-               --  its unconstrained base type, because the latter operation is
-               --  more complex and would also require an unchecked conversion.
-
-               if Is_Private_Type (Typ) then
-                  Comp_Typ := Underlying_Type (Typ);
-               else
-                  Comp_Typ := Typ;
-               end if;
-
-               --  Except for the case where the bounds of the type depend on a
-               --  discriminant, or else we would run into scoping issues.
-
-               Indx := First_Index (Comp_Typ);
-               while Present (Indx) loop
-                  Ityp := Etype (Indx);
-
-                  Lo := Type_Low_Bound (Ityp);
-                  Hi := Type_High_Bound (Ityp);
-
-                  if (Nkind (Lo) = N_Identifier
-                       and then Ekind (Entity (Lo)) = E_Discriminant)
-                    or else
-                     (Nkind (Hi) = N_Identifier
-                       and then Ekind (Entity (Hi)) = E_Discriminant)
-                  then
-                     Comp_Typ := Full_Type;
-                     exit;
-                  end if;
-
-                  Next_Index (Indx);
-               end loop;
-
-               return Expand_Array_Equality (Nod, Lhs, Rhs, Bodies, Comp_Typ);
-            end;
-         end if;
-
       --  Case of tagged record types
 
-      elsif Is_Tagged_Type (Full_Type) then
+      if Is_Tagged_Type (Full_Type) then
          Eq_Op := Find_Primitive_Eq (Typ);
          pragma Assert (Present (Eq_Op));
 
          return
            Make_Function_Call (Loc,
-             Name => New_Occurrence_Of (Eq_Op, Loc),
+             Name                   => New_Occurrence_Of (Eq_Op, Loc),
              Parameter_Associations =>
                New_List
                  (Unchecked_Convert_To (Etype (First_Formal (Eq_Op)), Lhs),
@@ -2606,7 +2499,7 @@ package body Exp_Ch4 is
                begin
                   return
                     Make_Function_Call (Loc,
-                      Name                  => New_Occurrence_Of (Eq_Op, Loc),
+                      Name                   => New_Occurrence_Of (Eq_Op, Loc),
                       Parameter_Associations => New_List (
                         OK_Convert_To (T, Lhs),
                         OK_Convert_To (T, Rhs)));
@@ -2765,10 +2658,10 @@ package body Exp_Ch4 is
             end;
 
          else
-            return Expand_Record_Equality (Nod, Full_Type, Lhs, Rhs, Bodies);
+            return Expand_Record_Equality (Nod, Full_Type, Lhs, Rhs);
          end if;
 
-      --  Non-composite types (always use predefined equality)
+      --  Case of non-record types (always use predefined equality)
 
       else
          return Make_Op_Eq (Loc, Left_Opnd => Lhs, Right_Opnd => Rhs);
@@ -4592,7 +4485,7 @@ package body Exp_Ch4 is
                        and then Nkind (Associated_Node_For_Itype (PtrT)) =
                                   N_Object_Declaration)
       then
-         Error_Msg_N ("??use of an anonymous access type allocator", N);
+         Error_Msg_N ("?_a?use of an anonymous access type allocator", N);
       end if;
 
       --  RM E.2.2(17). We enforce that the expected type of an allocator
@@ -8740,10 +8633,8 @@ package body Exp_Ch4 is
          else
             Remove_Side_Effects (Lhs);
             Remove_Side_Effects (Rhs);
-            Rewrite (N,
-              Expand_Record_Equality (N, Typl, Lhs, Rhs, Bodies));
+            Rewrite (N, Expand_Record_Equality (N, Typl, Lhs, Rhs));
 
-            Insert_Actions      (N, Bodies,           Suppress => All_Checks);
             Analyze_And_Resolve (N, Standard_Boolean, Suppress => All_Checks);
          end if;
 
@@ -8766,10 +8657,8 @@ package body Exp_Ch4 is
          Rewrite (N,
            Expand_Record_Equality (N, Typl,
              Unchecked_Convert_To (Typl, Lhs),
-             Unchecked_Convert_To (Typl, Rhs),
-             Bodies));
+             Unchecked_Convert_To (Typl, Rhs)));
 
-         Insert_Actions      (N, Bodies,           Suppress => All_Checks);
          Analyze_And_Resolve (N, Standard_Boolean, Suppress => All_Checks);
       end if;
 
@@ -13094,11 +12983,10 @@ package body Exp_Ch4 is
    --  otherwise the primitive "=" is used directly.
 
    function Expand_Record_Equality
-     (Nod    : Node_Id;
-      Typ    : Entity_Id;
-      Lhs    : Node_Id;
-      Rhs    : Node_Id;
-      Bodies : List_Id) return Node_Id
+     (Nod : Node_Id;
+      Typ : Entity_Id;
+      Lhs : Node_Id;
+      Rhs : Node_Id) return Node_Id
    is
       Loc : constant Source_Ptr := Sloc (Nod);
 
@@ -13116,41 +13004,35 @@ package body Exp_Ch4 is
       ------------------------
 
       function Element_To_Compare (C : Entity_Id) return Entity_Id is
-         Comp : Entity_Id;
+         Comp : Entity_Id := C;
 
       begin
-         Comp := C;
-         loop
-            --  Exit loop when the next element to be compared is found, or
-            --  there is no more such element.
-
-            exit when No (Comp);
-
-            exit when Ekind (Comp) in E_Discriminant | E_Component
-              and then not (
+         while Present (Comp) loop
+            --  Skip inherited components
 
-              --  Skip inherited components
+            --  Note: for a tagged type, we always generate the "=" primitive
+            --  for the base type (not on the first subtype), so the test for
+            --  Comp /= Original_Record_Component (Comp) is True for inherited
+            --  components only.
 
-              --  Note: for a tagged type, we always generate the "=" primitive
-              --  for the base type (not on the first subtype), so the test for
-              --  Comp /= Original_Record_Component (Comp) is True for
-              --  inherited components only.
-
-              (Is_Tagged_Type (Typ)
+            if (Is_Tagged_Type (Typ)
                 and then Comp /= Original_Record_Component (Comp))
 
-              --  Skip _Tag
+            --  Skip _Tag
 
               or else Chars (Comp) = Name_uTag
 
-              --  Skip interface elements (secondary tags???)
-
-              or else Is_Interface (Etype (Comp)));
+            --  Skip interface elements (secondary tags???)
 
-            Next_Entity (Comp);
+              or else Is_Interface (Etype (Comp))
+            then
+               Next_Component_Or_Discriminant (Comp);
+            else
+               return Comp;
+            end if;
          end loop;
 
-         return Comp;
+         return Empty;
       end Element_To_Compare;
 
    --  Start of processing for Expand_Record_Equality
@@ -13166,7 +13048,7 @@ package body Exp_Ch4 is
       --    and then Lhs.Cmpn = Rhs.Cmpn
 
       Result := New_Occurrence_Of (Standard_True, Loc);
-      C := Element_To_Compare (First_Entity (Typ));
+      C := Element_To_Compare (First_Component_Or_Discriminant (Typ));
       while Present (C) loop
          declare
             New_Lhs : Node_Id;
@@ -13191,8 +13073,7 @@ package body Exp_Ch4 is
                Rhs =>
                  Make_Selected_Component (Loc,
                    Prefix        => New_Rhs,
-                   Selector_Name => New_Occurrence_Of (C, Loc)),
-               Bodies => Bodies);
+                   Selector_Name => New_Occurrence_Of (C, Loc)));
 
             --  If some (sub)component is an unchecked_union, the whole
             --  operation will raise program error.
@@ -13224,7 +13105,7 @@ package body Exp_Ch4 is
          end;
 
          First_Time := False;
-         C := Element_To_Compare (Next_Entity (C));
+         C := Element_To_Compare (Next_Component_Or_Discriminant (C));
       end loop;
 
       return Result;
@@ -15384,6 +15265,17 @@ package body Exp_Ch4 is
    begin
       SCIL_Node := Empty;
 
+      --  We have to examine the corresponding record type when dealing with
+      --  protected types instead of the original, unexpanded, type.
+
+      if Ekind (Right_Type) = E_Protected_Type then
+         Right_Type := Corresponding_Record_Type (Right_Type);
+      end if;
+
+      if Ekind (Left_Type) = E_Protected_Type then
+         Left_Type := Corresponding_Record_Type (Left_Type);
+      end if;
+
       --  In the case where the type is an access type, the test is applied
       --  using the designated types (needed in Ada 2012 for implicit anonymous
       --  access conversions, for AI05-0149).
diff --git a/gcc/ada/exp_ch4.ads b/gcc/ada/exp_ch4.ads
index b1df4a0..b723adc 100644
--- a/gcc/ada/exp_ch4.ads
+++ b/gcc/ada/exp_ch4.ads
@@ -89,20 +89,16 @@ package Exp_Ch4 is
    --  while for records without variants only a simple expression is needed.
 
    function Expand_Record_Equality
-     (Nod    : Node_Id;
-      Typ    : Entity_Id;
-      Lhs    : Node_Id;
-      Rhs    : Node_Id;
-      Bodies : List_Id) return Node_Id;
+     (Nod : Node_Id;
+      Typ : Entity_Id;
+      Lhs : Node_Id;
+      Rhs : Node_Id) return Node_Id;
    --  Expand a record equality into an expression that compares the fields
    --  individually to yield the required Boolean result. Loc is the
    --  location for the generated nodes. Typ is the type of the record, and
    --  Lhs, Rhs are the record expressions to be compared, these
    --  expressions need not to be analyzed but have to be side-effect free.
-   --  Bodies is a list on which to attach bodies of local functions that
-   --  are created in the process. This is the responsibility of the caller
-   --  to insert those bodies at the right place. Nod provides the Sloc
-   --  value for generated code.
+   --  Nod provides the Sloc value for generated code.
 
    procedure Expand_Set_Membership (N : Node_Id);
    --  For each choice of a set membership, we create a simple equality or
diff --git a/gcc/ada/exp_ch5.adb b/gcc/ada/exp_ch5.adb
index 21ac2a2..47c6b80 100644
--- a/gcc/ada/exp_ch5.adb
+++ b/gcc/ada/exp_ch5.adb
@@ -2387,6 +2387,7 @@ package body Exp_Ch5 is
             Ent := Name (N);
             while Nkind (Ent) in N_Has_Entity
               and then Present (Entity (Ent))
+              and then Is_Object (Entity (Ent))
               and then Present (Renamed_Object (Entity (Ent)))
             loop
                Ent := Renamed_Object (Entity (Ent));
diff --git a/gcc/ada/exp_ch6.adb b/gcc/ada/exp_ch6.adb
index 3f83685..fa1a89a 100644
--- a/gcc/ada/exp_ch6.adb
+++ b/gcc/ada/exp_ch6.adb
@@ -2292,8 +2292,11 @@ package body Exp_Ch6 is
                null;
 
             elsif Is_Build_In_Place_Function_Call (Actual) then
-               Build_Activation_Chain_Entity (N);
-               Build_Master_Entity (Etype (Actual));
+               if Might_Have_Tasks (Etype (Actual)) then
+                  Build_Activation_Chain_Entity (N);
+                  Build_Master_Entity (Etype (Actual));
+               end if;
+
                Make_Build_In_Place_Call_In_Anonymous_Context (Actual);
 
             --  Ada 2005 (AI-318-02): Specialization of the previous case for
@@ -9707,7 +9710,9 @@ package body Exp_Ch6 is
          --  At this point, Defining_Identifier (Obj_Decl) is no longer equal
          --  to Obj_Def_Id.
 
-         Set_Renamed_Object (Defining_Identifier (Obj_Decl), Call_Deref);
+         pragma Assert (Ekind (Defining_Identifier (Obj_Decl)) = E_Void);
+         Set_Renamed_Object_Of_Possibly_Void
+           (Defining_Identifier (Obj_Decl), Call_Deref);
 
          --  If the original entity comes from source, then mark the new
          --  entity as needing debug information, even though it's defined
diff --git a/gcc/ada/exp_ch8.adb b/gcc/ada/exp_ch8.adb
index dba21ed..ff3c0eb 100644
--- a/gcc/ada/exp_ch8.adb
+++ b/gcc/ada/exp_ch8.adb
@@ -295,11 +295,11 @@ package body Exp_Ch8 is
          Set_Alias (Id, Empty);
          Set_Has_Completion (Id, False);
          Rewrite (N,
-           Make_Subprogram_Declaration (Sloc (N),
+           Make_Subprogram_Declaration (Loc,
              Specification => Specification (N)));
          Set_Has_Delayed_Freeze (Id);
 
-         Body_Id := Make_Defining_Identifier (Sloc (N), Chars (Id));
+         Body_Id := Make_Defining_Identifier (Loc, Chars (Id));
          Set_Debug_Info_Needed (Body_Id);
 
          if Has_Variant_Part (Typ) then
@@ -326,19 +326,16 @@ package body Exp_Ch8 is
                     Result_Definition        =>
                       New_Occurrence_Of (Standard_Boolean, Loc)),
                 Declarations               => Empty_List,
-                Handled_Statement_Sequence => Empty);
-
-            Set_Handled_Statement_Sequence (Decl,
-              Make_Handled_Sequence_Of_Statements (Loc,
-                Statements => New_List (
-                  Make_Simple_Return_Statement (Loc,
-                    Expression =>
-                      Expand_Record_Equality
-                        (Id,
-                         Typ    => Typ,
-                         Lhs    => Make_Identifier (Loc, Chars (Left)),
-                         Rhs    => Make_Identifier (Loc, Chars (Right)),
-                         Bodies => Declarations (Decl))))));
+                Handled_Statement_Sequence =>
+                  Make_Handled_Sequence_Of_Statements (Loc,
+                    Statements => New_List (
+                      Make_Simple_Return_Statement (Loc,
+                        Expression =>
+                          Expand_Record_Equality
+                            (Id,
+                             Typ => Typ,
+                             Lhs => Make_Identifier (Loc, Chars (Left)),
+                             Rhs => Make_Identifier (Loc, Chars (Right)))))));
          end if;
 
          return Decl;
@@ -375,7 +372,7 @@ package body Exp_Ch8 is
         and then Scope (Entity (Nam)) = Standard_Standard
       then
          declare
-            Typ  : constant Entity_Id := Etype (First_Formal (Id));
+            Typ : constant Entity_Id := Etype (First_Formal (Id));
 
          begin
             --  Check whether this is a renaming of a predefined equality on an
diff --git a/gcc/ada/exp_dbug.adb b/gcc/ada/exp_dbug.adb
index 96d78cc..19d546e 100644
--- a/gcc/ada/exp_dbug.adb
+++ b/gcc/ada/exp_dbug.adb
@@ -290,6 +290,11 @@ package body Exp_Dbug is
    --------------------------------
 
    function Debug_Renaming_Declaration (N : Node_Id) return Node_Id is
+      pragma Assert
+        (Nkind (N) in N_Object_Renaming_Declaration
+                    | N_Package_Renaming_Declaration
+                    | N_Exception_Renaming_Declaration);
+
       Loc : constant Source_Ptr := Sloc (N);
       Ent : constant Node_Id    := Defining_Entity (N);
       Nam : constant Node_Id    := Name (N);
@@ -410,7 +415,7 @@ package body Exp_Dbug is
                | N_Identifier
             =>
                if No (Entity (Ren))
-                 or else not Present (Renamed_Object (Entity (Ren)))
+                 or else not Present (Renamed_Entity_Or_Object (Entity (Ren)))
                then
                   exit;
                end if;
@@ -418,7 +423,7 @@ package body Exp_Dbug is
                --  This is a renaming of a renaming: traverse until the final
                --  renaming to see if anything is packed along the way.
 
-               Ren := Renamed_Object (Entity (Ren));
+               Ren := Renamed_Entity_Or_Object (Entity (Ren));
 
             when N_Selected_Component =>
                declare
diff --git a/gcc/ada/exp_disp.adb b/gcc/ada/exp_disp.adb
index 6ade54b..de2ba7a 100644
--- a/gcc/ada/exp_disp.adb
+++ b/gcc/ada/exp_disp.adb
@@ -66,6 +66,7 @@ with Sinfo.Utils;    use Sinfo.Utils;
 with Snames;         use Snames;
 with Stand;          use Stand;
 with Stringt;        use Stringt;
+with Strub;          use Strub;
 with SCIL_LL;        use SCIL_LL;
 with Tbuild;         use Tbuild;
 
@@ -846,6 +847,7 @@ package body Exp_Disp is
       end if;
 
       Subp_Typ     := Create_Itype (E_Subprogram_Type, Call_Node);
+      Copy_Strub_Mode (Subp_Typ, Subp);
       Subp_Ptr_Typ := Create_Itype (E_Access_Subprogram_Type, Call_Node);
       Set_Etype          (Subp_Typ, Res_Typ);
       Set_Returns_By_Ref (Subp_Typ, Returns_By_Ref (Subp));
@@ -3037,7 +3039,7 @@ package body Exp_Disp is
    begin
       pragma Assert (not Restriction_Active (No_Dispatching_Calls));
 
-      --  Null body is generated for interface types and non-concurrent
+      --  Null body is generated for interface types and nonconcurrent
       --  tagged types.
 
       if Is_Interface (Typ)
diff --git a/gcc/ada/exp_prag.adb b/gcc/ada/exp_prag.adb
index 27b4e7d..f0b4b0b 100644
--- a/gcc/ada/exp_prag.adb
+++ b/gcc/ada/exp_prag.adb
@@ -563,9 +563,9 @@ package body Exp_Prag is
             null;
 
          elsif Nam = Name_Assert then
-            Error_Msg_N ("?A?assertion will fail at run time", N);
+            Error_Msg_N ("?.a?assertion will fail at run time", N);
          else
-            Error_Msg_N ("?A?check will fail at run time", N);
+            Error_Msg_N ("?.a?check will fail at run time", N);
          end if;
       end if;
    end Expand_Pragma_Check;
diff --git a/gcc/ada/exp_util.adb b/gcc/ada/exp_util.adb
index 9bc9449..1bba059 100644
--- a/gcc/ada/exp_util.adb
+++ b/gcc/ada/exp_util.adb
@@ -315,10 +315,10 @@ package body Exp_Util is
 
          if Present (Msg_Node) then
             Error_Msg_N
-              ("info: atomic synchronization set for &?N?", Msg_Node);
+              ("info: atomic synchronization set for &?.n?", Msg_Node);
          else
             Error_Msg_N
-              ("info: atomic synchronization set?N?", N);
+              ("info: atomic synchronization set?.n?", N);
          end if;
       end if;
    end Activate_Atomic_Synchronization;
@@ -2849,7 +2849,7 @@ package body Exp_Util is
          if Inherited and Opt.List_Inherited_Aspects then
             Error_Msg_Sloc := Sloc (Prag);
             Error_Msg_N
-              ("info: & inherits `Invariant''Class` aspect from #?L?", Typ);
+              ("info: & inherits `Invariant''Class` aspect from #?.l?", Typ);
          end if;
 
          --  Add the pragma to the list of processed pragmas
@@ -10433,6 +10433,14 @@ package body Exp_Util is
          --  inherited ancestor primitive.
 
          elsif Present (Inher_Prim) then
+            --  It is possible that an internally generated alias could be
+            --  set to a subprogram which overrides the same aliased primitive,
+            --  so return Empty in this case.
+
+            if Ancestor_Primitive (Inher_Prim) = Subp then
+               return Empty;
+            end if;
+
             return Inher_Prim;
 
          --  Otherwise the current subprogram is the root of the inheritance or
@@ -11257,6 +11265,7 @@ package body Exp_Util is
 
          when others =>
             if Is_Entity_Name (N)
+              and then Is_Object (Entity (N))
               and then Present (Renamed_Object (Entity (N)))
             then
                return
diff --git a/gcc/ada/freeze.adb b/gcc/ada/freeze.adb
index fac7094..4d09956 100644
--- a/gcc/ada/freeze.adb
+++ b/gcc/ada/freeze.adb
@@ -69,6 +69,7 @@ with Sinfo.Utils;    use Sinfo.Utils;
 with Snames;         use Snames;
 with Stand;          use Stand;
 with Stringt;        use Stringt;
+with Strub;          use Strub;
 with Targparm;       use Targparm;
 with Tbuild;         use Tbuild;
 with Ttypes;         use Ttypes;
@@ -1790,6 +1791,10 @@ package body Freeze is
          return Result;
       end Needs_Wrapper;
 
+      Ifaces_List    : Elist_Id := No_Elist;
+      Ifaces_Listed  : Boolean := False;
+      --  Cache the list of interface operations inherited by R
+
    --  Start of processing for Check_Inherited_Conditions
 
    begin
@@ -1823,11 +1828,10 @@ package body Freeze is
       while Present (Op_Node) loop
          Prim := Node (Op_Node);
 
-         if Present (Overridden_Operation (Prim))
+         Par_Prim := Overridden_Operation (Prim);
+         if Present (Par_Prim)
            and then Comes_From_Source (Prim)
          then
-            Par_Prim := Overridden_Operation (Prim);
-
             --  When the primitive is an LSP wrapper we climb to the parent
             --  primitive that has the inherited contract.
 
@@ -1837,6 +1841,11 @@ package body Freeze is
                Par_Prim := LSP_Subprogram (Par_Prim);
             end if;
 
+            --  Check that overrider and overridden operations have
+            --  the same strub mode.
+
+            Check_Same_Strub_Mode (Prim, Par_Prim);
+
             --  Analyze the contract items of the overridden operation, before
             --  they are rewritten as pragmas.
 
@@ -1852,6 +1861,54 @@ package body Freeze is
             end if;
          end if;
 
+         --  Go over operations inherited from interfaces and check
+         --  them for strub mode compatibility as well.
+
+         if Has_Interfaces (R)
+           and then Is_Dispatching_Operation (Prim)
+           and then Find_Dispatching_Type (Prim) = R
+         then
+            declare
+               Elmt        : Elmt_Id;
+               Iface_Elmt  : Elmt_Id;
+               Iface       : Entity_Id;
+               Iface_Prim  : Entity_Id;
+
+            begin
+               --  Collect the interfaces only once. We haven't
+               --  finished freezing yet, so we can't use the faster
+               --  search from Sem_Disp.Covered_Interface_Primitives.
+
+               if not Ifaces_Listed then
+                  Collect_Interfaces (R, Ifaces_List);
+                  Ifaces_Listed := True;
+               end if;
+
+               Iface_Elmt := First_Elmt (Ifaces_List);
+               while Present (Iface_Elmt) loop
+                  Iface := Node (Iface_Elmt);
+
+                  Elmt := First_Elmt (Primitive_Operations (Iface));
+                  while Present (Elmt) loop
+                     Iface_Prim := Node (Elmt);
+
+                     if Iface_Prim /= Par_Prim
+                       and then Chars (Iface_Prim) = Chars (Prim)
+                       and then Comes_From_Source (Iface_Prim)
+                       and then (Is_Interface_Conformant
+                                   (R, Iface_Prim, Prim))
+                     then
+                        Check_Same_Strub_Mode (Prim, Iface_Prim);
+                     end if;
+
+                     Next_Elmt (Elmt);
+                  end loop;
+
+                  Next_Elmt (Iface_Elmt);
+               end loop;
+            end;
+         end if;
+
          Next_Elmt (Op_Node);
       end loop;
 
@@ -2337,7 +2394,7 @@ package body Freeze is
             --  created for entry parameters must be frozen.
 
             if Ekind (E) = E_Package
-              and then No (Renamed_Object (E))
+              and then No (Renamed_Entity (E))
               and then not Is_Child_Unit (E)
               and then not Is_Frozen (E)
             then
@@ -2504,6 +2561,18 @@ package body Freeze is
                Process_Default_Expressions (E, After);
             end if;
 
+            --  Check subprogram renamings for the same strub-mode.
+            --  Avoid rechecking dispatching operations, that's taken
+            --  care of in Check_Inherited_Conditions, that covers
+            --  inherited interface operations.
+
+            Item := Alias (E);
+            if Present (Item)
+              and then not Is_Dispatching_Operation (E)
+            then
+               Check_Same_Strub_Mode (E, Item);
+            end if;
+
             if not Has_Completion (E) then
                Decl := Unit_Declaration_Node (E);
 
@@ -3371,7 +3440,7 @@ package body Freeze is
 
                            Error_Msg_Uint_1 := Modv;
                            Error_Msg_N
-                             ("?M?2 '*'*^' may have been intended here",
+                             ("?.m?2 '*'*^' may have been intended here",
                               Modulus);
                         end;
                      end if;
@@ -6412,7 +6481,7 @@ package body Freeze is
                         then
                            Error_Msg_NE
                              ("useless postcondition, & is marked "
-                              & "No_Return?T?", Exp, E);
+                              & "No_Return?.t?", Exp, E);
                         end if;
                      end if;
 
@@ -6793,24 +6862,24 @@ package body Freeze is
                   if Sloc (SC) > Sloc (AC) then
                      Loc := SC;
                      Error_Msg_NE
-                       ("?Z?size is not a multiple of alignment for &",
+                       ("?.z?size is not a multiple of alignment for &",
                         Loc, E);
                      Error_Msg_Sloc := Sloc (AC);
                      Error_Msg_Uint_1 := Alignment (E);
-                     Error_Msg_N ("\?Z?alignment of ^ specified #", Loc);
+                     Error_Msg_N ("\?.z?alignment of ^ specified #", Loc);
 
                   else
                      Loc := AC;
                      Error_Msg_NE
-                       ("?Z?size is not a multiple of alignment for &",
+                       ("?.z?size is not a multiple of alignment for &",
                         Loc, E);
                      Error_Msg_Sloc := Sloc (SC);
                      Error_Msg_Uint_1 := RM_Size (E);
-                     Error_Msg_N ("\?Z?size of ^ specified #", Loc);
+                     Error_Msg_N ("\?.z?size of ^ specified #", Loc);
                   end if;
 
                   Error_Msg_Uint_1 := ((RM_Size (E) / Abits) + 1) * Abits;
-                  Error_Msg_N ("\?Z?Object_Size will be increased to ^", Loc);
+                  Error_Msg_N ("\?.z?Object_Size will be increased to ^", Loc);
                end if;
             end;
          end if;
@@ -9500,9 +9569,7 @@ package body Freeze is
          Minsiz : constant Uint := UI_From_Int (Minimum_Size (Typ));
 
       begin
-         if Known_RM_Size (Typ)
-           and then RM_Size (Typ) /= Uint_0
-         then
+         if Known_RM_Size (Typ) then
             if RM_Size (Typ) < Minsiz then
                Error_Msg_Uint_1 := RM_Size (Typ);
                Error_Msg_Uint_2 := Minsiz;
diff --git a/gcc/ada/gcc-interface/Make-lang.in b/gcc/ada/gcc-interface/Make-lang.in
index 61a627f..ae5158a 100644
--- a/gcc/ada/gcc-interface/Make-lang.in
+++ b/gcc/ada/gcc-interface/Make-lang.in
@@ -116,7 +116,7 @@ ADA_FLAGS_TO_PASS = \
 
 # List of Ada tools to build and install
 ADA_TOOLS=gnatbind gnatchop gnat gnatkr gnatlink gnatls gnatmake \
-          gnatname gnatprep gnatxref gnatfind gnatclean
+          gnatname gnatprep gnatclean
 
 # Say how to compile Ada programs.
 .SUFFIXES: .ada .adb .ads
@@ -440,6 +440,7 @@ GNAT_ADA_OBJS =	\
  ada/sprint.o	\
  ada/stand.o	\
  ada/stringt.o	\
+ ada/strub.o	\
  ada/style.o	\
  ada/styleg.o	\
  ada/stylesw.o	\
@@ -725,6 +726,14 @@ endif
 # For unwind-pe.h
 CFLAGS-ada/raise-gcc.o += -I$(srcdir)/../libgcc -DEH_MECHANISM_$(EH_MECHANISM)
 
+# Under aarch64 darwin, we need to include the iOS signal trampoline.
+
+ifeq ($(strip $(filter-out aarch64 arm64 darwin%,$(host_cpu) $(host_os))),)
+  EXTRA_HOST_OBJS=ada/sigtramp-ios.o
+else
+  EXTRA_HOST_OBJS =
+endif
+
 ada/libgnat/s-excmac.o: ada/libgnat/s-excmac.ads ada/libgnat/s-excmac.adb
 
 ada/libgnat/s-excmac.ads: $(srcdir)/ada/libgnat/s-excmac__$(EH_MECHANISM).ads
@@ -738,16 +747,16 @@ ada/libgnat/s-excmac.adb: $(srcdir)/ada/libgnat/s-excmac__$(EH_MECHANISM).adb
 # Needs to be built with CC=gcc
 # Since the RTL should be built with the latest compiler, remove the
 #  stamp target in the parent directory whenever gnat1 is rebuilt
-gnat1$(exeext): $(TARGET_ADA_SRCS) $(GNAT1_OBJS) $(ADA_BACKEND) libcommon-target.a \
-		$(LIBDEPS) $(ada.prev)
+gnat1$(exeext): $(TARGET_ADA_SRCS) $(GNAT1_OBJS) $(ADA_BACKEND) $(EXTRA_HOST_OBJS) libcommon-target.a \
+		$(EXTRA_HOST_OBJS) $(LIBDEPS) $(ada.prev)
 	@$(call LINK_PROGRESS,$(INDEX.ada),start)
-	+$(GCC_LLINK) -o $@ $(GNAT1_OBJS) $(ADA_BACKEND) $(CFLAGS) \
+	+$(GCC_LLINK) -o $@ $(GNAT1_OBJS) $(ADA_BACKEND) $(EXTRA_HOST_OBJS) $(CFLAGS) \
 	  libcommon-target.a $(LIBS) $(SYSLIBS) $(BACKENDLIBS) $(GNATLIB)
 	$(RM) stamp-gnatlib2-rts stamp-tools
 	@$(call LINK_PROGRESS,$(INDEX.ada),end)
 
-gnatbind$(exeext): ada/b_gnatb.o $(CONFIG_H) $(GNATBIND_OBJS) ggc-none.o libcommon-target.a $(LIBDEPS)
-	+$(GCC_LINK) -o $@ $(CFLAGS) ada/b_gnatb.o $(GNATBIND_OBJS) ggc-none.o libcommon-target.a $(LIBS) $(SYSLIBS) $(GNATLIB)
+gnatbind$(exeext): ada/b_gnatb.o $(CONFIG_H) $(GNATBIND_OBJS) $(EXTRA_HOST_OBJS) ggc-none.o libcommon-target.a $(LIBDEPS)
+	+$(GCC_LINK) -o $@ $(CFLAGS) ada/b_gnatb.o $(GNATBIND_OBJS) $(EXTRA_HOST_OBJS) ggc-none.o libcommon-target.a $(LIBS) $(SYSLIBS) $(GNATLIB)
 
 # use target-gcc target-gnatmake target-gnatbind target-gnatlink
 gnattools: $(GCC_PARTS) $(CONFIG_H) prefix.o force
@@ -838,8 +847,20 @@ ada.install-info: $(DESTDIR)$(infodir)/gnat_ugn.info \
 	$(DESTDIR)$(infodir)/gnat_rm.info \
 	$(DESTDIR)$(infodir)/gnat-style.info
 
-ada.dvi: doc/gnat_ugn.dvi \
-      doc/gnat_rm.dvi doc/gnat-style.dvi
+ADA_DVIFILES = doc/gnat_ugn.dvi \
+               doc/gnat_rm.dvi doc/gnat-style.dvi
+
+ada.dvi: $(ADA_DVIFILES)
+
+ada.install-dvi: $(ADA_DVIFILES)
+	@$(NORMAL_INSTALL)
+	test -z "$(dvidir)/gcc" || $(mkinstalldirs) "$(DESTDIR)$(dvidir)/gcc"
+	@list='$(ADA_DVIFILES)'; for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  f=$(dvi__strip_dir) \
+	  echo " $(INSTALL_DATA) '$$d$$p' '$(DESTDIR)$(dvidir)/gcc/$$f'"; \
+	  $(INSTALL_DATA) "$$d$$p" "$(DESTDIR)$(dvidir)/gcc/$$f"; \
+	done
 
 ADA_PDFFILES = doc/gnat_ugn.pdf \
                doc/gnat_rm.pdf doc/gnat-style.pdf
@@ -891,8 +912,7 @@ doc/gnat-style.pdf: ada/gnat-style.texi $(gcc_docdir)/include/fdl.texi
 # (cross). $(prefix) comes from the --program-prefix configure option,
 # or from the --target option if the former is not specified.
 # Do the same for the rest of the Ada tools (gnatchop, gnat, gnatkr,
-# gnatlink, gnatls, gnatmake, gnatname, gnatprep, gnatxref, gnatfind,
-# gnatclean).
+# gnatlink, gnatls, gnatmake, gnatname, gnatprep, gnatclean).
 # gnatdll is only used on Windows.
 ada.install-common: $(gnat_install_lib) gnat-install-tools
 
@@ -963,8 +983,6 @@ ada.distclean:
 	-$(RM) gnatmake$(exeext)
 	-$(RM) gnatname$(exeext)
 	-$(RM) gnatprep$(exeext)
-	-$(RM) gnatfind$(exeext)
-	-$(RM) gnatxref$(exeext)
 	-$(RM) gnatclean$(exeext)
 	-$(RM) ada/rts/*
 	-$(RMDIR) ada/rts
diff --git a/gcc/ada/gcc-interface/Makefile.in b/gcc/ada/gcc-interface/Makefile.in
index 4ab71977..53d0739 100644
--- a/gcc/ada/gcc-interface/Makefile.in
+++ b/gcc/ada/gcc-interface/Makefile.in
@@ -449,7 +449,7 @@ gnattools2: ../stamp-tools
 common-tools: ../stamp-tools
 	$(GNATMAKE) -j0 -c -b $(ADA_INCLUDES) \
 	  --GNATBIND="$(GNATBIND)" --GCC="$(CC) $(ALL_ADAFLAGS)" \
-	  gnatchop gnatcmd gnatkr gnatls gnatprep gnatxref gnatfind gnatname \
+	  gnatchop gnatcmd gnatkr gnatls gnatprep gnatname \
 	  gnatclean -bargs $(ADA_INCLUDES) $(GNATBIND_FLAGS)
 	$(GNATLINK) -v gnatcmd -o ../../gnat$(exeext) \
 	  --GCC="$(CC) $(ADA_INCLUDES)" --LINK="$(GCC_LINK)" $(TOOLS_LIBS)
@@ -461,10 +461,6 @@ common-tools: ../stamp-tools
 	  --GCC="$(CC) $(ADA_INCLUDES)" --LINK="$(GCC_LINK)" $(TOOLS_LIBS)
 	$(GNATLINK) -v gnatprep -o ../../gnatprep$(exeext) \
 	  --GCC="$(CC) $(ADA_INCLUDES)" --LINK="$(GCC_LINK)" $(TOOLS_LIBS)
-	$(GNATLINK) -v gnatxref -o ../../gnatxref$(exeext) \
-	  --GCC="$(CC) $(ADA_INCLUDES)" --LINK="$(GCC_LINK)" $(TOOLS_LIBS)
-	$(GNATLINK) -v gnatfind -o ../../gnatfind$(exeext) \
-	  --GCC="$(CC) $(ADA_INCLUDES)" --LINK="$(GCC_LINK)" $(TOOLS_LIBS)
 	$(GNATLINK) -v gnatname -o ../../gnatname$(exeext) \
 	  --GCC="$(CC) $(ADA_INCLUDES)" --LINK="$(GCC_LINK)" $(TOOLS_LIBS)
 	$(GNATLINK) -v gnatclean -o ../../gnatclean$(exeext) \
@@ -804,8 +800,8 @@ gnatlib-shared-darwin:
 		libgnat$(soext)
 	cd $(RTSDIR); $(LN_S) libgnarl$(hyphen)$(LIBRARY_VERSION)$(soext) \
 		libgnarl$(soext)
-	cd $(RTSDIR); dsymutil libgnat$(hyphen)$(LIBRARY_VERSION)$(soext)
-	cd $(RTSDIR); dsymutil libgnarl$(hyphen)$(LIBRARY_VERSION)$(soext)
+	cd $(RTSDIR); $(DSYMUTIL_FOR_TARGET) libgnat$(hyphen)$(LIBRARY_VERSION)$(soext)
+	cd $(RTSDIR); $(DSYMUTIL_FOR_TARGET) libgnarl$(hyphen)$(LIBRARY_VERSION)$(soext)
 
 gnatlib-shared:
 	$(MAKE) $(FLAGS_TO_PASS) \
diff --git a/gcc/ada/gcc-interface/ada-tree.h b/gcc/ada/gcc-interface/ada-tree.h
index 9fe52cf..0ec81bc 100644
--- a/gcc/ada/gcc-interface/ada-tree.h
+++ b/gcc/ada/gcc-interface/ada-tree.h
@@ -410,10 +410,6 @@ do {						   \
 
 /* Flags added to decl nodes.  */
 
-/* Nonzero in a FUNCTION_DECL that represents a stubbed function
-   discriminant.  */
-#define DECL_STUBBED_P(NODE) DECL_LANG_FLAG_0 (FUNCTION_DECL_CHECK (NODE))
-
 /* Nonzero in a VAR_DECL if it is guaranteed to be constant after having
    been elaborated and TREE_READONLY is not set on it.  */
 #define DECL_READONLY_ONCE_ELAB(NODE) DECL_LANG_FLAG_0 (VAR_DECL_CHECK (NODE))
diff --git a/gcc/ada/gcc-interface/decl.c b/gcc/ada/gcc-interface/decl.c
index 98b4aaf..449463e 100644
--- a/gcc/ada/gcc-interface/decl.c
+++ b/gcc/ada/gcc-interface/decl.c
@@ -4095,19 +4095,14 @@ gnat_to_gnu_entity (Entity_Id gnat_entity, tree gnu_expr, bool definition)
 	    else if (extern_flag && gnu_ext_name == DECL_NAME (realloc_decl))
 	      gnu_decl = realloc_decl;
 	    else
-	      {
-		gnu_decl
-		  = create_subprog_decl (gnu_entity_name, gnu_ext_name,
-					 gnu_type, gnu_param_list,
-					 inline_status, public_flag,
-					 extern_flag, artificial_p,
-					 debug_info_p,
-					 definition && imported_p, attr_list,
-					 gnat_entity);
-
-		DECL_STUBBED_P (gnu_decl)
-		  = (Convention (gnat_entity) == Convention_Stubbed);
-	      }
+	      gnu_decl
+		= create_subprog_decl (gnu_entity_name, gnu_ext_name,
+				       gnu_type, gnu_param_list,
+				       inline_status, public_flag,
+				       extern_flag, artificial_p,
+				       debug_info_p,
+				       definition && imported_p, attr_list,
+				       gnat_entity);
 	  }
       }
       break;
diff --git a/gcc/ada/gcc-interface/trans.c b/gcc/ada/gcc-interface/trans.c
index dc2a03c..a932ca2 100644
--- a/gcc/ada/gcc-interface/trans.c
+++ b/gcc/ada/gcc-interface/trans.c
@@ -4453,13 +4453,14 @@ static tree
 Call_to_gnu (Node_Id gnat_node, tree *gnu_result_type_p, tree gnu_target,
 	     atomic_acces_t atomic_access, bool atomic_sync)
 {
+  const Node_Id gnat_name = Name (gnat_node);
   const bool function_call = (Nkind (gnat_node) == N_Function_Call);
   const bool returning_value = (function_call && !gnu_target);
   /* The GCC node corresponding to the GNAT subprogram name.  This can either
      be a FUNCTION_DECL node if we are dealing with a standard subprogram call,
      or an indirect reference expression (an INDIRECT_REF node) pointing to a
      subprogram.  */
-  tree gnu_subprog = gnat_to_gnu (Name (gnat_node));
+  tree gnu_subprog = gnat_to_gnu (gnat_name);
   /* The FUNCTION_TYPE node giving the GCC type of the subprogram.  */
   tree gnu_subprog_type = TREE_TYPE (gnu_subprog);
   /* The return type of the FUNCTION_TYPE.  */
@@ -4482,50 +4483,16 @@ Call_to_gnu (Node_Id gnat_node, tree *gnu_result_type_p, tree gnu_target,
   atomic_acces_t aa_type;
   bool aa_sync;
 
-  gcc_assert (FUNC_OR_METHOD_TYPE_P (gnu_subprog_type));
-
-  /* If we are calling a stubbed function, raise Program_Error, but Elaborate
-     all our args first.  */
-  if (TREE_CODE (gnu_subprog) == FUNCTION_DECL && DECL_STUBBED_P (gnu_subprog))
-    {
-      tree call_expr = build_call_raise (PE_Stubbed_Subprogram_Called,
-					 gnat_node, N_Raise_Program_Error);
-
-      for (gnat_actual = First_Actual (gnat_node);
-	   Present (gnat_actual);
-	   gnat_actual = Next_Actual (gnat_actual))
-	add_stmt (gnat_to_gnu (gnat_actual));
-
-      if (returning_value)
-	{
-	  *gnu_result_type_p = gnu_result_type;
-	  return build1 (NULL_EXPR, gnu_result_type, call_expr);
-	}
-
-      return call_expr;
-    }
-
-  if (TREE_CODE (gnu_subprog) == FUNCTION_DECL)
-    {
-      /* For a call to a nested function, check the inlining status.  */
-      if (decl_function_context (gnu_subprog))
-	check_inlining_for_nested_subprog (gnu_subprog);
-
-      /* For a recursive call, avoid explosion due to recursive inlining.  */
-      if (gnu_subprog == current_function_decl)
-	DECL_DISREGARD_INLINE_LIMITS (gnu_subprog) = 0;
-    }
-
-  /* The only way we can be making a call via an access type is if Name is an
+  /* The only way we can make a call via an access type is if GNAT_NAME is an
      explicit dereference.  In that case, get the list of formal args from the
      type the access type is pointing to.  Otherwise, get the formals from the
      entity being called.  */
-  if (Nkind (Name (gnat_node)) == N_Explicit_Dereference)
+  if (Nkind (gnat_name) == N_Explicit_Dereference)
     {
       const Entity_Id gnat_prefix_type
-	= Underlying_Type (Etype (Prefix (Name (gnat_node))));
+	= Underlying_Type (Etype (Prefix (gnat_name)));
 
-      gnat_formal = First_Formal_With_Extras (Etype (Name (gnat_node)));
+      gnat_formal = First_Formal_With_Extras (Etype (gnat_name));
       variadic = IN (Convention (gnat_prefix_type), Convention_C_Variadic);
 
       /* If the access type doesn't require foreign-compatible representation,
@@ -4534,19 +4501,56 @@ Call_to_gnu (Node_Id gnat_node, tree *gnu_result_type_p, tree gnu_target,
 	= targetm.calls.custom_function_descriptors > 0
 	  && Can_Use_Internal_Rep (gnat_prefix_type);
     }
-  else if (Nkind (Name (gnat_node)) == N_Attribute_Reference)
+
+  else if (Nkind (gnat_name) == N_Attribute_Reference)
     {
       /* Assume here that this must be 'Elab_Body or 'Elab_Spec.  */
       gnat_formal = Empty;
       variadic = false;
       by_descriptor = false;
     }
+
   else
     {
-      gnat_formal = First_Formal_With_Extras (Entity (Name (gnat_node)));
-      variadic
-	= IN (Convention (Entity (Name (gnat_node))), Convention_C_Variadic);
+      gcc_checking_assert (Is_Entity_Name (gnat_name));
+
+      gnat_formal = First_Formal_With_Extras (Entity (gnat_name));
+      variadic = IN (Convention (Entity (gnat_name)), Convention_C_Variadic);
       by_descriptor = false;
+
+      /* If we are calling a stubbed function, then raise Program_Error, but
+	 elaborate all our args first.  */
+      if (Convention (Entity (gnat_name)) == Convention_Stubbed)
+	{
+	  tree call_expr = build_call_raise (PE_Stubbed_Subprogram_Called,
+					     gnat_node, N_Raise_Program_Error);
+
+	  for (gnat_actual = First_Actual (gnat_node);
+	       Present (gnat_actual);
+	       gnat_actual = Next_Actual (gnat_actual))
+	    add_stmt (gnat_to_gnu (gnat_actual));
+
+	  if (returning_value)
+	    {
+	      *gnu_result_type_p = gnu_result_type;
+	      return build1 (NULL_EXPR, gnu_result_type, call_expr);
+	    }
+
+	  return call_expr;
+	}
+    }
+
+  gcc_assert (FUNC_OR_METHOD_TYPE_P (gnu_subprog_type));
+
+  if (TREE_CODE (gnu_subprog) == FUNCTION_DECL)
+    {
+      /* For a call to a nested function, check the inlining status.  */
+      if (decl_function_context (gnu_subprog))
+	check_inlining_for_nested_subprog (gnu_subprog);
+
+      /* For a recursive call, avoid explosion due to recursive inlining.  */
+      if (gnu_subprog == current_function_decl)
+	DECL_DISREGARD_INLINE_LIMITS (gnu_subprog) = 0;
     }
 
   /* The lifetime of the temporaries created for the call ends right after the
@@ -4765,8 +4769,8 @@ Call_to_gnu (Node_Id gnat_node, tree *gnu_result_type_p, tree gnu_target,
 	  /* Do not initialize it for the _Init parameter of an initialization
 	     procedure since no data is meant to be passed in.  */
 	  if (Ekind (gnat_formal) == E_Out_Parameter
-	      && Is_Entity_Name (Name (gnat_node))
-	      && Is_Init_Proc (Entity (Name (gnat_node))))
+	      && Is_Entity_Name (gnat_name)
+	      && Is_Init_Proc (Entity (gnat_name)))
 	    gnu_name = gnu_temp = create_temporary ("A", TREE_TYPE (gnu_name));
 
 	  /* Initialize it on the fly like for an implicit temporary in the
@@ -5097,10 +5101,10 @@ Call_to_gnu (Node_Id gnat_node, tree *gnu_result_type_p, tree gnu_target,
       if (function_call)
 	gnu_cico_list = TREE_CHAIN (gnu_cico_list);
 
-      if (Nkind (Name (gnat_node)) == N_Explicit_Dereference)
-	gnat_formal = First_Formal_With_Extras (Etype (Name (gnat_node)));
+      if (Nkind (gnat_name) == N_Explicit_Dereference)
+	gnat_formal = First_Formal_With_Extras (Etype (gnat_name));
       else
-	gnat_formal = First_Formal_With_Extras (Entity (Name (gnat_node)));
+	gnat_formal = First_Formal_With_Extras (Entity (gnat_name));
 
       for (gnat_actual = First_Actual (gnat_node);
 	   Present (gnat_actual);
diff --git a/gcc/ada/gen_il-gen-gen_nodes.adb b/gcc/ada/gen_il-gen-gen_nodes.adb
index 3b6bd68..7125773 100644
--- a/gcc/ada/gen_il-gen-gen_nodes.adb
+++ b/gcc/ada/gen_il-gen-gen_nodes.adb
@@ -1675,16 +1675,29 @@ begin -- Gen_IL.Gen.Gen_Nodes
 
    Union (N_Is_Decl,
           Children =>
-            (N_Declaration,
+            (N_Aggregate,
+             N_Block_Statement,
+             N_Declaration,
              N_Discriminant_Specification,
+             N_Entry_Index_Specification,
              N_Enumeration_Type_Definition,
              N_Exception_Handler,
+             N_Explicit_Dereference,
+             N_Expression_With_Actions,
+             N_Extension_Aggregate,
+             N_Identifier,
+             N_Iterated_Component_Association,
              N_Later_Decl_Item,
+             N_Loop_Statement,
+             N_Null_Statement,
+             N_Number_Declaration,
              N_Package_Specification,
              N_Parameter_Specification,
              N_Renaming_Declaration,
-             N_Subprogram_Specification));
-   --  Nodes that can be returned by Declaration_Node
+             N_Quantified_Expression));
+   --  Nodes that can be returned by Declaration_Node; it can also return
+   --  Empty. Not all of these are true "declarations", but Declaration_Node
+   --  can return them in some cases.
 
    Union (N_Is_Range,
           Children =>
diff --git a/gcc/ada/gen_il-gen.adb b/gcc/ada/gen_il-gen.adb
index eed98ee..f058c5a 100644
--- a/gcc/ada/gen_il-gen.adb
+++ b/gcc/ada/gen_il-gen.adb
@@ -2157,7 +2157,8 @@ package body Gen_IL.Gen is
 
                   Put (S, F_Image (F) & " => (" &
                        Image (Field_Table (F).Field_Type) & "_Field, " &
-                       Image (Offset) & ")");
+                       Image (Offset) & ", " &
+                       Image (Field_Table (F).Type_Only) & ")");
 
                   FS := Field_Size (F);
                   FB := First_Bit (F, Offset);
@@ -2252,10 +2253,32 @@ package body Gen_IL.Gen is
          Decrease_Indent (S, 2);
          Put (S, ");" & LF & LF);
 
+         Put (S, "type Type_Only_Enum is" & LF);
+         Increase_Indent (S, 2);
+         Put (S, "(");
+
+         declare
+            First_Time : Boolean := True;
+         begin
+            for TO in Type_Only_Enum loop
+               if First_Time then
+                  First_Time := False;
+               else
+                  Put (S, ", ");
+               end if;
+
+               Put (S, Image (TO));
+            end loop;
+         end;
+
+         Decrease_Indent (S, 2);
+         Put (S, ");" & LF & LF);
+
          Put (S, "type Field_Descriptor is record" & LF);
          Increase_Indent (S, 3);
          Put (S, "Kind : Field_Kind;" & LF);
          Put (S, "Offset : Field_Offset;" & LF);
+         Put (S, "Type_Only : Type_Only_Enum;" & LF);
          Decrease_Indent (S, 3);
          Put (S, "end record;" & LF & LF);
 
diff --git a/gcc/ada/gen_il-gen.ads b/gcc/ada/gen_il-gen.ads
index 1d24ebf..56b0606 100644
--- a/gcc/ada/gen_il-gen.ads
+++ b/gcc/ada/gen_il-gen.ads
@@ -204,9 +204,22 @@ package Gen_IL.Gen is
    --  Gen_IL.Fields, and delete all occurrences from Gen_IL.Gen.Gen_Entities.
 
    --  If a field is not set, it is initialized by default to whatever value is
-   --  represented by all-zero bits, with two exceptions: Elist fields default
-   --  to No_Elist, and Uint fields default to Uint_0. In retrospect, it would
-   --  have been better to use No_Uint instead of Uint_0.
+   --  represented by all-zero bits, with some exceptions. This means Flags are
+   --  initialized to False, Node_Ids and List_Ids are initialized to Empty,
+   --  and enumeration fields are initialized to 'First of the type (assuming
+   --  there is no representation clause).
+   --
+   --  Elists default to No_Elist.
+   --
+   --  Fields of type Uint (but not its subtypes) are initialized to No_Uint.
+   --  Fields of subtypes Valid_Uint, Unat, Upos, Nonzero_Uint, and Ureal have
+   --  no default; it is an error to call a getter before calling the setter.
+   --  Likewise, other types whose range does not include zero have no default
+   --  (see package Types for the ranges).
+   --
+   --  If a node is created by a function in Nmake, then the defaults are
+   --  different from what is specified above. The parameters of Make_...
+   --  functions can have defaults specified; see Create_Syntactic_Field.
 
    procedure Create_Node_Union_Type
      (T : Abstract_Node; Children : Type_Array);
diff --git a/gcc/ada/gen_il-internals.ads b/gcc/ada/gen_il-internals.ads
index 7b095c0..3febf7f 100644
--- a/gcc/ada/gen_il-internals.ads
+++ b/gcc/ada/gen_il-internals.ads
@@ -147,6 +147,9 @@ package Gen_IL.Internals is
    --  The default is No_Type_Only, indicating the field is not one of
    --  these special "[... only]" ones.
 
+   function Image (Type_Only : Type_Only_Enum) return String is
+     (Capitalize (Type_Only'Img));
+
    Unknown_Offset : constant := -1;
    --  Initial value of Offset, so we can tell whether it has been set
 
diff --git a/gcc/ada/gnat_rm.texi b/gcc/ada/gnat_rm.texi
index 0a962ee..a8232f2 100644
--- a/gcc/ada/gnat_rm.texi
+++ b/gcc/ada/gnat_rm.texi
@@ -21,7 +21,7 @@
 
 @copying
 @quotation
-GNAT Reference Manual , Sep 28, 2021
+GNAT Reference Manual , Nov 08, 2021
 
 AdaCore
 
@@ -883,6 +883,7 @@ Security Hardening Features
 
 * Register Scrubbing:: 
 * Stack Scrubbing:: 
+* Hardened Conditionals:: 
 
 Obsolescent Features
 
@@ -3707,9 +3708,32 @@ overlaps the corresponding set of a later alternative, then the first
 set shall be a proper subset of the second (and the later alternative
 will not be executed if the earlier alternative “matches”). All possible
 values of the composite type shall be covered. The composite type of the
-selector shall be a nonlimited untagged (but possibly discriminated)
-record type, all of whose subcomponent subtypes are either static discrete
-subtypes or record types that meet the same restrictions.
+selector shall be an array or record type that is neither limited
+class-wide.
+
+If a subcomponent’s subtype does not meet certain restrictions, then
+the only value that can be specified for that subcomponent in a case
+choice expression is a “box” component association (which matches all
+possible values for the subcomponent). This restriction applies if
+
+
+@itemize -
+
+@item 
+the component subtype is not a record, array, or discrete type; or
+
+@item 
+the component subtype is subject to a non-static constraint or
+has a predicate; or
+
+@item 
+the component type is an enumeration type that is subject to an
+enumeration representation clause; or
+
+@item 
+the component type is a multidimensional array type or an
+array type with a nonstatic index subtype.
+@end itemize
 
 Support for casing on arrays (and on records that contain arrays) is
 currently subject to some restrictions. Non-positional
@@ -28841,6 +28865,7 @@ are provided by GNAT.
 @menu
 * Register Scrubbing:: 
 * Stack Scrubbing:: 
+* Hardened Conditionals:: 
 
 @end menu
 
@@ -28872,7 +28897,7 @@ For usage and more details on the command line option, and on the
 
 @c Stack Scrubbing:
 
-@node Stack Scrubbing,,Register Scrubbing,Security Hardening Features
+@node Stack Scrubbing,Hardened Conditionals,Register Scrubbing,Security Hardening Features
 @anchor{gnat_rm/security_hardening_features stack-scrubbing}@anchor{43e}
 @section Stack Scrubbing
 
@@ -28907,23 +28932,96 @@ Note that Ada secondary stacks are not scrubbed.  The restriction
 @code{No_Secondary_Stack} avoids their use, and thus their accidental
 preservation of data that should be scrubbed.
 
-Also note that the machine attribute is not integrated in the Ada type
-system.  Though it may modify subprogram and variable interfaces, it
-is not fully reflected in Ada types, @code{Access} attributes, renaming
-and overriding.  Every access type, renaming, and overriding and
-overridden dispatching operations that may refer to an entity with an
-attribute-modified interface must be annotated with the same
-interface-modifying attribute, or with an interface-compatible one.
-
-Even then, the pragma is currently only functional when applied to
-subprograms and scalar variables; other uses, such as directly on
-types and subtypes, may be silently ignored.  Specifically, it is not
-currently recommended to rely on any effects this pragma might be
-expected to have when calling subprograms through access-to-subprogram
-variables.
+Attributes @code{Access} and @code{Unconstrained_Access} of variables and
+constants with @code{strub} enabled require types with @code{strub} enabled;
+there is no way to express an access-to-strub type otherwise.
+@code{Unchecked_Access} bypasses this constraint, but the resulting
+access type designates a non-strub type.
+
+@example
+VI : Integer;
+XsVI : access Integer := VI'Access; -- Error.
+UXsVI : access Integer := VI'Unchecked_Access; -- OK,
+-- UXsVI.all does not enable strub in the enclosing subprogram.
+
+type Strub_Int is new Integer;
+pragma Machine_Attribute (Strub_Int, "strub");
+VSI : Strub_Int;
+XsVSI : access Strub_Int := VSI'Access; -- OK.
+-- XsVSI.all enables strub in the enclosing subprogram.
+@end example
+
+Every access-to-subprogram type, renaming, and overriding and
+overridden dispatching operations that may refer to a subprogram with
+an attribute-modified interface must be annotated with the same
+interface-modifying attribute.  Access-to-subprogram types can be
+explicitly converted to different strub modes, as long as they are
+interface-compatible (i.e., adding or removing @code{at-calls} is not
+allowed).  For example, a @code{strub}-@code{disabled} subprogram can be
+turned @code{callable} through such an explicit conversion:
+
+@example
+type TBar is access procedure;
+
+type TBar_Callable is access procedure;
+pragma Machine_Attribute (TBar_Callable, "strub", "callable");
+
+Bar_Callable_Ptr : constant TBar_Callable
+           := TBar_Callable (TBar'(Bar'Access));
+
+procedure Bar_Callable renames Bar_Callable_Ptr.all;
+pragma Machine_Attribute (Bar_Callable, "strub", "callable");
+@end example
+
+Note that the renaming declaration is expanded to a full subprogram
+body, it won’t be just an alias.  Only if it is inlined will it be as
+efficient as a call by dereferencing the access-to-subprogram constant
+Bar_Callable_Ptr.
+
+@c Hardened Conditionals:
+
+@node Hardened Conditionals,,Stack Scrubbing,Security Hardening Features
+@anchor{gnat_rm/security_hardening_features hardened-conditionals}@anchor{43f}
+@section Hardened Conditionals
+
+
+GNAT can harden conditionals to protect against control flow attacks.
+
+This is accomplished by two complementary transformations, each
+activated by a separate command-line option.
+
+The option @emph{-fharden-compares} enables hardening of compares that
+compute results stored in variables, adding verification that the
+reversed compare yields the opposite result.
+
+The option @emph{-fharden-conditional-branches} enables hardening of
+compares that guard conditional branches, adding verification of the
+reversed compare to both execution paths.
+
+These transformations are introduced late in the compilation pipeline,
+long after boolean expressions are decomposed into separate compares,
+each one turned into either a conditional branch or a compare whose
+result is stored in a boolean variable or temporary.  Compiler
+optimizations, if enabled, may also turn conditional branches into
+stored compares, and vice-versa, or into operations with implied
+conditionals (e.g. MIN and MAX).  Conditionals may also be optimized
+out entirely, if their value can be determined at compile time, and
+occasionally multiple compares can be combined into one.
+
+It is thus difficult to predict which of these two options will affect
+a specific compare operation expressed in source code.  Using both
+options ensures that every compare that is neither optimized out nor
+optimized into implied conditionals will be hardened.
+
+The addition of reversed compares can be observed by enabling the dump
+files of the corresponding passes, through command line options
+@emph{-fdump-tree-hardcmp} and @emph{-fdump-tree-hardcbr}, respectively.
+
+They are separate options, however, because of the significantly
+different performance impact of the hardening transformations.
 
 @node Obsolescent Features,Compatibility and Porting Guide,Security Hardening Features,Top
-@anchor{gnat_rm/obsolescent_features doc}@anchor{43f}@anchor{gnat_rm/obsolescent_features id1}@anchor{440}@anchor{gnat_rm/obsolescent_features obsolescent-features}@anchor{16}
+@anchor{gnat_rm/obsolescent_features doc}@anchor{440}@anchor{gnat_rm/obsolescent_features id1}@anchor{441}@anchor{gnat_rm/obsolescent_features obsolescent-features}@anchor{16}
 @chapter Obsolescent Features
 
 
@@ -28942,7 +29040,7 @@ compatibility purposes.
 @end menu
 
 @node pragma No_Run_Time,pragma Ravenscar,,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id2}@anchor{441}@anchor{gnat_rm/obsolescent_features pragma-no-run-time}@anchor{442}
+@anchor{gnat_rm/obsolescent_features id2}@anchor{442}@anchor{gnat_rm/obsolescent_features pragma-no-run-time}@anchor{443}
 @section pragma No_Run_Time
 
 
@@ -28955,7 +29053,7 @@ preferred usage is to use an appropriately configured run-time that
 includes just those features that are to be made accessible.
 
 @node pragma Ravenscar,pragma Restricted_Run_Time,pragma No_Run_Time,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id3}@anchor{443}@anchor{gnat_rm/obsolescent_features pragma-ravenscar}@anchor{444}
+@anchor{gnat_rm/obsolescent_features id3}@anchor{444}@anchor{gnat_rm/obsolescent_features pragma-ravenscar}@anchor{445}
 @section pragma Ravenscar
 
 
@@ -28964,7 +29062,7 @@ The pragma @code{Ravenscar} has exactly the same effect as pragma
 is part of the new Ada 2005 standard.
 
 @node pragma Restricted_Run_Time,pragma Task_Info,pragma Ravenscar,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id4}@anchor{445}@anchor{gnat_rm/obsolescent_features pragma-restricted-run-time}@anchor{446}
+@anchor{gnat_rm/obsolescent_features id4}@anchor{446}@anchor{gnat_rm/obsolescent_features pragma-restricted-run-time}@anchor{447}
 @section pragma Restricted_Run_Time
 
 
@@ -28974,7 +29072,7 @@ preferred since the Ada 2005 pragma @code{Profile} is intended for
 this kind of implementation dependent addition.
 
 @node pragma Task_Info,package System Task_Info s-tasinf ads,pragma Restricted_Run_Time,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id5}@anchor{447}@anchor{gnat_rm/obsolescent_features pragma-task-info}@anchor{448}
+@anchor{gnat_rm/obsolescent_features id5}@anchor{448}@anchor{gnat_rm/obsolescent_features pragma-task-info}@anchor{449}
 @section pragma Task_Info
 
 
@@ -29000,7 +29098,7 @@ in the spec of package System.Task_Info in the runtime
 library.
 
 @node package System Task_Info s-tasinf ads,,pragma Task_Info,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features package-system-task-info}@anchor{449}@anchor{gnat_rm/obsolescent_features package-system-task-info-s-tasinf-ads}@anchor{44a}
+@anchor{gnat_rm/obsolescent_features package-system-task-info}@anchor{44a}@anchor{gnat_rm/obsolescent_features package-system-task-info-s-tasinf-ads}@anchor{44b}
 @section package System.Task_Info (@code{s-tasinf.ads})
 
 
@@ -29010,7 +29108,7 @@ to support the @code{Task_Info} pragma. The predefined Ada package
 standard replacement for GNAT’s @code{Task_Info} functionality.
 
 @node Compatibility and Porting Guide,GNU Free Documentation License,Obsolescent Features,Top
-@anchor{gnat_rm/compatibility_and_porting_guide doc}@anchor{44b}@anchor{gnat_rm/compatibility_and_porting_guide compatibility-and-porting-guide}@anchor{17}@anchor{gnat_rm/compatibility_and_porting_guide id1}@anchor{44c}
+@anchor{gnat_rm/compatibility_and_porting_guide doc}@anchor{44c}@anchor{gnat_rm/compatibility_and_porting_guide compatibility-and-porting-guide}@anchor{17}@anchor{gnat_rm/compatibility_and_porting_guide id1}@anchor{44d}
 @chapter Compatibility and Porting Guide
 
 
@@ -29032,7 +29130,7 @@ applications developed in other Ada environments.
 @end menu
 
 @node Writing Portable Fixed-Point Declarations,Compatibility with Ada 83,,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide id2}@anchor{44d}@anchor{gnat_rm/compatibility_and_porting_guide writing-portable-fixed-point-declarations}@anchor{44e}
+@anchor{gnat_rm/compatibility_and_porting_guide id2}@anchor{44e}@anchor{gnat_rm/compatibility_and_porting_guide writing-portable-fixed-point-declarations}@anchor{44f}
 @section Writing Portable Fixed-Point Declarations
 
 
@@ -29154,7 +29252,7 @@ If you follow this scheme you will be guaranteed that your fixed-point
 types will be portable.
 
 @node Compatibility with Ada 83,Compatibility between Ada 95 and Ada 2005,Writing Portable Fixed-Point Declarations,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-ada-83}@anchor{44f}@anchor{gnat_rm/compatibility_and_porting_guide id3}@anchor{450}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-ada-83}@anchor{450}@anchor{gnat_rm/compatibility_and_porting_guide id3}@anchor{451}
 @section Compatibility with Ada 83
 
 
@@ -29182,7 +29280,7 @@ following subsections treat the most likely issues to be encountered.
 @end menu
 
 @node Legal Ada 83 programs that are illegal in Ada 95,More deterministic semantics,,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide id4}@anchor{451}@anchor{gnat_rm/compatibility_and_porting_guide legal-ada-83-programs-that-are-illegal-in-ada-95}@anchor{452}
+@anchor{gnat_rm/compatibility_and_porting_guide id4}@anchor{452}@anchor{gnat_rm/compatibility_and_porting_guide legal-ada-83-programs-that-are-illegal-in-ada-95}@anchor{453}
 @subsection Legal Ada 83 programs that are illegal in Ada 95
 
 
@@ -29282,7 +29380,7 @@ the fix is usually simply to add the @code{(<>)} to the generic declaration.
 @end itemize
 
 @node More deterministic semantics,Changed semantics,Legal Ada 83 programs that are illegal in Ada 95,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide id5}@anchor{453}@anchor{gnat_rm/compatibility_and_porting_guide more-deterministic-semantics}@anchor{454}
+@anchor{gnat_rm/compatibility_and_porting_guide id5}@anchor{454}@anchor{gnat_rm/compatibility_and_porting_guide more-deterministic-semantics}@anchor{455}
 @subsection More deterministic semantics
 
 
@@ -29310,7 +29408,7 @@ which open select branches are executed.
 @end itemize
 
 @node Changed semantics,Other language compatibility issues,More deterministic semantics,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide changed-semantics}@anchor{455}@anchor{gnat_rm/compatibility_and_porting_guide id6}@anchor{456}
+@anchor{gnat_rm/compatibility_and_porting_guide changed-semantics}@anchor{456}@anchor{gnat_rm/compatibility_and_porting_guide id6}@anchor{457}
 @subsection Changed semantics
 
 
@@ -29352,7 +29450,7 @@ covers only the restricted range.
 @end itemize
 
 @node Other language compatibility issues,,Changed semantics,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide id7}@anchor{457}@anchor{gnat_rm/compatibility_and_porting_guide other-language-compatibility-issues}@anchor{458}
+@anchor{gnat_rm/compatibility_and_porting_guide id7}@anchor{458}@anchor{gnat_rm/compatibility_and_porting_guide other-language-compatibility-issues}@anchor{459}
 @subsection Other language compatibility issues
 
 
@@ -29385,7 +29483,7 @@ include @code{pragma Interface} and the floating point type attributes
 @end itemize
 
 @node Compatibility between Ada 95 and Ada 2005,Implementation-dependent characteristics,Compatibility with Ada 83,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-between-ada-95-and-ada-2005}@anchor{459}@anchor{gnat_rm/compatibility_and_porting_guide id8}@anchor{45a}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-between-ada-95-and-ada-2005}@anchor{45a}@anchor{gnat_rm/compatibility_and_porting_guide id8}@anchor{45b}
 @section Compatibility between Ada 95 and Ada 2005
 
 
@@ -29457,7 +29555,7 @@ can declare a function returning a value from an anonymous access type.
 @end itemize
 
 @node Implementation-dependent characteristics,Compatibility with Other Ada Systems,Compatibility between Ada 95 and Ada 2005,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide id9}@anchor{45b}@anchor{gnat_rm/compatibility_and_porting_guide implementation-dependent-characteristics}@anchor{45c}
+@anchor{gnat_rm/compatibility_and_porting_guide id9}@anchor{45c}@anchor{gnat_rm/compatibility_and_porting_guide implementation-dependent-characteristics}@anchor{45d}
 @section Implementation-dependent characteristics
 
 
@@ -29480,7 +29578,7 @@ transition from certain Ada 83 compilers.
 @end menu
 
 @node Implementation-defined pragmas,Implementation-defined attributes,,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id10}@anchor{45d}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-pragmas}@anchor{45e}
+@anchor{gnat_rm/compatibility_and_porting_guide id10}@anchor{45e}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-pragmas}@anchor{45f}
 @subsection Implementation-defined pragmas
 
 
@@ -29502,7 +29600,7 @@ avoiding compiler rejection of units that contain such pragmas; they are not
 relevant in a GNAT context and hence are not otherwise implemented.
 
 @node Implementation-defined attributes,Libraries,Implementation-defined pragmas,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id11}@anchor{45f}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-attributes}@anchor{460}
+@anchor{gnat_rm/compatibility_and_porting_guide id11}@anchor{460}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-attributes}@anchor{461}
 @subsection Implementation-defined attributes
 
 
@@ -29516,7 +29614,7 @@ Ada 83, GNAT supplies the attributes @code{Bit}, @code{Machine_Size} and
 @code{Type_Class}.
 
 @node Libraries,Elaboration order,Implementation-defined attributes,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id12}@anchor{461}@anchor{gnat_rm/compatibility_and_porting_guide libraries}@anchor{462}
+@anchor{gnat_rm/compatibility_and_porting_guide id12}@anchor{462}@anchor{gnat_rm/compatibility_and_porting_guide libraries}@anchor{463}
 @subsection Libraries
 
 
@@ -29545,7 +29643,7 @@ be preferable to retrofit the application using modular types.
 @end itemize
 
 @node Elaboration order,Target-specific aspects,Libraries,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide elaboration-order}@anchor{463}@anchor{gnat_rm/compatibility_and_porting_guide id13}@anchor{464}
+@anchor{gnat_rm/compatibility_and_porting_guide elaboration-order}@anchor{464}@anchor{gnat_rm/compatibility_and_porting_guide id13}@anchor{465}
 @subsection Elaboration order
 
 
@@ -29581,7 +29679,7 @@ pragmas either globally (as an effect of the @emph{-gnatE} switch) or locally
 @end itemize
 
 @node Target-specific aspects,,Elaboration order,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id14}@anchor{465}@anchor{gnat_rm/compatibility_and_porting_guide target-specific-aspects}@anchor{466}
+@anchor{gnat_rm/compatibility_and_porting_guide id14}@anchor{466}@anchor{gnat_rm/compatibility_and_porting_guide target-specific-aspects}@anchor{467}
 @subsection Target-specific aspects
 
 
@@ -29594,10 +29692,10 @@ on the robustness of the original design.  Moreover, Ada 95 (and thus
 Ada 2005 and Ada 2012) are sometimes
 incompatible with typical Ada 83 compiler practices regarding implicit
 packing, the meaning of the Size attribute, and the size of access values.
-GNAT’s approach to these issues is described in @ref{467,,Representation Clauses}.
+GNAT’s approach to these issues is described in @ref{468,,Representation Clauses}.
 
 @node Compatibility with Other Ada Systems,Representation Clauses,Implementation-dependent characteristics,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-other-ada-systems}@anchor{468}@anchor{gnat_rm/compatibility_and_porting_guide id15}@anchor{469}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-other-ada-systems}@anchor{469}@anchor{gnat_rm/compatibility_and_porting_guide id15}@anchor{46a}
 @section Compatibility with Other Ada Systems
 
 
@@ -29640,7 +29738,7 @@ far beyond this minimal set, as described in the next section.
 @end itemize
 
 @node Representation Clauses,Compatibility with HP Ada 83,Compatibility with Other Ada Systems,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide id16}@anchor{46a}@anchor{gnat_rm/compatibility_and_porting_guide representation-clauses}@anchor{467}
+@anchor{gnat_rm/compatibility_and_porting_guide id16}@anchor{46b}@anchor{gnat_rm/compatibility_and_porting_guide representation-clauses}@anchor{468}
 @section Representation Clauses
 
 
@@ -29733,7 +29831,7 @@ with thin pointers.
 @end itemize
 
 @node Compatibility with HP Ada 83,,Representation Clauses,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-hp-ada-83}@anchor{46b}@anchor{gnat_rm/compatibility_and_porting_guide id17}@anchor{46c}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-hp-ada-83}@anchor{46c}@anchor{gnat_rm/compatibility_and_porting_guide id17}@anchor{46d}
 @section Compatibility with HP Ada 83
 
 
@@ -29763,7 +29861,7 @@ extension of package System.
 @end itemize
 
 @node GNU Free Documentation License,Index,Compatibility and Porting Guide,Top
-@anchor{share/gnu_free_documentation_license doc}@anchor{46d}@anchor{share/gnu_free_documentation_license gnu-fdl}@anchor{1}@anchor{share/gnu_free_documentation_license gnu-free-documentation-license}@anchor{46e}
+@anchor{share/gnu_free_documentation_license doc}@anchor{46e}@anchor{share/gnu_free_documentation_license gnu-fdl}@anchor{1}@anchor{share/gnu_free_documentation_license gnu-free-documentation-license}@anchor{46f}
 @chapter GNU Free Documentation License
 
 
diff --git a/gcc/ada/gnat_ugn.texi b/gcc/ada/gnat_ugn.texi
index cae1fad..00195e8 100644
--- a/gcc/ada/gnat_ugn.texi
+++ b/gcc/ada/gnat_ugn.texi
@@ -21,7 +21,7 @@
 
 @copying
 @quotation
-GNAT User's Guide for Native Platforms , Oct 19, 2021
+GNAT User's Guide for Native Platforms , Nov 08, 2021
 
 AdaCore
 
@@ -433,6 +433,7 @@ Specifying a Run-Time Library
 GNU/Linux Topics
 
 * Required Packages on GNU/Linux:: 
+* A GNU/Linux Debug Quirk:: 
 
 Microsoft Windows Topics
 
@@ -22650,14 +22651,15 @@ have sufficient priviledge for running the container image.
 
 
 This section describes topics that are specific to GNU/Linux platforms.
-
+@anchor{gnat_ugn/platform_specific_information required-packages-on-gnu-linux}@anchor{1bd}
 @menu
 * Required Packages on GNU/Linux:: 
+* A GNU/Linux Debug Quirk:: 
 
 @end menu
 
-@node Required Packages on GNU/Linux,,,GNU/Linux Topics
-@anchor{gnat_ugn/platform_specific_information id7}@anchor{1bd}@anchor{gnat_ugn/platform_specific_information required-packages-on-gnu-linux}@anchor{1be}
+@node Required Packages on GNU/Linux,A GNU/Linux Debug Quirk,,GNU/Linux Topics
+@anchor{gnat_ugn/platform_specific_information a-gnu-linux-debug-quirk}@anchor{1be}@anchor{gnat_ugn/platform_specific_information id7}@anchor{1bf}
 @subsection Required Packages on GNU/Linux
 
 
@@ -22690,10 +22692,28 @@ Debian, Ubuntu: @code{libc6:i386}, @code{libc6-dev:i386}, @code{lib32ncursesw5}
 Other GNU/Linux distributions might be choosing a different name
 for those packages.
 
+@node A GNU/Linux Debug Quirk,,Required Packages on GNU/Linux,GNU/Linux Topics
+@anchor{gnat_ugn/platform_specific_information id8}@anchor{1c0}
+@subsection A GNU/Linux Debug Quirk
+
+
+On SuSE 15, some kernels have a defect causing issues when debugging
+programs using threads or Ada tasks. Due to the lack of documentation
+found regarding this kernel issue, we can only provide limited
+information about which kernels are impacted: kernel version 5.3.18 is
+known to be impacted, and kernels in the 5.14 range or newer are
+believed to fix this problem.
+
+The bug affects the debugging of 32-bit processes on a 64-bit system.
+Symptoms can vary: Unexpected @code{SIGABRT} signals being received by
+the program, “The futex facility returned an unexpected error code”
+error message, and inferior programs hanging indefinitely range among
+the symptoms most commonly observed.
+
 @geindex Windows
 
 @node Microsoft Windows Topics,Mac OS Topics,GNU/Linux Topics,Platform-Specific Information
-@anchor{gnat_ugn/platform_specific_information id8}@anchor{1bf}@anchor{gnat_ugn/platform_specific_information microsoft-windows-topics}@anchor{1c0}
+@anchor{gnat_ugn/platform_specific_information id9}@anchor{1c1}@anchor{gnat_ugn/platform_specific_information microsoft-windows-topics}@anchor{1c2}
 @section Microsoft Windows Topics
 
 
@@ -22714,7 +22734,7 @@ platforms.
 @end menu
 
 @node Using GNAT on Windows,Using a network installation of GNAT,,Microsoft Windows Topics
-@anchor{gnat_ugn/platform_specific_information id9}@anchor{1c1}@anchor{gnat_ugn/platform_specific_information using-gnat-on-windows}@anchor{1c2}
+@anchor{gnat_ugn/platform_specific_information id10}@anchor{1c3}@anchor{gnat_ugn/platform_specific_information using-gnat-on-windows}@anchor{1c4}
 @subsection Using GNAT on Windows
 
 
@@ -22791,7 +22811,7 @@ uninstall or integrate different GNAT products.
 @end itemize
 
 @node Using a network installation of GNAT,CONSOLE and WINDOWS subsystems,Using GNAT on Windows,Microsoft Windows Topics
-@anchor{gnat_ugn/platform_specific_information id10}@anchor{1c3}@anchor{gnat_ugn/platform_specific_information using-a-network-installation-of-gnat}@anchor{1c4}
+@anchor{gnat_ugn/platform_specific_information id11}@anchor{1c5}@anchor{gnat_ugn/platform_specific_information using-a-network-installation-of-gnat}@anchor{1c6}
 @subsection Using a network installation of GNAT
 
 
@@ -22818,7 +22838,7 @@ transfer of large amounts of data across the network and will likely cause
 serious performance penalty.
 
 @node CONSOLE and WINDOWS subsystems,Temporary Files,Using a network installation of GNAT,Microsoft Windows Topics
-@anchor{gnat_ugn/platform_specific_information console-and-windows-subsystems}@anchor{1c5}@anchor{gnat_ugn/platform_specific_information id11}@anchor{1c6}
+@anchor{gnat_ugn/platform_specific_information console-and-windows-subsystems}@anchor{1c7}@anchor{gnat_ugn/platform_specific_information id12}@anchor{1c8}
 @subsection CONSOLE and WINDOWS subsystems
 
 
@@ -22843,7 +22863,7 @@ $ gnatmake winprog -largs -mwindows
 @end quotation
 
 @node Temporary Files,Disabling Command Line Argument Expansion,CONSOLE and WINDOWS subsystems,Microsoft Windows Topics
-@anchor{gnat_ugn/platform_specific_information id12}@anchor{1c7}@anchor{gnat_ugn/platform_specific_information temporary-files}@anchor{1c8}
+@anchor{gnat_ugn/platform_specific_information id13}@anchor{1c9}@anchor{gnat_ugn/platform_specific_information temporary-files}@anchor{1ca}
 @subsection Temporary Files
 
 
@@ -22882,7 +22902,7 @@ environments where you may not have write access to some
 directories.
 
 @node Disabling Command Line Argument Expansion,Windows Socket Timeouts,Temporary Files,Microsoft Windows Topics
-@anchor{gnat_ugn/platform_specific_information disabling-command-line-argument-expansion}@anchor{1c9}
+@anchor{gnat_ugn/platform_specific_information disabling-command-line-argument-expansion}@anchor{1cb}
 @subsection Disabling Command Line Argument Expansion
 
 
@@ -22953,7 +22973,7 @@ Ada.Command_Line.Argument (1) -> "'*.txt'"
 @end example
 
 @node Windows Socket Timeouts,Mixed-Language Programming on Windows,Disabling Command Line Argument Expansion,Microsoft Windows Topics
-@anchor{gnat_ugn/platform_specific_information windows-socket-timeouts}@anchor{1ca}
+@anchor{gnat_ugn/platform_specific_information windows-socket-timeouts}@anchor{1cc}
 @subsection Windows Socket Timeouts
 
 
@@ -22999,7 +23019,7 @@ shorter than 500 ms is needed on these Windows versions, a call to
 Check_Selector should be added before any socket read or write operations.
 
 @node Mixed-Language Programming on Windows,Windows Specific Add-Ons,Windows Socket Timeouts,Microsoft Windows Topics
-@anchor{gnat_ugn/platform_specific_information id13}@anchor{1cb}@anchor{gnat_ugn/platform_specific_information mixed-language-programming-on-windows}@anchor{1cc}
+@anchor{gnat_ugn/platform_specific_information id14}@anchor{1cd}@anchor{gnat_ugn/platform_specific_information mixed-language-programming-on-windows}@anchor{1ce}
 @subsection Mixed-Language Programming on Windows
 
 
@@ -23021,12 +23041,12 @@ to use the Microsoft tools for your C++ code, you have two choices:
 Encapsulate your C++ code in a DLL to be linked with your Ada
 application. In this case, use the Microsoft or whatever environment to
 build the DLL and use GNAT to build your executable
-(@ref{1cd,,Using DLLs with GNAT}).
+(@ref{1cf,,Using DLLs with GNAT}).
 
 @item 
 Or you can encapsulate your Ada code in a DLL to be linked with the
 other part of your application. In this case, use GNAT to build the DLL
-(@ref{1ce,,Building DLLs with GNAT Project files}) and use the Microsoft
+(@ref{1d0,,Building DLLs with GNAT Project files}) and use the Microsoft
 or whatever environment to build your executable.
 @end itemize
 
@@ -23083,7 +23103,7 @@ native SEH support is used.
 @end menu
 
 @node Windows Calling Conventions,Introduction to Dynamic Link Libraries DLLs,,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information id14}@anchor{1cf}@anchor{gnat_ugn/platform_specific_information windows-calling-conventions}@anchor{1d0}
+@anchor{gnat_ugn/platform_specific_information id15}@anchor{1d1}@anchor{gnat_ugn/platform_specific_information windows-calling-conventions}@anchor{1d2}
 @subsubsection Windows Calling Conventions
 
 
@@ -23128,7 +23148,7 @@ are available for Windows:
 @end menu
 
 @node C Calling Convention,Stdcall Calling Convention,,Windows Calling Conventions
-@anchor{gnat_ugn/platform_specific_information c-calling-convention}@anchor{1d1}@anchor{gnat_ugn/platform_specific_information id15}@anchor{1d2}
+@anchor{gnat_ugn/platform_specific_information c-calling-convention}@anchor{1d3}@anchor{gnat_ugn/platform_specific_information id16}@anchor{1d4}
 @subsubsection @code{C} Calling Convention
 
 
@@ -23170,10 +23190,10 @@ is missing, as in the above example, this parameter is set to be the
 When importing a variable defined in C, you should always use the @code{C}
 calling convention unless the object containing the variable is part of a
 DLL (in which case you should use the @code{Stdcall} calling
-convention, @ref{1d3,,Stdcall Calling Convention}).
+convention, @ref{1d5,,Stdcall Calling Convention}).
 
 @node Stdcall Calling Convention,Win32 Calling Convention,C Calling Convention,Windows Calling Conventions
-@anchor{gnat_ugn/platform_specific_information id16}@anchor{1d4}@anchor{gnat_ugn/platform_specific_information stdcall-calling-convention}@anchor{1d3}
+@anchor{gnat_ugn/platform_specific_information id17}@anchor{1d6}@anchor{gnat_ugn/platform_specific_information stdcall-calling-convention}@anchor{1d5}
 @subsubsection @code{Stdcall} Calling Convention
 
 
@@ -23270,7 +23290,7 @@ Note that to ease building cross-platform bindings this convention
 will be handled as a @code{C} calling convention on non-Windows platforms.
 
 @node Win32 Calling Convention,DLL Calling Convention,Stdcall Calling Convention,Windows Calling Conventions
-@anchor{gnat_ugn/platform_specific_information id17}@anchor{1d5}@anchor{gnat_ugn/platform_specific_information win32-calling-convention}@anchor{1d6}
+@anchor{gnat_ugn/platform_specific_information id18}@anchor{1d7}@anchor{gnat_ugn/platform_specific_information win32-calling-convention}@anchor{1d8}
 @subsubsection @code{Win32} Calling Convention
 
 
@@ -23278,7 +23298,7 @@ This convention, which is GNAT-specific is fully equivalent to the
 @code{Stdcall} calling convention described above.
 
 @node DLL Calling Convention,,Win32 Calling Convention,Windows Calling Conventions
-@anchor{gnat_ugn/platform_specific_information dll-calling-convention}@anchor{1d7}@anchor{gnat_ugn/platform_specific_information id18}@anchor{1d8}
+@anchor{gnat_ugn/platform_specific_information dll-calling-convention}@anchor{1d9}@anchor{gnat_ugn/platform_specific_information id19}@anchor{1da}
 @subsubsection @code{DLL} Calling Convention
 
 
@@ -23286,7 +23306,7 @@ This convention, which is GNAT-specific is fully equivalent to the
 @code{Stdcall} calling convention described above.
 
 @node Introduction to Dynamic Link Libraries DLLs,Using DLLs with GNAT,Windows Calling Conventions,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information id19}@anchor{1d9}@anchor{gnat_ugn/platform_specific_information introduction-to-dynamic-link-libraries-dlls}@anchor{1da}
+@anchor{gnat_ugn/platform_specific_information id20}@anchor{1db}@anchor{gnat_ugn/platform_specific_information introduction-to-dynamic-link-libraries-dlls}@anchor{1dc}
 @subsubsection Introduction to Dynamic Link Libraries (DLLs)
 
 
@@ -23370,10 +23390,10 @@ As a side note, an interesting difference between Microsoft DLLs and
 Unix shared libraries, is the fact that on most Unix systems all public
 routines are exported by default in a Unix shared library, while under
 Windows it is possible (but not required) to list exported routines in
-a definition file (see @ref{1db,,The Definition File}).
+a definition file (see @ref{1dd,,The Definition File}).
 
 @node Using DLLs with GNAT,Building DLLs with GNAT Project files,Introduction to Dynamic Link Libraries DLLs,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information id20}@anchor{1dc}@anchor{gnat_ugn/platform_specific_information using-dlls-with-gnat}@anchor{1cd}
+@anchor{gnat_ugn/platform_specific_information id21}@anchor{1de}@anchor{gnat_ugn/platform_specific_information using-dlls-with-gnat}@anchor{1cf}
 @subsubsection Using DLLs with GNAT
 
 
@@ -23464,7 +23484,7 @@ example a fictitious DLL called @code{API.dll}.
 @end menu
 
 @node Creating an Ada Spec for the DLL Services,Creating an Import Library,,Using DLLs with GNAT
-@anchor{gnat_ugn/platform_specific_information creating-an-ada-spec-for-the-dll-services}@anchor{1dd}@anchor{gnat_ugn/platform_specific_information id21}@anchor{1de}
+@anchor{gnat_ugn/platform_specific_information creating-an-ada-spec-for-the-dll-services}@anchor{1df}@anchor{gnat_ugn/platform_specific_information id22}@anchor{1e0}
 @subsubsection Creating an Ada Spec for the DLL Services
 
 
@@ -23504,7 +23524,7 @@ end API;
 @end quotation
 
 @node Creating an Import Library,,Creating an Ada Spec for the DLL Services,Using DLLs with GNAT
-@anchor{gnat_ugn/platform_specific_information creating-an-import-library}@anchor{1df}@anchor{gnat_ugn/platform_specific_information id22}@anchor{1e0}
+@anchor{gnat_ugn/platform_specific_information creating-an-import-library}@anchor{1e1}@anchor{gnat_ugn/platform_specific_information id23}@anchor{1e2}
 @subsubsection Creating an Import Library
 
 
@@ -23518,7 +23538,7 @@ as in this case it is possible to link directly against the
 DLL. Otherwise read on.
 
 @geindex Definition file
-@anchor{gnat_ugn/platform_specific_information the-definition-file}@anchor{1db}
+@anchor{gnat_ugn/platform_specific_information the-definition-file}@anchor{1dd}
 @subsubheading The Definition File
 
 
@@ -23566,17 +23586,17 @@ EXPORTS
 @end table
 
 Note that you must specify the correct suffix (@code{@@@emph{nn}})
-(see @ref{1d0,,Windows Calling Conventions}) for a Stdcall
+(see @ref{1d2,,Windows Calling Conventions}) for a Stdcall
 calling convention function in the exported symbols list.
 
 There can actually be other sections in a definition file, but these
 sections are not relevant to the discussion at hand.
-@anchor{gnat_ugn/platform_specific_information create-def-file-automatically}@anchor{1e1}
+@anchor{gnat_ugn/platform_specific_information create-def-file-automatically}@anchor{1e3}
 @subsubheading Creating a Definition File Automatically
 
 
 You can automatically create the definition file @code{API.def}
-(see @ref{1db,,The Definition File}) from a DLL.
+(see @ref{1dd,,The Definition File}) from a DLL.
 For that use the @code{dlltool} program as follows:
 
 @quotation
@@ -23586,7 +23606,7 @@ $ dlltool API.dll -z API.def --export-all-symbols
 @end example
 
 Note that if some routines in the DLL have the @code{Stdcall} convention
-(@ref{1d0,,Windows Calling Conventions}) with stripped @code{@@@emph{nn}}
+(@ref{1d2,,Windows Calling Conventions}) with stripped @code{@@@emph{nn}}
 suffix then you’ll have to edit @code{api.def} to add it, and specify
 @code{-k} to @code{gnatdll} when creating the import library.
 
@@ -23610,13 +23630,13 @@ tells you what symbol is expected. You just have to go back to the
 definition file and add the right suffix.
 @end itemize
 @end quotation
-@anchor{gnat_ugn/platform_specific_information gnat-style-import-library}@anchor{1e2}
+@anchor{gnat_ugn/platform_specific_information gnat-style-import-library}@anchor{1e4}
 @subsubheading GNAT-Style Import Library
 
 
 To create a static import library from @code{API.dll} with the GNAT tools
 you should create the .def file, then use @code{gnatdll} tool
-(see @ref{1e3,,Using gnatdll}) as follows:
+(see @ref{1e5,,Using gnatdll}) as follows:
 
 @quotation
 
@@ -23632,15 +23652,15 @@ definition file name is @code{xyz.def}, the import library name will
 be @code{libxyz.a}. Note that in the previous example option
 @code{-e} could have been removed because the name of the definition
 file (before the @code{.def} suffix) is the same as the name of the
-DLL (@ref{1e3,,Using gnatdll} for more information about @code{gnatdll}).
+DLL (@ref{1e5,,Using gnatdll} for more information about @code{gnatdll}).
 @end quotation
-@anchor{gnat_ugn/platform_specific_information msvs-style-import-library}@anchor{1e4}
+@anchor{gnat_ugn/platform_specific_information msvs-style-import-library}@anchor{1e6}
 @subsubheading Microsoft-Style Import Library
 
 
 A Microsoft import library is needed only if you plan to make an
 Ada DLL available to applications developed with Microsoft
-tools (@ref{1cc,,Mixed-Language Programming on Windows}).
+tools (@ref{1ce,,Mixed-Language Programming on Windows}).
 
 To create a Microsoft-style import library for @code{API.dll} you
 should create the .def file, then build the actual import library using
@@ -23664,7 +23684,7 @@ See the Microsoft documentation for further details about the usage of
 @end quotation
 
 @node Building DLLs with GNAT Project files,Building DLLs with GNAT,Using DLLs with GNAT,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information building-dlls-with-gnat-project-files}@anchor{1ce}@anchor{gnat_ugn/platform_specific_information id23}@anchor{1e5}
+@anchor{gnat_ugn/platform_specific_information building-dlls-with-gnat-project-files}@anchor{1d0}@anchor{gnat_ugn/platform_specific_information id24}@anchor{1e7}
 @subsubsection Building DLLs with GNAT Project files
 
 
@@ -23680,7 +23700,7 @@ when inside the @code{DllMain} routine which is used for auto-initialization
 of shared libraries, so it is not possible to have library level tasks in SALs.
 
 @node Building DLLs with GNAT,Building DLLs with gnatdll,Building DLLs with GNAT Project files,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information building-dlls-with-gnat}@anchor{1e6}@anchor{gnat_ugn/platform_specific_information id24}@anchor{1e7}
+@anchor{gnat_ugn/platform_specific_information building-dlls-with-gnat}@anchor{1e8}@anchor{gnat_ugn/platform_specific_information id25}@anchor{1e9}
 @subsubsection Building DLLs with GNAT
 
 
@@ -23711,7 +23731,7 @@ $ gcc -shared -shared-libgcc -o api.dll obj1.o obj2.o ...
 It is important to note that in this case all symbols found in the
 object files are automatically exported. It is possible to restrict
 the set of symbols to export by passing to @code{gcc} a definition
-file (see @ref{1db,,The Definition File}).
+file (see @ref{1dd,,The Definition File}).
 For example:
 
 @example
@@ -23749,7 +23769,7 @@ $ gnatmake main -Iapilib -bargs -shared -largs -Lapilib -lAPI
 @end quotation
 
 @node Building DLLs with gnatdll,Ada DLLs and Finalization,Building DLLs with GNAT,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information building-dlls-with-gnatdll}@anchor{1e8}@anchor{gnat_ugn/platform_specific_information id25}@anchor{1e9}
+@anchor{gnat_ugn/platform_specific_information building-dlls-with-gnatdll}@anchor{1ea}@anchor{gnat_ugn/platform_specific_information id26}@anchor{1eb}
 @subsubsection Building DLLs with gnatdll
 
 
@@ -23757,8 +23777,8 @@ $ gnatmake main -Iapilib -bargs -shared -largs -Lapilib -lAPI
 @geindex building
 
 Note that it is preferred to use GNAT Project files
-(@ref{1ce,,Building DLLs with GNAT Project files}) or the built-in GNAT
-DLL support (@ref{1e6,,Building DLLs with GNAT}) or to build DLLs.
+(@ref{1d0,,Building DLLs with GNAT Project files}) or the built-in GNAT
+DLL support (@ref{1e8,,Building DLLs with GNAT}) or to build DLLs.
 
 This section explains how to build DLLs containing Ada code using
 @code{gnatdll}. These DLLs will be referred to as Ada DLLs in the
@@ -23774,20 +23794,20 @@ non-Ada applications are as follows:
 You need to mark each Ada entity exported by the DLL with a @code{C} or
 @code{Stdcall} calling convention to avoid any Ada name mangling for the
 entities exported by the DLL
-(see @ref{1ea,,Exporting Ada Entities}). You can
+(see @ref{1ec,,Exporting Ada Entities}). You can
 skip this step if you plan to use the Ada DLL only from Ada applications.
 
 @item 
 Your Ada code must export an initialization routine which calls the routine
 @code{adainit} generated by @code{gnatbind} to perform the elaboration of
-the Ada code in the DLL (@ref{1eb,,Ada DLLs and Elaboration}). The initialization
+the Ada code in the DLL (@ref{1ed,,Ada DLLs and Elaboration}). The initialization
 routine exported by the Ada DLL must be invoked by the clients of the DLL
 to initialize the DLL.
 
 @item 
 When useful, the DLL should also export a finalization routine which calls
 routine @code{adafinal} generated by @code{gnatbind} to perform the
-finalization of the Ada code in the DLL (@ref{1ec,,Ada DLLs and Finalization}).
+finalization of the Ada code in the DLL (@ref{1ee,,Ada DLLs and Finalization}).
 The finalization routine exported by the Ada DLL must be invoked by the
 clients of the DLL when the DLL services are no further needed.
 
@@ -23797,11 +23817,11 @@ of the programming languages to which you plan to make the DLL available.
 
 @item 
 You must provide a definition file listing the exported entities
-(@ref{1db,,The Definition File}).
+(@ref{1dd,,The Definition File}).
 
 @item 
 Finally you must use @code{gnatdll} to produce the DLL and the import
-library (@ref{1e3,,Using gnatdll}).
+library (@ref{1e5,,Using gnatdll}).
 @end itemize
 
 Note that a relocatable DLL stripped using the @code{strip}
@@ -23821,7 +23841,7 @@ chapter of the @emph{GPRbuild User’s Guide}.
 @end menu
 
 @node Limitations When Using Ada DLLs from Ada,Exporting Ada Entities,,Building DLLs with gnatdll
-@anchor{gnat_ugn/platform_specific_information limitations-when-using-ada-dlls-from-ada}@anchor{1ed}
+@anchor{gnat_ugn/platform_specific_information limitations-when-using-ada-dlls-from-ada}@anchor{1ef}
 @subsubsection Limitations When Using Ada DLLs from Ada
 
 
@@ -23842,7 +23862,7 @@ It is completely safe to exchange plain elementary, array or record types,
 Windows object handles, etc.
 
 @node Exporting Ada Entities,Ada DLLs and Elaboration,Limitations When Using Ada DLLs from Ada,Building DLLs with gnatdll
-@anchor{gnat_ugn/platform_specific_information exporting-ada-entities}@anchor{1ea}@anchor{gnat_ugn/platform_specific_information id26}@anchor{1ee}
+@anchor{gnat_ugn/platform_specific_information exporting-ada-entities}@anchor{1ec}@anchor{gnat_ugn/platform_specific_information id27}@anchor{1f0}
 @subsubsection Exporting Ada Entities
 
 
@@ -23942,10 +23962,10 @@ end API;
 Note that if you do not export the Ada entities with a @code{C} or
 @code{Stdcall} convention you will have to provide the mangled Ada names
 in the definition file of the Ada DLL
-(@ref{1ef,,Creating the Definition File}).
+(@ref{1f1,,Creating the Definition File}).
 
 @node Ada DLLs and Elaboration,,Exporting Ada Entities,Building DLLs with gnatdll
-@anchor{gnat_ugn/platform_specific_information ada-dlls-and-elaboration}@anchor{1eb}@anchor{gnat_ugn/platform_specific_information id27}@anchor{1f0}
+@anchor{gnat_ugn/platform_specific_information ada-dlls-and-elaboration}@anchor{1ed}@anchor{gnat_ugn/platform_specific_information id28}@anchor{1f2}
 @subsubsection Ada DLLs and Elaboration
 
 
@@ -23963,7 +23983,7 @@ the Ada elaboration routine @code{adainit} generated by the GNAT binder
 (@ref{a0,,Binding with Non-Ada Main Programs}). See the body of
 @code{Initialize_Api} for an example. Note that the GNAT binder is
 automatically invoked during the DLL build process by the @code{gnatdll}
-tool (@ref{1e3,,Using gnatdll}).
+tool (@ref{1e5,,Using gnatdll}).
 
 When a DLL is loaded, Windows systematically invokes a routine called
 @code{DllMain}. It would therefore be possible to call @code{adainit}
@@ -23976,7 +23996,7 @@ time), which means that the GNAT run-time will deadlock waiting for the
 newly created task to complete its initialization.
 
 @node Ada DLLs and Finalization,Creating a Spec for Ada DLLs,Building DLLs with gnatdll,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information ada-dlls-and-finalization}@anchor{1ec}@anchor{gnat_ugn/platform_specific_information id28}@anchor{1f1}
+@anchor{gnat_ugn/platform_specific_information ada-dlls-and-finalization}@anchor{1ee}@anchor{gnat_ugn/platform_specific_information id29}@anchor{1f3}
 @subsubsection Ada DLLs and Finalization
 
 
@@ -23991,10 +24011,10 @@ routine @code{adafinal} generated by the GNAT binder
 See the body of @code{Finalize_Api} for an
 example. As already pointed out the GNAT binder is automatically invoked
 during the DLL build process by the @code{gnatdll} tool
-(@ref{1e3,,Using gnatdll}).
+(@ref{1e5,,Using gnatdll}).
 
 @node Creating a Spec for Ada DLLs,GNAT and Windows Resources,Ada DLLs and Finalization,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information creating-a-spec-for-ada-dlls}@anchor{1f2}@anchor{gnat_ugn/platform_specific_information id29}@anchor{1f3}
+@anchor{gnat_ugn/platform_specific_information creating-a-spec-for-ada-dlls}@anchor{1f4}@anchor{gnat_ugn/platform_specific_information id30}@anchor{1f5}
 @subsubsection Creating a Spec for Ada DLLs
 
 
@@ -24052,7 +24072,7 @@ end API;
 @end menu
 
 @node Creating the Definition File,Using gnatdll,,Creating a Spec for Ada DLLs
-@anchor{gnat_ugn/platform_specific_information creating-the-definition-file}@anchor{1ef}@anchor{gnat_ugn/platform_specific_information id30}@anchor{1f4}
+@anchor{gnat_ugn/platform_specific_information creating-the-definition-file}@anchor{1f1}@anchor{gnat_ugn/platform_specific_information id31}@anchor{1f6}
 @subsubsection Creating the Definition File
 
 
@@ -24088,7 +24108,7 @@ EXPORTS
 @end quotation
 
 @node Using gnatdll,,Creating the Definition File,Creating a Spec for Ada DLLs
-@anchor{gnat_ugn/platform_specific_information id31}@anchor{1f5}@anchor{gnat_ugn/platform_specific_information using-gnatdll}@anchor{1e3}
+@anchor{gnat_ugn/platform_specific_information id32}@anchor{1f7}@anchor{gnat_ugn/platform_specific_information using-gnatdll}@anchor{1e5}
 @subsubsection Using @code{gnatdll}
 
 
@@ -24299,7 +24319,7 @@ asks @code{gnatlink} to generate the routines @code{DllMain} and
 is loaded into memory.
 
 @item 
-@code{gnatdll} uses @code{dlltool} (see @ref{1f6,,Using dlltool}) to build the
+@code{gnatdll} uses @code{dlltool} (see @ref{1f8,,Using dlltool}) to build the
 export table (@code{api.exp}). The export table contains the relocation
 information in a form which can be used during the final link to ensure
 that the Windows loader is able to place the DLL anywhere in memory.
@@ -24338,7 +24358,7 @@ $ gnatbind -n api
 $ gnatlink api api.exp -o api.dll -mdll
 @end example
 @end itemize
-@anchor{gnat_ugn/platform_specific_information using-dlltool}@anchor{1f6}
+@anchor{gnat_ugn/platform_specific_information using-dlltool}@anchor{1f8}
 @subsubheading Using @code{dlltool}
 
 
@@ -24397,7 +24417,7 @@ DLL in the static import library generated by @code{dlltool} with switch
 @item @code{-k}
 
 Kill @code{@@@emph{nn}} from exported names
-(@ref{1d0,,Windows Calling Conventions}
+(@ref{1d2,,Windows Calling Conventions}
 for a discussion about @code{Stdcall}-style symbols.
 @end table
 
@@ -24453,7 +24473,7 @@ Use @code{assembler-name} as the assembler. The default is @code{as}.
 @end table
 
 @node GNAT and Windows Resources,Using GNAT DLLs from Microsoft Visual Studio Applications,Creating a Spec for Ada DLLs,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information gnat-and-windows-resources}@anchor{1f7}@anchor{gnat_ugn/platform_specific_information id32}@anchor{1f8}
+@anchor{gnat_ugn/platform_specific_information gnat-and-windows-resources}@anchor{1f9}@anchor{gnat_ugn/platform_specific_information id33}@anchor{1fa}
 @subsubsection GNAT and Windows Resources
 
 
@@ -24548,7 +24568,7 @@ the corresponding Microsoft documentation.
 @end menu
 
 @node Building Resources,Compiling Resources,,GNAT and Windows Resources
-@anchor{gnat_ugn/platform_specific_information building-resources}@anchor{1f9}@anchor{gnat_ugn/platform_specific_information id33}@anchor{1fa}
+@anchor{gnat_ugn/platform_specific_information building-resources}@anchor{1fb}@anchor{gnat_ugn/platform_specific_information id34}@anchor{1fc}
 @subsubsection Building Resources
 
 
@@ -24568,7 +24588,7 @@ complete description of the resource script language can be found in the
 Microsoft documentation.
 
 @node Compiling Resources,Using Resources,Building Resources,GNAT and Windows Resources
-@anchor{gnat_ugn/platform_specific_information compiling-resources}@anchor{1fb}@anchor{gnat_ugn/platform_specific_information id34}@anchor{1fc}
+@anchor{gnat_ugn/platform_specific_information compiling-resources}@anchor{1fd}@anchor{gnat_ugn/platform_specific_information id35}@anchor{1fe}
 @subsubsection Compiling Resources
 
 
@@ -24610,7 +24630,7 @@ $ windres -i myres.res -o myres.o
 @end quotation
 
 @node Using Resources,,Compiling Resources,GNAT and Windows Resources
-@anchor{gnat_ugn/platform_specific_information id35}@anchor{1fd}@anchor{gnat_ugn/platform_specific_information using-resources}@anchor{1fe}
+@anchor{gnat_ugn/platform_specific_information id36}@anchor{1ff}@anchor{gnat_ugn/platform_specific_information using-resources}@anchor{200}
 @subsubsection Using Resources
 
 
@@ -24630,7 +24650,7 @@ $ gnatmake myprog -largs myres.o
 @end quotation
 
 @node Using GNAT DLLs from Microsoft Visual Studio Applications,Debugging a DLL,GNAT and Windows Resources,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information using-gnat-dll-from-msvs}@anchor{1ff}@anchor{gnat_ugn/platform_specific_information using-gnat-dlls-from-microsoft-visual-studio-applications}@anchor{200}
+@anchor{gnat_ugn/platform_specific_information using-gnat-dll-from-msvs}@anchor{201}@anchor{gnat_ugn/platform_specific_information using-gnat-dlls-from-microsoft-visual-studio-applications}@anchor{202}
 @subsubsection Using GNAT DLLs from Microsoft Visual Studio Applications
 
 
@@ -24664,7 +24684,7 @@ $ gprbuild -p mylib.gpr
 @item 
 Produce a .def file for the symbols you need to interface with, either by
 hand or automatically with possibly some manual adjustments
-(see @ref{1e1,,Creating Definition File Automatically}):
+(see @ref{1e3,,Creating Definition File Automatically}):
 @end enumerate
 
 @quotation
@@ -24681,7 +24701,7 @@ $ dlltool libmylib.dll -z libmylib.def --export-all-symbols
 Make sure that MSVS command-line tools are accessible on the path.
 
 @item 
-Create the Microsoft-style import library (see @ref{1e4,,MSVS-Style Import Library}):
+Create the Microsoft-style import library (see @ref{1e6,,MSVS-Style Import Library}):
 @end enumerate
 
 @quotation
@@ -24723,7 +24743,7 @@ or copy the DLL into into the directory containing the .exe.
 @end enumerate
 
 @node Debugging a DLL,Setting Stack Size from gnatlink,Using GNAT DLLs from Microsoft Visual Studio Applications,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information debugging-a-dll}@anchor{201}@anchor{gnat_ugn/platform_specific_information id36}@anchor{202}
+@anchor{gnat_ugn/platform_specific_information debugging-a-dll}@anchor{203}@anchor{gnat_ugn/platform_specific_information id37}@anchor{204}
 @subsubsection Debugging a DLL
 
 
@@ -24761,7 +24781,7 @@ tools suite used to build the DLL.
 @end menu
 
 @node Program and DLL Both Built with GCC/GNAT,Program Built with Foreign Tools and DLL Built with GCC/GNAT,,Debugging a DLL
-@anchor{gnat_ugn/platform_specific_information id37}@anchor{203}@anchor{gnat_ugn/platform_specific_information program-and-dll-both-built-with-gcc-gnat}@anchor{204}
+@anchor{gnat_ugn/platform_specific_information id38}@anchor{205}@anchor{gnat_ugn/platform_specific_information program-and-dll-both-built-with-gcc-gnat}@anchor{206}
 @subsubsection Program and DLL Both Built with GCC/GNAT
 
 
@@ -24771,7 +24791,7 @@ the process. Let’s suppose here that the main procedure is named
 @code{ada_main} and that in the DLL there is an entry point named
 @code{ada_dll}.
 
-The DLL (@ref{1da,,Introduction to Dynamic Link Libraries (DLLs)}) and
+The DLL (@ref{1dc,,Introduction to Dynamic Link Libraries (DLLs)}) and
 program must have been built with the debugging information (see GNAT -g
 switch). Here are the step-by-step instructions for debugging it:
 
@@ -24811,7 +24831,7 @@ you can use the standard approach to debug the whole program
 (@ref{14d,,Running and Debugging Ada Programs}).
 
 @node Program Built with Foreign Tools and DLL Built with GCC/GNAT,,Program and DLL Both Built with GCC/GNAT,Debugging a DLL
-@anchor{gnat_ugn/platform_specific_information id38}@anchor{205}@anchor{gnat_ugn/platform_specific_information program-built-with-foreign-tools-and-dll-built-with-gcc-gnat}@anchor{206}
+@anchor{gnat_ugn/platform_specific_information id39}@anchor{207}@anchor{gnat_ugn/platform_specific_information program-built-with-foreign-tools-and-dll-built-with-gcc-gnat}@anchor{208}
 @subsubsection Program Built with Foreign Tools and DLL Built with GCC/GNAT
 
 
@@ -24828,7 +24848,7 @@ example some C code built with Microsoft Visual C) and that there is a
 DLL named @code{test.dll} containing an Ada entry point named
 @code{ada_dll}.
 
-The DLL (see @ref{1da,,Introduction to Dynamic Link Libraries (DLLs)}) must have
+The DLL (see @ref{1dc,,Introduction to Dynamic Link Libraries (DLLs)}) must have
 been built with debugging information (see the GNAT @code{-g} option).
 
 @subsubheading Debugging the DLL Directly
@@ -24967,7 +24987,7 @@ approach to debug a program as described in
 @ref{14d,,Running and Debugging Ada Programs}.
 
 @node Setting Stack Size from gnatlink,Setting Heap Size from gnatlink,Debugging a DLL,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information id39}@anchor{207}@anchor{gnat_ugn/platform_specific_information setting-stack-size-from-gnatlink}@anchor{127}
+@anchor{gnat_ugn/platform_specific_information id40}@anchor{209}@anchor{gnat_ugn/platform_specific_information setting-stack-size-from-gnatlink}@anchor{127}
 @subsubsection Setting Stack Size from @code{gnatlink}
 
 
@@ -25010,7 +25030,7 @@ because the comma is a separator for this option.
 @end itemize
 
 @node Setting Heap Size from gnatlink,,Setting Stack Size from gnatlink,Mixed-Language Programming on Windows
-@anchor{gnat_ugn/platform_specific_information id40}@anchor{208}@anchor{gnat_ugn/platform_specific_information setting-heap-size-from-gnatlink}@anchor{128}
+@anchor{gnat_ugn/platform_specific_information id41}@anchor{20a}@anchor{gnat_ugn/platform_specific_information setting-heap-size-from-gnatlink}@anchor{128}
 @subsubsection Setting Heap Size from @code{gnatlink}
 
 
@@ -25043,7 +25063,7 @@ because the comma is a separator for this option.
 @end itemize
 
 @node Windows Specific Add-Ons,,Mixed-Language Programming on Windows,Microsoft Windows Topics
-@anchor{gnat_ugn/platform_specific_information win32-specific-addons}@anchor{209}@anchor{gnat_ugn/platform_specific_information windows-specific-add-ons}@anchor{20a}
+@anchor{gnat_ugn/platform_specific_information win32-specific-addons}@anchor{20b}@anchor{gnat_ugn/platform_specific_information windows-specific-add-ons}@anchor{20c}
 @subsection Windows Specific Add-Ons
 
 
@@ -25056,7 +25076,7 @@ This section describes the Windows specific add-ons.
 @end menu
 
 @node Win32Ada,wPOSIX,,Windows Specific Add-Ons
-@anchor{gnat_ugn/platform_specific_information id41}@anchor{20b}@anchor{gnat_ugn/platform_specific_information win32ada}@anchor{20c}
+@anchor{gnat_ugn/platform_specific_information id42}@anchor{20d}@anchor{gnat_ugn/platform_specific_information win32ada}@anchor{20e}
 @subsubsection Win32Ada
 
 
@@ -25087,7 +25107,7 @@ gprbuild p.gpr
 @end quotation
 
 @node wPOSIX,,Win32Ada,Windows Specific Add-Ons
-@anchor{gnat_ugn/platform_specific_information id42}@anchor{20d}@anchor{gnat_ugn/platform_specific_information wposix}@anchor{20e}
+@anchor{gnat_ugn/platform_specific_information id43}@anchor{20f}@anchor{gnat_ugn/platform_specific_information wposix}@anchor{210}
 @subsubsection wPOSIX
 
 
@@ -25120,7 +25140,7 @@ gprbuild p.gpr
 @end quotation
 
 @node Mac OS Topics,,Microsoft Windows Topics,Platform-Specific Information
-@anchor{gnat_ugn/platform_specific_information id43}@anchor{20f}@anchor{gnat_ugn/platform_specific_information mac-os-topics}@anchor{210}
+@anchor{gnat_ugn/platform_specific_information id44}@anchor{211}@anchor{gnat_ugn/platform_specific_information mac-os-topics}@anchor{212}
 @section Mac OS Topics
 
 
@@ -25135,7 +25155,7 @@ platform.
 @end menu
 
 @node Codesigning the Debugger,,,Mac OS Topics
-@anchor{gnat_ugn/platform_specific_information codesigning-the-debugger}@anchor{211}
+@anchor{gnat_ugn/platform_specific_information codesigning-the-debugger}@anchor{213}
 @subsection Codesigning the Debugger
 
 
@@ -25216,7 +25236,7 @@ the location where you installed GNAT.  Also, be sure that users are
 in the Unix group @code{_developer}.
 
 @node Example of Binder Output File,Elaboration Order Handling in GNAT,Platform-Specific Information,Top
-@anchor{gnat_ugn/example_of_binder_output doc}@anchor{212}@anchor{gnat_ugn/example_of_binder_output example-of-binder-output-file}@anchor{e}@anchor{gnat_ugn/example_of_binder_output id1}@anchor{213}
+@anchor{gnat_ugn/example_of_binder_output doc}@anchor{214}@anchor{gnat_ugn/example_of_binder_output example-of-binder-output-file}@anchor{e}@anchor{gnat_ugn/example_of_binder_output id1}@anchor{215}
 @chapter Example of Binder Output File
 
 
@@ -25968,7 +25988,7 @@ elaboration code in your own application).
 @c -- Example: A |withing| unit has a |with| clause, it |withs| a |withed| unit
 
 @node Elaboration Order Handling in GNAT,Inline Assembler,Example of Binder Output File,Top
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat doc}@anchor{214}@anchor{gnat_ugn/elaboration_order_handling_in_gnat elaboration-order-handling-in-gnat}@anchor{f}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id1}@anchor{215}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat doc}@anchor{216}@anchor{gnat_ugn/elaboration_order_handling_in_gnat elaboration-order-handling-in-gnat}@anchor{f}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id1}@anchor{217}
 @chapter Elaboration Order Handling in GNAT
 
 
@@ -25998,7 +26018,7 @@ GNAT, either automatically or with explicit programming features.
 @end menu
 
 @node Elaboration Code,Elaboration Order,,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat elaboration-code}@anchor{216}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id2}@anchor{217}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat elaboration-code}@anchor{218}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id2}@anchor{219}
 @section Elaboration Code
 
 
@@ -26146,7 +26166,7 @@ elaborated.
 @end itemize
 
 @node Elaboration Order,Checking the Elaboration Order,Elaboration Code,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat elaboration-order}@anchor{218}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id3}@anchor{219}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat elaboration-order}@anchor{21a}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id3}@anchor{21b}
 @section Elaboration Order
 
 
@@ -26315,7 +26335,7 @@ however a compiler may not always find such an order due to complications with
 respect to control and data flow.
 
 @node Checking the Elaboration Order,Controlling the Elaboration Order in Ada,Elaboration Order,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat checking-the-elaboration-order}@anchor{21a}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id4}@anchor{21b}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat checking-the-elaboration-order}@anchor{21c}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id4}@anchor{21d}
 @section Checking the Elaboration Order
 
 
@@ -26376,7 +26396,7 @@ order.
 @end itemize
 
 @node Controlling the Elaboration Order in Ada,Controlling the Elaboration Order in GNAT,Checking the Elaboration Order,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat controlling-the-elaboration-order-in-ada}@anchor{21c}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id5}@anchor{21d}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat controlling-the-elaboration-order-in-ada}@anchor{21e}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id5}@anchor{21f}
 @section Controlling the Elaboration Order in Ada
 
 
@@ -26704,7 +26724,7 @@ is that the program continues to stay in the last state (one or more correct
 orders exist) even if maintenance changes the bodies of targets.
 
 @node Controlling the Elaboration Order in GNAT,Mixing Elaboration Models,Controlling the Elaboration Order in Ada,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat controlling-the-elaboration-order-in-gnat}@anchor{21e}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id6}@anchor{21f}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat controlling-the-elaboration-order-in-gnat}@anchor{220}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id6}@anchor{221}
 @section Controlling the Elaboration Order in GNAT
 
 
@@ -26834,7 +26854,7 @@ The dynamic, legacy, and static models can be relaxed using compiler switch
 may not diagnose certain elaboration issues or install run-time checks.
 
 @node Mixing Elaboration Models,ABE Diagnostics,Controlling the Elaboration Order in GNAT,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat id7}@anchor{220}@anchor{gnat_ugn/elaboration_order_handling_in_gnat mixing-elaboration-models}@anchor{221}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat id7}@anchor{222}@anchor{gnat_ugn/elaboration_order_handling_in_gnat mixing-elaboration-models}@anchor{223}
 @section Mixing Elaboration Models
 
 
@@ -26881,7 +26901,7 @@ warning:   "y.ads" which has static elaboration checks
 The warnings can be suppressed by binder switch @code{-ws}.
 
 @node ABE Diagnostics,SPARK Diagnostics,Mixing Elaboration Models,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat abe-diagnostics}@anchor{222}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id8}@anchor{223}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat abe-diagnostics}@anchor{224}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id8}@anchor{225}
 @section ABE Diagnostics
 
 
@@ -26988,7 +27008,7 @@ declaration @code{Safe} because the body of function @code{ABE} has already been
 elaborated at that point.
 
 @node SPARK Diagnostics,Elaboration Circularities,ABE Diagnostics,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat id9}@anchor{224}@anchor{gnat_ugn/elaboration_order_handling_in_gnat spark-diagnostics}@anchor{225}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat id9}@anchor{226}@anchor{gnat_ugn/elaboration_order_handling_in_gnat spark-diagnostics}@anchor{227}
 @section SPARK Diagnostics
 
 
@@ -27014,7 +27034,7 @@ rules.
 @end quotation
 
 @node Elaboration Circularities,Resolving Elaboration Circularities,SPARK Diagnostics,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat elaboration-circularities}@anchor{226}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id10}@anchor{227}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat elaboration-circularities}@anchor{228}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id10}@anchor{229}
 @section Elaboration Circularities
 
 
@@ -27114,7 +27134,7 @@ This section enumerates various tactics for eliminating the circularity.
 @end itemize
 
 @node Resolving Elaboration Circularities,Elaboration-related Compiler Switches,Elaboration Circularities,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat id11}@anchor{228}@anchor{gnat_ugn/elaboration_order_handling_in_gnat resolving-elaboration-circularities}@anchor{229}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat id11}@anchor{22a}@anchor{gnat_ugn/elaboration_order_handling_in_gnat resolving-elaboration-circularities}@anchor{22b}
 @section Resolving Elaboration Circularities
 
 
@@ -27385,7 +27405,7 @@ Use the relaxed dynamic-elaboration model, with compiler switches
 @end itemize
 
 @node Elaboration-related Compiler Switches,Summary of Procedures for Elaboration Control,Resolving Elaboration Circularities,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat elaboration-related-compiler-switches}@anchor{22a}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id12}@anchor{22b}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat elaboration-related-compiler-switches}@anchor{22c}@anchor{gnat_ugn/elaboration_order_handling_in_gnat id12}@anchor{22d}
 @section Elaboration-related Compiler Switches
 
 
@@ -27566,7 +27586,7 @@ checks. The example above will still fail at run time with an ABE.
 @end table
 
 @node Summary of Procedures for Elaboration Control,Inspecting the Chosen Elaboration Order,Elaboration-related Compiler Switches,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat id13}@anchor{22c}@anchor{gnat_ugn/elaboration_order_handling_in_gnat summary-of-procedures-for-elaboration-control}@anchor{22d}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat id13}@anchor{22e}@anchor{gnat_ugn/elaboration_order_handling_in_gnat summary-of-procedures-for-elaboration-control}@anchor{22f}
 @section Summary of Procedures for Elaboration Control
 
 
@@ -27624,7 +27644,7 @@ Use the relaxed dynamic elaboration model, with compiler switches
 @end itemize
 
 @node Inspecting the Chosen Elaboration Order,,Summary of Procedures for Elaboration Control,Elaboration Order Handling in GNAT
-@anchor{gnat_ugn/elaboration_order_handling_in_gnat id14}@anchor{22e}@anchor{gnat_ugn/elaboration_order_handling_in_gnat inspecting-the-chosen-elaboration-order}@anchor{22f}
+@anchor{gnat_ugn/elaboration_order_handling_in_gnat id14}@anchor{230}@anchor{gnat_ugn/elaboration_order_handling_in_gnat inspecting-the-chosen-elaboration-order}@anchor{231}
 @section Inspecting the Chosen Elaboration Order
 
 
@@ -27767,7 +27787,7 @@ gdbstr (body)
 @end quotation
 
 @node Inline Assembler,GNU Free Documentation License,Elaboration Order Handling in GNAT,Top
-@anchor{gnat_ugn/inline_assembler doc}@anchor{230}@anchor{gnat_ugn/inline_assembler id1}@anchor{231}@anchor{gnat_ugn/inline_assembler inline-assembler}@anchor{10}
+@anchor{gnat_ugn/inline_assembler doc}@anchor{232}@anchor{gnat_ugn/inline_assembler id1}@anchor{233}@anchor{gnat_ugn/inline_assembler inline-assembler}@anchor{10}
 @chapter Inline Assembler
 
 
@@ -27826,7 +27846,7 @@ and with assembly language programming.
 @end menu
 
 @node Basic Assembler Syntax,A Simple Example of Inline Assembler,,Inline Assembler
-@anchor{gnat_ugn/inline_assembler basic-assembler-syntax}@anchor{232}@anchor{gnat_ugn/inline_assembler id2}@anchor{233}
+@anchor{gnat_ugn/inline_assembler basic-assembler-syntax}@anchor{234}@anchor{gnat_ugn/inline_assembler id2}@anchor{235}
 @section Basic Assembler Syntax
 
 
@@ -27942,7 +27962,7 @@ Intel: Destination first; for example @code{mov eax, 4}@w{ }
 
 
 @node A Simple Example of Inline Assembler,Output Variables in Inline Assembler,Basic Assembler Syntax,Inline Assembler
-@anchor{gnat_ugn/inline_assembler a-simple-example-of-inline-assembler}@anchor{234}@anchor{gnat_ugn/inline_assembler id3}@anchor{235}
+@anchor{gnat_ugn/inline_assembler a-simple-example-of-inline-assembler}@anchor{236}@anchor{gnat_ugn/inline_assembler id3}@anchor{237}
 @section A Simple Example of Inline Assembler
 
 
@@ -28091,7 +28111,7 @@ If there are no errors, @code{as} will generate an object file
 @code{nothing.out}.
 
 @node Output Variables in Inline Assembler,Input Variables in Inline Assembler,A Simple Example of Inline Assembler,Inline Assembler
-@anchor{gnat_ugn/inline_assembler id4}@anchor{236}@anchor{gnat_ugn/inline_assembler output-variables-in-inline-assembler}@anchor{237}
+@anchor{gnat_ugn/inline_assembler id4}@anchor{238}@anchor{gnat_ugn/inline_assembler output-variables-in-inline-assembler}@anchor{239}
 @section Output Variables in Inline Assembler
 
 
@@ -28458,7 +28478,7 @@ end Get_Flags_3;
 @end quotation
 
 @node Input Variables in Inline Assembler,Inlining Inline Assembler Code,Output Variables in Inline Assembler,Inline Assembler
-@anchor{gnat_ugn/inline_assembler id5}@anchor{238}@anchor{gnat_ugn/inline_assembler input-variables-in-inline-assembler}@anchor{239}
+@anchor{gnat_ugn/inline_assembler id5}@anchor{23a}@anchor{gnat_ugn/inline_assembler input-variables-in-inline-assembler}@anchor{23b}
 @section Input Variables in Inline Assembler
 
 
@@ -28547,7 +28567,7 @@ _increment__incr.1:
 @end quotation
 
 @node Inlining Inline Assembler Code,Other Asm Functionality,Input Variables in Inline Assembler,Inline Assembler
-@anchor{gnat_ugn/inline_assembler id6}@anchor{23a}@anchor{gnat_ugn/inline_assembler inlining-inline-assembler-code}@anchor{23b}
+@anchor{gnat_ugn/inline_assembler id6}@anchor{23c}@anchor{gnat_ugn/inline_assembler inlining-inline-assembler-code}@anchor{23d}
 @section Inlining Inline Assembler Code
 
 
@@ -28618,7 +28638,7 @@ movl %esi,%eax
 thus saving the overhead of stack frame setup and an out-of-line call.
 
 @node Other Asm Functionality,,Inlining Inline Assembler Code,Inline Assembler
-@anchor{gnat_ugn/inline_assembler id7}@anchor{23c}@anchor{gnat_ugn/inline_assembler other-asm-functionality}@anchor{23d}
+@anchor{gnat_ugn/inline_assembler id7}@anchor{23e}@anchor{gnat_ugn/inline_assembler other-asm-functionality}@anchor{23f}
 @section Other @code{Asm} Functionality
 
 
@@ -28633,7 +28653,7 @@ and @code{Volatile}, which inhibits unwanted optimizations.
 @end menu
 
 @node The Clobber Parameter,The Volatile Parameter,,Other Asm Functionality
-@anchor{gnat_ugn/inline_assembler id8}@anchor{23e}@anchor{gnat_ugn/inline_assembler the-clobber-parameter}@anchor{23f}
+@anchor{gnat_ugn/inline_assembler id8}@anchor{240}@anchor{gnat_ugn/inline_assembler the-clobber-parameter}@anchor{241}
 @subsection The @code{Clobber} Parameter
 
 
@@ -28697,7 +28717,7 @@ Use ‘register’ name @code{memory} if you changed a memory location
 @end itemize
 
 @node The Volatile Parameter,,The Clobber Parameter,Other Asm Functionality
-@anchor{gnat_ugn/inline_assembler id9}@anchor{240}@anchor{gnat_ugn/inline_assembler the-volatile-parameter}@anchor{241}
+@anchor{gnat_ugn/inline_assembler id9}@anchor{242}@anchor{gnat_ugn/inline_assembler the-volatile-parameter}@anchor{243}
 @subsection The @code{Volatile} Parameter
 
 
@@ -28733,7 +28753,7 @@ to @code{True} only if the compiler’s optimizations have created
 problems.
 
 @node GNU Free Documentation License,Index,Inline Assembler,Top
-@anchor{share/gnu_free_documentation_license doc}@anchor{242}@anchor{share/gnu_free_documentation_license gnu-fdl}@anchor{1}@anchor{share/gnu_free_documentation_license gnu-free-documentation-license}@anchor{243}
+@anchor{share/gnu_free_documentation_license doc}@anchor{244}@anchor{share/gnu_free_documentation_license gnu-fdl}@anchor{1}@anchor{share/gnu_free_documentation_license gnu-free-documentation-license}@anchor{245}
 @chapter GNU Free Documentation License
 
 
diff --git a/gcc/ada/gnatls.adb b/gcc/ada/gnatls.adb
index 68990e1..b20cda4 100644
--- a/gcc/ada/gnatls.adb
+++ b/gcc/ada/gnatls.adb
@@ -1339,61 +1339,57 @@ procedure Gnatls is
          if Gpr_Prj_Path_File.all /= "" then
             FD := Open_Read (Gpr_Prj_Path_File.all, GNAT.OS_Lib.Text);
 
-            if FD = Invalid_FD then
-               Osint.Fail
-                 ("warning: could not read project path file """
-                  & Gpr_Prj_Path_File.all & """");
-            end if;
+            if FD /= Invalid_FD then
+               Len := Integer (File_Length (FD));
 
-            Len := Integer (File_Length (FD));
+               declare
+                  Buffer : String (1 .. Len);
+                  Index  : Positive := 1;
+                  Last   : Positive;
+                  Tmp    : String_Access;
 
-            declare
-               Buffer : String (1 .. Len);
-               Index  : Positive := 1;
-               Last   : Positive;
-               Tmp    : String_Access;
+               begin
+                  --  Read the file
 
-            begin
-               --  Read the file
+                  Len := Read (FD, Buffer (1)'Address, Len);
+                  Close (FD);
 
-               Len := Read (FD, Buffer (1)'Address, Len);
-               Close (FD);
+                  --  Scan the file line by line
 
-               --  Scan the file line by line
+                  while Index < Buffer'Last loop
 
-               while Index < Buffer'Last loop
+                     --  Find the end of line
 
-                  --  Find the end of line
+                     Last := Index;
+                     while Last <= Buffer'Last
+                       and then Buffer (Last) /= ASCII.LF
+                       and then Buffer (Last) /= ASCII.CR
+                     loop
+                        Last := Last + 1;
+                     end loop;
 
-                  Last := Index;
-                  while Last <= Buffer'Last
-                    and then Buffer (Last) /= ASCII.LF
-                    and then Buffer (Last) /= ASCII.CR
-                  loop
-                     Last := Last + 1;
-                  end loop;
-
-                  --  Ignore empty lines
+                     --  Ignore empty lines
 
-                  if Last > Index then
-                     Tmp := Self;
-                     Self :=
-                       new String'
-                         (Tmp.all & Path_Separator &
-                          Buffer (Index .. Last - 1));
-                     Free (Tmp);
-                  end if;
+                     if Last > Index then
+                        Tmp := Self;
+                        Self :=
+                          new String'
+                            (Tmp.all & Path_Separator &
+                             Buffer (Index .. Last - 1));
+                        Free (Tmp);
+                     end if;
 
-                  --  Find the beginning of the next line
+                     --  Find the beginning of the next line
 
-                  Index := Last;
-                  while Buffer (Index) = ASCII.CR or else
-                        Buffer (Index) = ASCII.LF
-                  loop
-                     Index := Index + 1;
+                     Index := Last;
+                     while Buffer (Index) = ASCII.CR or else
+                           Buffer (Index) = ASCII.LF
+                     loop
+                        Index := Index + 1;
+                     end loop;
                   end loop;
-               end loop;
-            end;
+               end;
+            end if;
 
          end if;
 
diff --git a/gcc/ada/inline.adb b/gcc/ada/inline.adb
index 773b376..08c454d 100644
--- a/gcc/ada/inline.adb
+++ b/gcc/ada/inline.adb
@@ -327,8 +327,8 @@ package body Inline is
    --    Unreferenced
 
    procedure Reset_Actual_Mapping_For_Inlined_Call (Subp : Entity_Id);
-   --  Reset the Renamed_Object flags on the formals of Subp, which can be set
-   --  by a call to Establish_Actual_Mapping_For_Inlined_Call.
+   --  Reset the Renamed_Object field to Empty on all formals of Subp, which
+   --  can be set by a call to Establish_Actual_Mapping_For_Inlined_Call.
 
    ------------------------------
    -- Deferred Cleanup Actions --
@@ -2894,7 +2894,7 @@ package body Inline is
          if Present (Renamed_Object (F)) then
 
             --  If expander is active, it is an error to try to inline a
-            --  recursive program. In GNATprove mode, just indicate that the
+            --  recursive subprogram. In GNATprove mode, just indicate that the
             --  inlining will not happen, and mark the subprogram as not always
             --  inlined.
 
diff --git a/gcc/ada/lib-xref.adb b/gcc/ada/lib-xref.adb
index 17de886..2c3c372 100644
--- a/gcc/ada/lib-xref.adb
+++ b/gcc/ada/lib-xref.adb
@@ -481,7 +481,9 @@ package body Lib.Xref is
                            --  e.g. function call, slicing of a function call,
                            --  pointer dereference, etc.
 
-                           if No (Obj) then
+                           if No (Obj)
+                             or else Ekind (Obj) = E_Enumeration_Literal
+                           then
                               return Empty;
                            end if;
                         else
diff --git a/gcc/ada/libgnarl/s-tasini.adb b/gcc/ada/libgnarl/s-tasini.adb
index 9acfa49..8d242ee 100644
--- a/gcc/ada/libgnarl/s-tasini.adb
+++ b/gcc/ada/libgnarl/s-tasini.adb
@@ -126,6 +126,7 @@ package body System.Tasking.Initialization is
       if Self_ID /= null
         and then Self_ID.Deferral_Level = 0
         and then Self_ID.Pending_ATC_Level < Self_ID.ATC_Nesting_Level
+        and then not Self_ID.Aborting
       then
          return 1;
       else
diff --git a/gcc/ada/libgnarl/s-tasini.ads b/gcc/ada/libgnarl/s-tasini.ads
index f35df576..a551d24 100644
--- a/gcc/ada/libgnarl/s-tasini.ads
+++ b/gcc/ada/libgnarl/s-tasini.ads
@@ -115,7 +115,7 @@ package System.Tasking.Initialization is
 
    function Check_Abort_Status return Integer;
    --  Returns Boolean'Pos (True) iff abort signal should raise
-   --  Standard'Abort_Signal. Only used by IRIX currently.
+   --  Standard'Abort_Signal. Only used by RTEMS currently.
 
    --------------------------
    -- Change Base Priority --
diff --git a/gcc/ada/libgnarl/s-taskin.adb b/gcc/ada/libgnarl/s-taskin.adb
index ab50af1..1e6f42d 100644
--- a/gcc/ada/libgnarl/s-taskin.adb
+++ b/gcc/ada/libgnarl/s-taskin.adb
@@ -127,7 +127,7 @@ package body System.Tasking is
       end if;
       pragma Assert (T.Common.Domain /= null);
 
-      T.Common.Current_Priority         := 0;
+      T.Common.Current_Priority         := Priority'First;
       T.Common.Protected_Action_Nesting := 0;
       T.Common.Call                     := null;
       T.Common.Task_Arg                 := Task_Arg;
diff --git a/gcc/ada/libgnarl/s-taskin.ads b/gcc/ada/libgnarl/s-taskin.ads
index cf560b5..1751553 100644
--- a/gcc/ada/libgnarl/s-taskin.ads
+++ b/gcc/ada/libgnarl/s-taskin.ads
@@ -773,7 +773,10 @@ package System.Tasking is
    -- Priority info --
    -------------------
 
-   Unspecified_Priority : constant Integer := System.Priority'First - 1;
+   Unspecified_Priority : constant Integer := -1;
+   --  Indicates that a task has an unspecified priority. This is hardcoded as
+   --  -1 rather than System.Priority'First - 1 as the value needs to be used
+   --  in init.c to specify that the main task has no specified priority.
 
    Priority_Not_Boosted : constant Integer := System.Priority'First - 1;
    --  Definition of Priority actually has to come from the RTS configuration
diff --git a/gcc/ada/libgnat/a-except.ads b/gcc/ada/libgnat/a-except.ads
index 1608e79..b6c8bb5 100644
--- a/gcc/ada/libgnat/a-except.ads
+++ b/gcc/ada/libgnat/a-except.ads
@@ -184,14 +184,9 @@ private
    --  Raise_Exception_Always if it can determine this is the case. The Export
    --  allows this routine to be accessed from Pure units.
 
-   --  Make these callable from strub contexts.
    pragma Machine_Attribute (Raise_Exception_Always,
                              "strub", "callable");
-   pragma Machine_Attribute (Raise_Exception,
-                             "strub", "callable");
-   --  This property should arguably be visible to callers, but let's
-   --  keep it private for now.  In practice, it doesn't matter, since
-   --  it's only checked in the back end.
+   --  Make it callable from strub contexts
 
    procedure Raise_From_Controlled_Operation (X : Exception_Occurrence);
    pragma No_Return (Raise_From_Controlled_Operation);
diff --git a/gcc/ada/libgnat/a-nbnbin__ghost.adb b/gcc/ada/libgnat/a-nbnbig.adb
index 7d22086..d04d2a4 100644
--- a/gcc/ada/libgnat/a-nbnbin__ghost.adb
+++ b/gcc/ada/libgnat/a-nbnbig.adb
@@ -2,7 +2,7 @@
 --                                                                          --
 --                         GNAT RUN-TIME COMPONENTS                         --
 --                                                                          --
---                  ADA.NUMERICS.BIG_NUMBERS.BIG_INTEGERS                   --
+--               ADA.NUMERICS.BIG_NUMBERS.BIG_INTEGERS_GHOST                --
 --                                                                          --
 --                                 B o d y                                  --
 --                                                                          --
@@ -33,7 +33,12 @@
 --  currently does not compile instantiations of the spec with imported ghost
 --  generics for packages Signed_Conversions and Unsigned_Conversions.
 
-package body Ada.Numerics.Big_Numbers.Big_Integers with
+--  Ghost code in this unit is meant for analysis only, not for run-time
+--  checking. This is enforced by setting the assertion policy to Ignore.
+
+pragma Assertion_Policy (Ghost => Ignore);
+
+package body Ada.Numerics.Big_Numbers.Big_Integers_Ghost with
    SPARK_Mode => Off
 is
 
@@ -73,4 +78,4 @@ is
 
    end Unsigned_Conversions;
 
-end Ada.Numerics.Big_Numbers.Big_Integers;
+end Ada.Numerics.Big_Numbers.Big_Integers_Ghost;
diff --git a/gcc/ada/libgnat/a-nbnbin__ghost.ads b/gcc/ada/libgnat/a-nbnbig.ads
index 3663dd7..f574e78 100644
--- a/gcc/ada/libgnat/a-nbnbin__ghost.ads
+++ b/gcc/ada/libgnat/a-nbnbig.ads
@@ -2,7 +2,7 @@
 --                                                                          --
 --                         GNAT RUN-TIME COMPONENTS                         --
 --                                                                          --
---                  ADA.NUMERICS.BIG_NUMBERS.BIG_INTEGERS                   --
+--               ADA.NUMERICS.BIG_NUMBERS.BIG_INTEGERS_GHOST                --
 --                                                                          --
 --                                 S p e c                                  --
 --                                                                          --
@@ -13,10 +13,24 @@
 --                                                                          --
 ------------------------------------------------------------------------------
 
-package Ada.Numerics.Big_Numbers.Big_Integers with
+--  This unit is provided as a replacement for the standard unit
+--  Ada.Numerics.Big_Numbers.Big_Integers when only proof with SPARK is
+--  intended. It cannot be used for execution, as all subprograms are marked
+--  imported with no definition.
+
+--  Contrary to Ada.Numerics.Big_Numbers.Big_Integers, this unit does not
+--  depend on System or Ada.Finalization, which makes it more convenient for
+--  use in run-time units.
+
+--  Ghost code in this unit is meant for analysis only, not for run-time
+--  checking. This is enforced by setting the assertion policy to Ignore.
+
+pragma Assertion_Policy (Ghost => Ignore);
+
+package Ada.Numerics.Big_Numbers.Big_Integers_Ghost with
    SPARK_Mode,
    Ghost,
-   Preelaborate
+   Pure
 is
    type Big_Integer is private
      with Integer_Literal => From_Universal_Image;
@@ -199,4 +213,4 @@ private
 
    type Big_Integer is null record;
 
-end Ada.Numerics.Big_Numbers.Big_Integers;
+end Ada.Numerics.Big_Numbers.Big_Integers_Ghost;
diff --git a/gcc/ada/libgnat/a-nbnbin__gmp.adb b/gcc/ada/libgnat/a-nbnbin__gmp.adb
index 880e9a3..1516f49 100644
--- a/gcc/ada/libgnat/a-nbnbin__gmp.adb
+++ b/gcc/ada/libgnat/a-nbnbin__gmp.adb
@@ -327,7 +327,7 @@ package body Ada.Numerics.Big_Numbers.Big_Integers is
    -- From_String --
    -----------------
 
-   function From_String (Arg : String) return Big_Integer is
+   function From_String (Arg : String) return Valid_Big_Integer is
       function mpz_set_str
         (this : access mpz_t;
          str  : System.Address;
diff --git a/gcc/ada/libgnat/a-strsup.ads b/gcc/ada/libgnat/a-strsup.ads
index 7428e9c..ae4339f 100644
--- a/gcc/ada/libgnat/a-strsup.ads
+++ b/gcc/ada/libgnat/a-strsup.ads
@@ -76,7 +76,8 @@ package Ada.Strings.Superbounded with SPARK_Mode is
    --  that they can be renamed in Ada.Strings.Bounded.Generic_Bounded_Length.
 
    function Super_Length (Source : Super_String) return Natural
-   is (Source.Current_Length);
+   is (Source.Current_Length)
+   with Global => null;
 
    --------------------------------------------------------
    -- Conversion, Concatenation, and Selection Functions --
@@ -620,7 +621,8 @@ package Ada.Strings.Superbounded with SPARK_Mode is
    is (if Index <= Source.Current_Length
        then Source.Data (Index)
        else raise Index_Error)
-   with Pre => Index <= Super_Length (Source);
+   with Pre    => Index <= Super_Length (Source),
+        Global => null;
 
    procedure Super_Replace_Element
      (Source : in out Super_String;
@@ -649,8 +651,9 @@ package Ada.Strings.Superbounded with SPARK_Mode is
           --  get the null string in accordance with normal Ada slice rules.
 
           String (Source.Data (Low .. High)))
-   with Pre => Low - 1 <= Super_Length (Source)
-                 and then High <= Super_Length (Source);
+   with Pre    => Low - 1 <= Super_Length (Source)
+                    and then High <= Super_Length (Source),
+        Global => null;
 
    function Super_Slice
      (Source : Super_String;
diff --git a/gcc/ada/libgnat/a-strunb.adb b/gcc/ada/libgnat/a-strunb.adb
index 4727f965..b0e455b 100644
--- a/gcc/ada/libgnat/a-strunb.adb
+++ b/gcc/ada/libgnat/a-strunb.adb
@@ -505,8 +505,14 @@ package body Ada.Strings.Unbounded is
       --  Note: Don't try to free statically allocated null string
 
       if Object.Reference /= Null_String'Access then
-         Deallocate (Object.Reference);
-         Object.Reference := Null_Unbounded_String.Reference;
+         declare
+            Old : String_Access := Object.Reference;
+            --  The original reference cannot be null, so we must create a
+            --  copy which will become null when deallocated.
+         begin
+            Deallocate (Old);
+            Object.Reference := Null_Unbounded_String.Reference;
+         end;
          Object.Last := 0;
       end if;
    end Finalize;
@@ -827,9 +833,13 @@ package body Ada.Strings.Unbounded is
             Tmp : constant String_Access :=
               new String (1 .. New_Rounded_Up_Size);
 
+            Old : String_Access := Source.Reference;
+            --  The original reference cannot be null, so we must create a copy
+            --  which will become null when deallocated.
+
          begin
             Tmp (1 .. Source.Last) := Source.Reference (1 .. Source.Last);
-            Free (Source.Reference);
+            Free (Old);
             Source.Reference := Tmp;
          end;
       end if;
diff --git a/gcc/ada/libgnat/a-strunb.ads b/gcc/ada/libgnat/a-strunb.ads
index b3050fd..2f5bd94 100644
--- a/gcc/ada/libgnat/a-strunb.ads
+++ b/gcc/ada/libgnat/a-strunb.ads
@@ -746,8 +746,8 @@ private
      renames To_Unbounded_String;
 
    type Unbounded_String is new AF.Controlled with record
-      Reference : String_Access := Null_String'Access;
-      Last      : Natural       := 0;
+      Reference : not null String_Access := Null_String'Access;
+      Last      : Natural                := 0;
    end record with Put_Image => Put_Image;
 
    procedure Put_Image
diff --git a/gcc/ada/libgnat/a-strunb__shared.ads b/gcc/ada/libgnat/a-strunb__shared.ads
index 2cf6780..34430a7 100644
--- a/gcc/ada/libgnat/a-strunb__shared.ads
+++ b/gcc/ada/libgnat/a-strunb__shared.ads
@@ -45,6 +45,8 @@ pragma Assertion_Policy (Pre => Ignore);
 
 --  This version is supported on:
 --    - all Alpha platforms
+--    - all AARCH64 platforms
+--    - all ARM platforms
 --    - all ia64 platforms
 --    - all PowerPC platforms
 --    - all SPARC V9 platforms
diff --git a/gcc/ada/libgnat/s-aridou.adb b/gcc/ada/libgnat/s-aridou.adb
index b47c319..67f2440 100644
--- a/gcc/ada/libgnat/s-aridou.adb
+++ b/gcc/ada/libgnat/s-aridou.adb
@@ -31,7 +31,20 @@
 
 with Ada.Unchecked_Conversion;
 
-package body System.Arith_Double is
+package body System.Arith_Double
+  with SPARK_Mode
+is
+   --  Contracts, ghost code, loop invariants and assertions in this unit are
+   --  meant for analysis only, not for run-time checking, as it would be too
+   --  costly otherwise. This is enforced by setting the assertion policy to
+   --  Ignore.
+
+   pragma Assertion_Policy (Pre            => Ignore,
+                            Post           => Ignore,
+                            Contract_Cases => Ignore,
+                            Ghost          => Ignore,
+                            Loop_Invariant => Ignore,
+                            Assert         => Ignore);
 
    pragma Suppress (Overflow_Check);
    pragma Suppress (Range_Check);
@@ -42,6 +55,30 @@ package body System.Arith_Double is
    Double_Size : constant Natural := Double_Int'Size;
    Single_Size : constant Natural := Double_Int'Size / 2;
 
+   --  Power-of-two constants. Use the names Big_2xx32, Big_2xx63 and Big_2xx64
+   --  even if Single_Size might not be 32 and Double_Size might not be 64, as
+   --  this facilitates code and proof understanding, compared to more generic
+   --  names.
+
+   pragma Warnings
+     (Off, "non-preelaborable call not allowed in preelaborated unit",
+      Reason => "Ghost code is not compiled");
+   pragma Warnings
+     (Off, "non-static constant in preelaborated unit",
+      Reason => "Ghost code is not compiled");
+   Big_2xx32 : constant Big_Integer :=
+     Big (Double_Int'(2 ** Single_Size))
+   with Ghost;
+   Big_2xx63 : constant Big_Integer :=
+     Big (Double_Uns'(2 ** (Double_Size - 1)))
+   with Ghost;
+   Big_2xx64 : constant Big_Integer :=
+     Big (Double_Uns'(2 ** Double_Size - 1)) + 1
+   with Ghost;
+   pragma Warnings
+     (On, "non-preelaborable call not allowed in preelaborated unit");
+   pragma Warnings (On, "non-static constant in preelaborated unit");
+
    -----------------------
    -- Local Subprograms --
    -----------------------
@@ -57,7 +94,9 @@ package body System.Arith_Double is
    --  Length doubling multiplication
 
    function "/" (A : Double_Uns; B : Single_Uns) return Double_Uns is
-     (A / Double_Uns (B));
+     (A / Double_Uns (B))
+   with
+     Pre => B /= 0;
    --  Length doubling division
 
    function "&" (Hi, Lo : Single_Uns) return Double_Uns is
@@ -66,16 +105,34 @@ package body System.Arith_Double is
 
    function "abs" (X : Double_Int) return Double_Uns is
      (if X = Double_Int'First
-      then 2 ** (Double_Size - 1)
+      then Double_Uns'(2 ** (Double_Size - 1))
       else Double_Uns (Double_Int'(abs X)));
    --  Convert absolute value of X to unsigned. Note that we can't just use
    --  the expression of the Else since it overflows for X = Double_Int'First.
 
    function "rem" (A : Double_Uns; B : Single_Uns) return Double_Uns is
-     (A rem Double_Uns (B));
+     (A rem Double_Uns (B))
+   with
+     Pre => B /= 0;
    --  Length doubling remainder
 
-   function Le3 (X1, X2, X3, Y1, Y2, Y3 : Single_Uns) return Boolean;
+   function Big_2xx (N : Natural) return Big_Integer is
+     (Big (Double_Uns'(2 ** N)))
+   with
+     Ghost,
+     Pre  => N < Double_Size,
+     Post => Big_2xx'Result > 0;
+   --  2**N as a big integer
+
+   function Big3 (X1, X2, X3 : Single_Uns) return Big_Integer is
+     (Big_2xx32 * Big_2xx32 * Big (Double_Uns (X1))
+                + Big_2xx32 * Big (Double_Uns (X2)) + Big (Double_Uns (X3)))
+   with Ghost;
+   --  X1&X2&X3 as a big integer
+
+   function Le3 (X1, X2, X3, Y1, Y2, Y3 : Single_Uns) return Boolean
+   with
+     Post => Le3'Result = (Big3 (X1, X2, X3) <= Big3 (Y1, Y2, Y3));
    --  Determines if (3 * Single_Size)-bit value X1&X2&X3 <= Y1&Y2&Y3
 
    function Lo (A : Double_Uns) return Single_Uns is
@@ -86,16 +143,27 @@ package body System.Arith_Double is
      (Single_Uns (Shift_Right (A, Single_Size)));
    --  High order half of double value
 
-   procedure Sub3 (X1, X2, X3 : in out Single_Uns; Y1, Y2, Y3 : Single_Uns);
+   procedure Sub3 (X1, X2, X3 : in out Single_Uns; Y1, Y2, Y3 : Single_Uns)
+   with
+     Pre  => Big3 (X1, X2, X3) >= Big3 (Y1, Y2, Y3),
+     Post => Big3 (X1, X2, X3) = Big3 (X1, X2, X3)'Old - Big3 (Y1, Y2, Y3);
    --  Computes X1&X2&X3 := X1&X2&X3 - Y1&Y1&Y3 mod 2 ** (3 * Single_Size)
 
-   function To_Neg_Int (A : Double_Uns) return Double_Int;
+   function To_Neg_Int (A : Double_Uns) return Double_Int
+   with
+     Annotate => (GNATprove, Terminating),
+     Pre      => In_Double_Int_Range (-Big (A)),
+     Post     => Big (To_Neg_Int'Result) = -Big (A);
    --  Convert to negative integer equivalent. If the input is in the range
    --  0 .. 2 ** (Double_Size - 1), then the corresponding nonpositive signed
    --  integer (obtained by negating the given value) is returned, otherwise
    --  constraint error is raised.
 
-   function To_Pos_Int (A : Double_Uns) return Double_Int;
+   function To_Pos_Int (A : Double_Uns) return Double_Int
+   with
+     Annotate => (GNATprove, Terminating),
+     Pre      => In_Double_Int_Range (Big (A)),
+     Post     => Big (To_Pos_Int'Result) = Big (A);
    --  Convert to positive integer equivalent. If the input is in the range
    --  0 .. 2 ** (Double_Size - 1) - 1, then the corresponding non-negative
    --  signed integer is returned, otherwise constraint error is raised.
@@ -104,6 +172,396 @@ package body System.Arith_Double is
    pragma No_Return (Raise_Error);
    --  Raise constraint error with appropriate message
 
+   ------------------
+   -- Local Lemmas --
+   ------------------
+
+   procedure Inline_Le3 (X1, X2, X3, Y1, Y2, Y3 : Single_Uns)
+   with
+     Ghost,
+     Pre  => Le3 (X1, X2, X3, Y1, Y2, Y3),
+     Post => Big3 (X1, X2, X3) <= Big3 (Y1, Y2, Y3);
+
+   procedure Lemma_Abs_Commutation (X : Double_Int)
+   with
+     Ghost,
+     Post => abs (Big (X)) = Big (Double_Uns'(abs X));
+
+   procedure Lemma_Abs_Div_Commutation (X, Y : Big_Integer)
+   with
+     Ghost,
+     Pre  => Y /= 0,
+     Post => abs (X / Y) = abs X / abs Y;
+
+   procedure Lemma_Abs_Mult_Commutation (X, Y : Big_Integer)
+   with
+     Ghost,
+     Post => abs (X * Y) = abs X * abs Y;
+
+   procedure Lemma_Abs_Rem_Commutation (X, Y : Big_Integer)
+   with
+     Ghost,
+     Pre  => Y /= 0,
+     Post => abs (X rem Y) = (abs X) rem (abs Y);
+
+   procedure Lemma_Add_Commutation (X : Double_Uns; Y : Single_Uns)
+   with
+     Ghost,
+     Pre  => X <= 2 ** Double_Size - 2 ** Single_Size,
+     Post => Big (X) + Big (Double_Uns (Y)) = Big (X + Double_Uns (Y));
+
+   procedure Lemma_Add_One (X : Double_Uns)
+   with
+     Ghost,
+     Pre  => X /= Double_Uns'Last,
+     Post => Big (X + Double_Uns'(1)) = Big (X) + 1;
+
+   procedure Lemma_Bounded_Powers_Of_2_Increasing (M, N : Natural)
+   with
+     Ghost,
+     Pre  => M < N and then N < Double_Size,
+     Post => Double_Uns'(2)**M < Double_Uns'(2)**N;
+
+   procedure Lemma_Deep_Mult_Commutation
+     (Factor : Big_Integer;
+      X, Y   : Single_Uns)
+   with
+     Ghost,
+     Post =>
+       Factor * Big (Double_Uns (X)) * Big (Double_Uns (Y)) =
+         Factor * Big (Double_Uns (X) * Double_Uns (Y));
+
+   procedure Lemma_Div_Commutation (X, Y : Double_Uns)
+   with
+     Ghost,
+     Pre  => Y /= 0,
+     Post => Big (X) / Big (Y) = Big (X / Y);
+
+   procedure Lemma_Div_Definition
+     (A : Double_Uns;
+      B : Single_Uns;
+      Q : Double_Uns;
+      R : Double_Uns)
+   with
+     Ghost,
+     Pre  => B /= 0 and then Q = A / B and then R = A rem B,
+     Post => Big (A) = Big (Double_Uns (B)) * Big (Q) + Big (R);
+
+   procedure Lemma_Div_Ge (X, Y, Z : Big_Integer)
+   with
+     Ghost,
+     Pre  => Z > 0 and then X >= Y * Z,
+     Post => X / Z >= Y;
+
+   procedure Lemma_Div_Lt (X, Y, Z : Big_Natural)
+   with
+     Ghost,
+     Pre  => Z > 0 and then X < Y * Z,
+     Post => X / Z < Y;
+
+   procedure Lemma_Div_Eq (A, B, S, R : Big_Integer)
+   with
+     Ghost,
+     Pre  => A * S = B * S + R and then S /= 0,
+     Post => A = B + R / S;
+
+   procedure Lemma_Double_Shift (X : Double_Uns; S, S1 : Double_Uns)
+   with
+     Ghost,
+     Pre  => S <= Double_Uns (Double_Size)
+       and then S1 <= Double_Uns (Double_Size),
+     Post => Shift_Left (Shift_Left (X, Natural (S)), Natural (S1)) =
+             Shift_Left (X, Natural (S + S1));
+
+   procedure Lemma_Double_Shift (X : Single_Uns; S, S1 : Natural)
+   with
+     Ghost,
+     Pre  => S <= Single_Size - S1,
+     Post => Shift_Left (Shift_Left (X, S), S1) = Shift_Left (X, S + S1);
+
+   procedure Lemma_Double_Shift (X : Double_Uns; S, S1 : Natural)
+   with
+     Ghost,
+     Pre  => S <= Double_Size - S1,
+     Post => Shift_Left (Shift_Left (X, S), S1) = Shift_Left (X, S + S1);
+
+   procedure Lemma_Double_Shift_Right (X : Double_Uns; S, S1 : Double_Uns)
+   with
+     Ghost,
+     Pre  => S <= Double_Uns (Double_Size)
+       and then S1 <= Double_Uns (Double_Size),
+     Post => Shift_Right (Shift_Right (X, Natural (S)), Natural (S1)) =
+             Shift_Right (X, Natural (S + S1));
+
+   procedure Lemma_Double_Shift_Right (X : Double_Uns; S, S1 : Natural)
+   with
+     Ghost,
+     Pre  => S <= Double_Size - S1,
+     Post => Shift_Right (Shift_Right (X, S), S1) = Shift_Right (X, S + S1);
+
+   procedure Lemma_Ge_Commutation (A, B : Double_Uns)
+   with
+     Ghost,
+     Pre  => A >= B,
+     Post => Big (A) >= Big (B);
+
+   procedure Lemma_Ge_Mult (A, B, C, D : Big_Integer)
+   with
+     Ghost,
+     Pre  => A >= B and then B * C >= D and then C > 0,
+     Post => A * C >= D;
+
+   procedure Lemma_Gt_Commutation (A, B : Double_Uns)
+   with
+     Ghost,
+     Pre  => A > B,
+     Post => Big (A) > Big (B);
+
+   procedure Lemma_Gt_Mult (A, B, C, D : Big_Integer)
+   with
+     Ghost,
+     Pre  => A >= B and then B * C > D and then C > 0,
+     Post => A * C > D;
+
+   procedure Lemma_Hi_Lo (Xu : Double_Uns; Xhi, Xlo : Single_Uns)
+   with
+     Ghost,
+     Pre  => Xhi = Hi (Xu) and Xlo = Lo (Xu),
+     Post =>
+       Big (Xu) = Big_2xx32 * Big (Double_Uns (Xhi)) + Big (Double_Uns (Xlo));
+
+   procedure Lemma_Hi_Lo_3 (Xu : Double_Uns; Xhi, Xlo : Single_Uns)
+   with
+     Ghost,
+     Pre  => Xhi = Hi (Xu) and then Xlo = Lo (Xu),
+     Post => Big (Xu) = Big3 (0, Xhi, Xlo);
+
+   procedure Lemma_Lo_Is_Ident (X : Double_Uns)
+   with
+     Ghost,
+     Pre  => Big (X) < Big_2xx32,
+     Post => Double_Uns (Lo (X)) = X;
+
+   procedure Lemma_Lt_Commutation (A, B : Double_Uns)
+   with
+     Ghost,
+     Pre  => A < B,
+     Post => Big (A) < Big (B);
+
+   procedure Lemma_Lt_Mult (A, B, C, D : Big_Integer)
+   with
+     Ghost,
+     Pre  => A < B and then B * C <= D and then C > 0,
+     Post => A * C < D;
+
+   procedure Lemma_Mult_Commutation (X, Y : Single_Uns)
+   with
+     Ghost,
+     Post =>
+       Big (Double_Uns (X)) * Big (Double_Uns (Y)) =
+         Big (Double_Uns (X) * Double_Uns (Y));
+
+   procedure Lemma_Mult_Commutation (X, Y : Double_Int)
+   with
+     Ghost,
+     Pre  => In_Double_Int_Range (Big (X) * Big (Y)),
+     Post => Big (X) * Big (Y) = Big (X * Y);
+
+   procedure Lemma_Mult_Commutation (X, Y, Z : Double_Uns)
+   with
+     Ghost,
+     Pre  => Big (X) * Big (Y) < Big_2xx64 and then Z = X * Y,
+     Post => Big (X) * Big (Y) = Big (Z);
+
+   procedure Lemma_Mult_Decomposition
+     (Mult               : Big_Integer;
+      Xu, Yu             : Double_Uns;
+      Xhi, Xlo, Yhi, Ylo : Single_Uns)
+   with
+     Ghost,
+     Pre  => Mult = Big (Xu) * Big (Yu)
+       and then Xhi = Hi (Xu)
+       and then Xlo = Lo (Xu)
+       and then Yhi = Hi (Yu)
+       and then Ylo = Lo (Yu),
+     Post => Mult =
+               Big_2xx32 * Big_2xx32 * (Big (Double_Uns'(Xhi * Yhi)))
+                         + Big_2xx32 * (Big (Double_Uns'(Xhi * Ylo)))
+                         + Big_2xx32 * (Big (Double_Uns'(Xlo * Yhi)))
+                                     + (Big (Double_Uns'(Xlo * Ylo)));
+
+   procedure Lemma_Mult_Distribution (X, Y, Z : Big_Integer)
+   with
+     Ghost,
+     Post => X * (Y + Z) = X * Y + X * Z;
+
+   procedure Lemma_Neg_Div (X, Y : Big_Integer)
+   with
+     Ghost,
+     Pre  => Y /= 0,
+     Post => X / Y = (-X) / (-Y);
+
+   procedure Lemma_Neg_Rem (X, Y : Big_Integer)
+   with
+     Ghost,
+     Pre  => Y /= 0,
+     Post => X rem Y = X rem (-Y);
+
+   procedure Lemma_Powers_Of_2 (M, N : Natural)
+   with
+     Ghost,
+     Pre  => M < Double_Size
+       and then N < Double_Size
+       and then M + N <= Double_Size,
+     Post =>
+       Big_2xx (M) * Big_2xx (N) =
+         (if M + N = Double_Size then Big_2xx64 else Big_2xx (M + N));
+
+   procedure Lemma_Powers_Of_2_Commutation (M : Natural)
+   with
+     Ghost,
+     Subprogram_Variant => (Decreases => M),
+     Pre  => M <= Double_Size,
+     Post => Big (Double_Uns'(2))**M =
+              (if M < Double_Size then Big_2xx (M) else Big_2xx64);
+
+   procedure Lemma_Powers_Of_2_Increasing (M, N : Natural)
+   with
+     Ghost,
+     Subprogram_Variant => (Increases => M),
+     Pre  => M < N,
+     Post => Big (Double_Uns'(2))**M < Big (Double_Uns'(2))**N;
+
+   procedure Lemma_Rem_Abs (X, Y : Big_Integer)
+   with
+     Ghost,
+     Pre  => Y /= 0,
+     Post => X rem Y = X rem (abs Y);
+   pragma Unreferenced (Lemma_Rem_Abs);
+
+   procedure Lemma_Rem_Commutation (X, Y : Double_Uns)
+   with
+     Ghost,
+     Pre  => Y /= 0,
+     Post => Big (X) rem Big (Y) = Big (X rem Y);
+
+   procedure Lemma_Rem_Is_Ident (X, Y : Big_Integer)
+   with
+     Ghost,
+     Pre  => abs X < abs Y,
+     Post => X rem Y = X;
+   pragma Unreferenced (Lemma_Rem_Is_Ident);
+
+   procedure Lemma_Rem_Sign (X, Y : Big_Integer)
+   with
+     Ghost,
+     Pre  => Y /= 0,
+     Post => Same_Sign (X rem Y, X);
+   pragma Unreferenced (Lemma_Rem_Sign);
+
+   procedure Lemma_Rev_Div_Definition (A, B, Q, R : Big_Natural)
+   with
+     Ghost,
+     Pre  => A = B * Q + R and then R < B,
+     Post => Q = A / B and then R = A rem B;
+
+   procedure Lemma_Shift_Right (X : Double_Uns; Shift : Natural)
+   with
+     Ghost,
+     Pre  => Shift < Double_Size,
+     Post => Big (Shift_Right (X, Shift)) = Big (X) / Big_2xx (Shift);
+
+   procedure Lemma_Shift_Without_Drop
+     (X, Y  : Double_Uns;
+      Mask  : Single_Uns;
+      Shift : Natural)
+   with
+     Ghost,
+     Pre  => (Hi (X) and Mask) = 0  --  X has the first Shift bits off
+       and then Shift <= Single_Size
+       and then Mask = Shift_Left (Single_Uns'Last, Single_Size - Shift)
+       and then Y = Shift_Left (X, Shift),
+     Post => Big (Y) = Big_2xx (Shift) * Big (X);
+
+   procedure Lemma_Simplify (X, Y : Big_Integer)
+   with
+     Ghost,
+     Pre  => Y /= 0,
+     Post => X * Y / Y = X;
+
+   procedure Lemma_Substitution (A, B, C, C1, D : Big_Integer)
+   with
+     Ghost,
+     Pre  => C = C1 and then A = B * C + D,
+     Post => A = B * C1 + D;
+
+   procedure Lemma_Subtract_Commutation (X, Y : Double_Uns)
+   with
+     Ghost,
+     Pre  => X >= Y,
+     Post => Big (X) - Big (Y) = Big (X - Y);
+
+   procedure Lemma_Subtract_Double_Uns (X, Y : Double_Int)
+   with
+     Ghost,
+     Pre  => X >= 0 and then X <= Y,
+     Post => Double_Uns (Y - X) = Double_Uns (Y) - Double_Uns (X);
+
+   procedure Lemma_Word_Commutation (X : Single_Uns)
+   with
+     Ghost,
+     Post => Big_2xx32 * Big (Double_Uns (X)) = Big (2**32 * Double_Uns (X));
+
+   -----------------------------
+   -- Local lemma null bodies --
+   -----------------------------
+
+   procedure Inline_Le3 (X1, X2, X3, Y1, Y2, Y3 : Single_Uns) is null;
+   procedure Lemma_Abs_Commutation (X : Double_Int) is null;
+   procedure Lemma_Abs_Mult_Commutation (X, Y : Big_Integer) is null;
+   procedure Lemma_Add_Commutation (X : Double_Uns; Y : Single_Uns) is null;
+   procedure Lemma_Add_One (X : Double_Uns) is null;
+   procedure Lemma_Bounded_Powers_Of_2_Increasing (M, N : Natural) is null;
+   procedure Lemma_Deep_Mult_Commutation
+     (Factor : Big_Integer;
+      X, Y   : Single_Uns)
+   is null;
+   procedure Lemma_Div_Commutation (X, Y : Double_Uns) is null;
+   procedure Lemma_Div_Definition
+     (A : Double_Uns;
+      B : Single_Uns;
+      Q : Double_Uns;
+      R : Double_Uns)
+   is null;
+   procedure Lemma_Div_Ge (X, Y, Z : Big_Integer) is null;
+   procedure Lemma_Div_Lt (X, Y, Z : Big_Natural) is null;
+   procedure Lemma_Div_Eq (A, B, S, R : Big_Integer) is null;
+   procedure Lemma_Double_Shift (X : Double_Uns; S, S1 : Double_Uns) is null;
+   procedure Lemma_Double_Shift (X : Single_Uns; S, S1 : Natural) is null;
+   procedure Lemma_Double_Shift_Right (X : Double_Uns; S, S1 : Double_Uns)
+   is null;
+   procedure Lemma_Ge_Commutation (A, B : Double_Uns) is null;
+   procedure Lemma_Ge_Mult (A, B, C, D : Big_Integer) is null;
+   procedure Lemma_Gt_Commutation (A, B : Double_Uns) is null;
+   procedure Lemma_Gt_Mult (A, B, C, D : Big_Integer) is null;
+   procedure Lemma_Lo_Is_Ident (X : Double_Uns) is null;
+   procedure Lemma_Lt_Commutation (A, B : Double_Uns) is null;
+   procedure Lemma_Lt_Mult (A, B, C, D : Big_Integer) is null;
+   procedure Lemma_Mult_Commutation (X, Y : Single_Uns) is null;
+   procedure Lemma_Mult_Commutation (X, Y : Double_Int) is null;
+   procedure Lemma_Mult_Commutation (X, Y, Z : Double_Uns) is null;
+   procedure Lemma_Mult_Distribution (X, Y, Z : Big_Integer) is null;
+   procedure Lemma_Neg_Rem (X, Y : Big_Integer) is null;
+   procedure Lemma_Rem_Commutation (X, Y : Double_Uns) is null;
+   procedure Lemma_Rem_Is_Ident (X, Y : Big_Integer) is null;
+   procedure Lemma_Rem_Sign (X, Y : Big_Integer) is null;
+   procedure Lemma_Rev_Div_Definition (A, B, Q, R : Big_Natural) is null;
+   procedure Lemma_Simplify (X, Y : Big_Integer) is null;
+   procedure Lemma_Substitution (A, B, C, C1, D : Big_Integer) is null;
+   procedure Lemma_Subtract_Commutation (X, Y : Double_Uns) is null;
+   procedure Lemma_Subtract_Double_Uns (X, Y : Double_Int) is null;
+   procedure Lemma_Word_Commutation (X : Single_Uns) is null;
+
    --------------------------
    -- Add_With_Ovflo_Check --
    --------------------------
@@ -111,18 +569,124 @@ package body System.Arith_Double is
    function Add_With_Ovflo_Check (X, Y : Double_Int) return Double_Int is
       R : constant Double_Int := To_Int (To_Uns (X) + To_Uns (Y));
 
+      --  Local lemmas
+
+      procedure Prove_Negative_X
+      with
+        Ghost,
+        Pre  => X < 0 and then (Y > 0 or else R < 0),
+        Post => R = X + Y;
+
+      procedure Prove_Non_Negative_X
+      with
+        Ghost,
+        Pre  => X >= 0 and then (Y < 0 or else R >= 0),
+        Post => R = X + Y;
+
+      procedure Prove_Overflow_Case
+      with
+        Ghost,
+        Pre  =>
+          (if X >= 0 then Y >= 0 and then R < 0
+                     else Y <= 0 and then R >= 0),
+        Post => not In_Double_Int_Range (Big (X) + Big (Y));
+
+      ----------------------
+      -- Prove_Negative_X --
+      ----------------------
+
+      procedure Prove_Negative_X is
+      begin
+         if X = Double_Int'First then
+            if Y > 0 then
+               null;
+            else
+               pragma Assert
+                 (To_Uns (X) + To_Uns (Y) =
+                    2 ** (Double_Size - 1) - Double_Uns (-Y));
+               pragma Assert  --  as R < 0
+                 (To_Uns (X) + To_Uns (Y) >= 2 ** (Double_Size - 1));
+               pragma Assert (Y = 0);
+            end if;
+
+         elsif Y = Double_Int'First then
+            pragma Assert
+              (To_Uns (X) + To_Uns (Y) =
+                 2 ** (Double_Size - 1) - Double_Uns (-X));
+            pragma Assert (False);
+
+         elsif Y <= 0 then
+            pragma Assert
+              (To_Uns (X) + To_Uns (Y) = -Double_Uns (-X) - Double_Uns (-Y));
+
+         else  --  Y > 0, 0 > X > Double_Int'First
+            declare
+               Ru : constant Double_Uns := To_Uns (X) + To_Uns (Y);
+            begin
+               pragma Assert (Ru = -Double_Uns (-X) + Double_Uns (Y));
+               if Ru < 2 ** (Double_Size - 1) then  --  R >= 0
+                  Lemma_Subtract_Double_Uns (-X, Y);
+                  pragma Assert (Ru = Double_Uns (X + Y));
+
+               elsif Ru = 2 ** (Double_Size - 1) then
+                  pragma Assert (Double_Uns (Y) < 2 ** (Double_Size - 1));
+                  pragma Assert (Double_Uns (-X) < 2 ** (Double_Size - 1));
+                  pragma Assert (False);
+
+               else
+                  pragma Assert
+                    (R = -Double_Int (-(-Double_Uns (-X) + Double_Uns (Y))));
+                  pragma Assert
+                    (R = -Double_Int (-Double_Uns (Y) + Double_Uns (-X)));
+               end if;
+            end;
+         end if;
+      end Prove_Negative_X;
+
+      --------------------------
+      -- Prove_Non_Negative_X --
+      --------------------------
+
+      procedure Prove_Non_Negative_X is
+      begin
+         if Y >= 0 or else Y = Double_Int'First then
+            null;
+         else
+            pragma Assert
+              (To_Uns (X) + To_Uns (Y) = Double_Uns (X) - Double_Uns (-Y));
+         end if;
+      end Prove_Non_Negative_X;
+
+      -------------------------
+      -- Prove_Overflow_Case --
+      -------------------------
+
+      procedure Prove_Overflow_Case is
+      begin
+         if X < 0 and then X /= Double_Int'First and then Y /= Double_Int'First
+         then
+            pragma Assert
+              (To_Uns (X) + To_Uns (Y) = -Double_Uns (-X) - Double_Uns (-Y));
+         end if;
+      end Prove_Overflow_Case;
+
+   --  Start of processing for Add_With_Ovflo_Check
+
    begin
       if X >= 0 then
          if Y < 0 or else R >= 0 then
+            Prove_Non_Negative_X;
             return R;
          end if;
 
       else -- X < 0
          if Y > 0 or else R < 0 then
+            Prove_Negative_X;
             return R;
          end if;
       end if;
 
+      Prove_Overflow_Case;
       Raise_Error;
    end Add_With_Ovflo_Check;
 
@@ -147,24 +711,173 @@ package body System.Arith_Double is
 
       T1, T2     : Double_Uns;
       Du, Qu, Ru : Double_Uns;
-      Den_Pos    : Boolean;
+      Den_Pos    : constant Boolean := (Y < 0) = (Z < 0);
+
+      --  Local ghost variables
+
+      Mult  : constant Big_Integer := abs (Big (Y) * Big (Z)) with Ghost;
+      Quot  : Big_Integer with Ghost;
+      Big_R : Big_Integer with Ghost;
+      Big_Q : Big_Integer with Ghost;
+
+      --  Local lemmas
+
+      function Is_Division_By_Zero_Case return Boolean is
+        (Y = 0 or else Z = 0)
+      with Ghost;
+
+      function Is_Overflow_Case return Boolean is
+        (not In_Double_Int_Range (Big (X) / (Big (Y) * Big (Z))))
+      with
+        Ghost,
+        Pre => Y /= 0 and Z /= 0;
+
+      procedure Prove_Overflow_Case
+      with
+        Ghost,
+        Pre  => X = Double_Int'First and then Big (Y) * Big (Z) = -1,
+        Post => Is_Overflow_Case;
+      --  Proves the special case where -2**(Double_Size - 1) is divided by -1,
+      --  generating an overflow.
+
+      procedure Prove_Quotient_Zero
+      with
+        Ghost,
+        Pre  => Mult >= Big_2xx64
+          and then
+            not (Mult = Big_2xx64 and then X = Double_Int'First and then Round)
+          and then Q = 0
+          and then R = X,
+        Post => Big (R) = Big (X) rem (Big (Y) * Big (Z))
+          and then
+            (if Round then
+               Big (Q) = Round_Quotient (Big (X), Big (Y) * Big (Z),
+                                         Big (X) / (Big (Y) * Big (Z)),
+                                         Big (R))
+             else Big (Q) = Big (X) / (Big (Y) * Big (Z)));
+      --  Proves the general case where divisor doesn't fit in Double_Uns and
+      --  quotient is 0.
+
+      procedure Prove_Round_To_One
+      with
+        Ghost,
+        Pre  => Mult = Big_2xx64
+          and then X = Double_Int'First
+          and then Q = (if Den_Pos then -1 else 1)
+          and then R = X
+          and then Round,
+        Post => Big (R) = Big (X) rem (Big (Y) * Big (Z))
+          and then Big (Q) = Round_Quotient (Big (X), Big (Y) * Big (Z),
+                                             Big (X) / (Big (Y) * Big (Z)),
+                                             Big (R));
+      --  Proves the special case where the divisor doesn't fit in Double_Uns
+      --  but quotient is still 1 or -1 due to rounding
+      --  (abs (Y*Z) = 2**Double_Size and X = -2**(Double_Size - 1) and Round).
+
+      procedure Prove_Rounding_Case
+      with
+        Ghost,
+        Pre  => Mult /= 0
+          and then Quot = Big (X) / (Big (Y) * Big (Z))
+          and then Big_R = Big (X) rem (Big (Y) * Big (Z))
+          and then Big_Q =
+            Round_Quotient (Big (X), Big (Y) * Big (Z), Quot, Big_R)
+          and then Big (Ru) = abs Big_R
+          and then Big (Du) = Mult
+          and then Big (Qu) =
+            (if Ru > (Du - Double_Uns'(1)) / Double_Uns'(2)
+             then abs Quot + 1
+             else abs Quot),
+        Post => abs Big_Q = Big (Qu);
+      --  Proves correctness of the rounding of the unsigned quotient
+
+      procedure Prove_Signs
+      with
+        Ghost,
+        Pre  => Mult /= 0
+          and then Quot = Big (X) / (Big (Y) * Big (Z))
+          and then Big_R = Big (X) rem (Big (Y) * Big (Z))
+          and then Big_Q =
+            (if Round then
+               Round_Quotient (Big (X), Big (Y) * Big (Z), Quot, Big_R)
+             else Quot)
+          and then Big (Ru) = abs Big_R
+          and then Big (Qu) = abs Big_Q
+          and then R = (if X >= 0 then To_Int (Ru) else To_Int (-Ru))
+          and then
+            Q = (if (X >= 0) = Den_Pos then To_Int (Qu) else To_Int (-Qu))
+          and then not (X = Double_Int'First and then Big (Y) * Big (Z) = -1),
+        Post => Big (R) = Big_R and then Big (Q) = Big_Q;
+      --  Proves final signs match the intended result after the unsigned
+      --  division is done.
+
+      -----------------------------
+      -- Local lemma null bodies --
+      -----------------------------
+
+      procedure Prove_Overflow_Case is null;
+      procedure Prove_Quotient_Zero is null;
+      procedure Prove_Round_To_One is null;
+
+      -------------------------
+      -- Prove_Rounding_Case --
+      -------------------------
+
+      procedure Prove_Rounding_Case is
+      begin
+         if Same_Sign (Big (X), Big (Y) * Big (Z)) then
+            null;
+         end if;
+      end Prove_Rounding_Case;
+
+      -----------------
+      -- Prove_Signs --
+      -----------------
+
+      procedure Prove_Signs is
+      begin
+         if (X >= 0) = Den_Pos then
+            pragma Assert (Quot >= 0);
+            pragma Assert (Big_Q >= 0);
+            pragma Assert (Big (Q) = Big_Q);
+         else
+            pragma Assert ((X >= 0) /= (Big (Y) * Big (Z) >= 0));
+            pragma Assert (Quot <= 0);
+            pragma Assert (Big_Q <= 0);
+            pragma Assert (Big (R) = Big_R);
+         end if;
+      end Prove_Signs;
+
+   --  Start of processing for Double_Divide
 
    begin
       if Yu = 0 or else Zu = 0 then
+         pragma Assert (Is_Division_By_Zero_Case);
          Raise_Error;
+         pragma Annotate
+           (GNATprove, Intentional, "call to nonreturning subprogram",
+            "Constraint_Error is raised in case of division by zero");
       end if;
 
-      --  Set final signs (RM 4.5.5(27-30))
-
-      Den_Pos := (Y < 0) = (Z < 0);
+      pragma Assert (Mult /= 0);
+      pragma Assert (Den_Pos = (Big (Y) * Big (Z) > 0));
+      Quot := Big (X) / (Big (Y) * Big (Z));
+      Big_R := Big (X) rem (Big (Y) * Big (Z));
+      if Round then
+         Big_Q := Round_Quotient (Big (X), Big (Y) * Big (Z), Quot, Big_R);
+      else
+         Big_Q := Quot;
+      end if;
+      Lemma_Mult_Decomposition (Mult, Yu, Zu, Yhi, Ylo, Zhi, Zlo);
 
       --  Compute Y * Z. Note that if the result overflows Double_Uns, then
       --  the rounded result is zero, except for the very special case where
-      --  X = -2 ** (Double_Size - 1) and abs(Y*Z) = 2 ** Double_Size, when
+      --  X = -2 ** (Double_Size - 1) and abs (Y * Z) = 2 ** Double_Size, when
       --  Round is True.
 
       if Yhi /= 0 then
          if Zhi /= 0 then
+            R := X;
 
             --  Handle the special case when Round is True
 
@@ -176,11 +889,23 @@ package body System.Arith_Double is
               and then Round
             then
                Q := (if Den_Pos then -1 else 1);
+
+               Prove_Round_To_One;
+
             else
                Q := 0;
+
+               pragma Assert (Big (Double_Uns'(Yhi * Zhi)) >= 1);
+               if Yhi > 1 or else Zhi > 1 then
+                  pragma Assert (Big (Double_Uns'(Yhi * Zhi)) > 1);
+               elsif Zlo > 0 then
+                  pragma Assert (Big (Double_Uns'(Yhi * Zlo)) > 0);
+               elsif Ylo > 0 then
+                  pragma Assert (Big (Double_Uns'(Ylo * Zhi)) > 0);
+               end if;
+               Prove_Quotient_Zero;
             end if;
 
-            R := X;
             return;
          else
             T2 := Yhi * Zlo;
@@ -191,9 +916,34 @@ package body System.Arith_Double is
       end if;
 
       T1 := Ylo * Zlo;
+
+      pragma Assert (Big (T2) = Big (Double_Uns'(Yhi * Zlo))
+                              + Big (Double_Uns'(Ylo * Zhi)));
+      Lemma_Mult_Distribution (Big_2xx32, Big (Double_Uns'(Yhi * Zlo)),
+                                          Big (Double_Uns'(Ylo * Zhi)));
+      pragma Assert (Mult = Big_2xx32 * Big (T2) + Big (T1));
+      Lemma_Hi_Lo (T1, Hi (T1), Lo (T1));
+      pragma Assert
+        (Mult = Big_2xx32 * Big (T2)
+              + Big_2xx32 * Big (Double_Uns (Hi (T1)))
+                          + Big (Double_Uns (Lo (T1))));
+      Lemma_Mult_Distribution (Big_2xx32, Big (T2),
+                                          Big (Double_Uns (Hi (T1))));
+      Lemma_Add_Commutation (T2, Hi (T1));
+
       T2 := T2 + Hi (T1);
 
+      pragma Assert (Mult = Big_2xx32 * Big (T2) + Big (Double_Uns (Lo (T1))));
+      Lemma_Hi_Lo (T2, Hi (T2), Lo (T2));
+      Lemma_Mult_Distribution (Big_2xx32, Big (Double_Uns (Hi (T2))),
+                                          Big (Double_Uns (Lo (T2))));
+      pragma Assert
+        (Mult = Big_2xx64 * Big (Double_Uns (Hi (T2)))
+              + Big_2xx32 * Big (Double_Uns (Lo (T2)))
+                          + Big (Double_Uns (Lo (T1))));
+
       if Hi (T2) /= 0 then
+         R := X;
 
          --  Handle the special case when Round is True
 
@@ -204,20 +954,53 @@ package body System.Arith_Double is
            and then Round
          then
             Q := (if Den_Pos then -1 else 1);
+
+            Prove_Round_To_One;
+
          else
             Q := 0;
+
+            pragma Assert (Big (Double_Uns (Hi (T2))) >= 1);
+            pragma Assert (Big (Double_Uns (Lo (T2))) >= 0);
+            pragma Assert (Big (Double_Uns (Lo (T1))) >= 0);
+            pragma Assert (Mult >= Big_2xx64);
+            if Hi (T2) > 1 then
+               pragma Assert (Big (Double_Uns (Hi (T2))) > 1);
+            elsif Lo (T2) > 0 then
+               pragma Assert (Big (Double_Uns (Lo (T2))) > 0);
+            elsif Lo (T1) > 0 then
+               pragma Assert (Double_Uns (Lo (T1)) > 0);
+               Lemma_Gt_Commutation (Double_Uns (Lo (T1)), 0);
+               pragma Assert (Big (Double_Uns (Lo (T1))) > 0);
+            end if;
+            Prove_Quotient_Zero;
          end if;
 
-         R := X;
          return;
       end if;
 
       Du := Lo (T2) & Lo (T1);
 
+      Lemma_Hi_Lo (Du, Lo (T2), Lo (T1));
+      pragma Assert (Mult = Big (Du));
+      pragma Assert (Du /= 0);
+      --  Multiplication of 2-limb arguments Yu and Zu leads to 4-limb result
+      --  (where each limb is a single value). Cases where 4 limbs are needed
+      --  require Yhi /= 0 and Zhi /= 0 and lead to early exit. Remaining cases
+      --  where 3 limbs are needed correspond to Hi(T2) /= 0 and lead to early
+      --  exit. Thus, at this point, the result fits in 2 limbs which are
+      --  exactly Lo (T2) and Lo (T1), which corresponds to the value of Du.
+      --  As the case where one of Yu or Zu is null also led to early exit,
+      --  we have Du /= 0 here.
+
       --  Check overflow case of largest negative number divided by -1
 
       if X = Double_Int'First and then Du = 1 and then not Den_Pos then
+         Prove_Overflow_Case;
          Raise_Error;
+         pragma Annotate
+           (GNATprove, Intentional, "call to nonreturning subprogram",
+            "Constraint_Error is raised in case of overflow");
       end if;
 
       --  Perform the actual division
@@ -231,31 +1014,52 @@ package body System.Arith_Double is
       --  exactly Lo(T2) and Lo(T1), which corresponds to the value of Du.
       --  As the case where one of Yu or Zu is null also led to early exit,
       --  we have Du/=0 here.
+
       Qu := Xu / Du;
       Ru := Xu rem Du;
 
+      Lemma_Div_Commutation (Xu, Du);
+      Lemma_Abs_Div_Commutation (Big (X), Big (Y) * Big (Z));
+      Lemma_Abs_Commutation (X);
+      pragma Assert (abs Quot = Big (Qu));
+      Lemma_Rem_Commutation (Xu, Du);
+      Lemma_Abs_Rem_Commutation (Big (X), Big (Y) * Big (Z));
+      pragma Assert (abs Big_R = Big (Ru));
+
       --  Deal with rounding case
 
-      if Round and then Ru > (Du - Double_Uns'(1)) / Double_Uns'(2) then
-         Qu := Qu + Double_Uns'(1);
+      if Round then
+         if Ru > (Du - Double_Uns'(1)) / Double_Uns'(2) then
+            Lemma_Add_Commutation (Qu, 1);
+
+            Qu := Qu + Double_Uns'(1);
+         end if;
+
+         Prove_Rounding_Case;
       end if;
 
+      pragma Assert (abs Big_Q = Big (Qu));
+
+      --  Set final signs (RM 4.5.5(27-30))
+
       --  Case of dividend (X) sign positive
 
       if X >= 0 then
          R := To_Int (Ru);
-         Q := (if Den_Pos then To_Int (Qu) else -To_Int (Qu));
-
-      --  Case of dividend (X) sign negative
+         Q := (if Den_Pos then To_Int (Qu) else To_Int (-Qu));
 
       --  We perform the unary minus operation on the unsigned value
       --  before conversion to signed, to avoid a possible overflow
       --  for value -2 ** (Double_Size - 1), both for computing R and Q.
 
+      --  Case of dividend (X) sign negative
+
       else
          R := To_Int (-Ru);
          Q := (if Den_Pos then To_Int (-Qu) else To_Int (Qu));
       end if;
+
+      Prove_Signs;
    end Double_Divide;
 
    ---------
@@ -278,6 +1082,259 @@ package body System.Arith_Double is
    end Le3;
 
    -------------------------------
+   -- Lemma_Abs_Div_Commutation --
+   -------------------------------
+
+   procedure Lemma_Abs_Div_Commutation (X, Y : Big_Integer) is
+   begin
+      if Y < 0 then
+         if X < 0 then
+            pragma Assert (abs (X / Y) = abs (X / (-Y)));
+         else
+            Lemma_Neg_Div (X, Y);
+            pragma Assert (abs (X / Y) = abs ((-X) / (-Y)));
+         end if;
+      end if;
+   end Lemma_Abs_Div_Commutation;
+
+   -------------------------------
+   -- Lemma_Abs_Rem_Commutation --
+   -------------------------------
+
+   procedure Lemma_Abs_Rem_Commutation (X, Y : Big_Integer) is
+   begin
+      if Y < 0 then
+         Lemma_Neg_Rem (X, Y);
+         if X < 0 then
+            pragma Assert (X rem Y = -((-X) rem (-Y)));
+            pragma Assert (abs (X rem Y) = (abs X) rem (abs Y));
+         else
+            pragma Assert (abs (X rem Y) = (abs X) rem (abs Y));
+         end if;
+      end if;
+   end Lemma_Abs_Rem_Commutation;
+
+   ------------------------
+   -- Lemma_Double_Shift --
+   ------------------------
+
+   procedure Lemma_Double_Shift (X : Double_Uns; S, S1 : Natural) is
+   begin
+      Lemma_Double_Shift (X, Double_Uns (S), Double_Uns (S1));
+      pragma Assert (Shift_Left (Shift_Left (X, S), S1)
+        = Shift_Left (Shift_Left (X, S), Natural (Double_Uns (S1))));
+      pragma Assert (Shift_Left (X, S + S1)
+        = Shift_Left (X, Natural (Double_Uns (S + S1))));
+   end Lemma_Double_Shift;
+
+   ------------------------------
+   -- Lemma_Double_Shift_Right --
+   ------------------------------
+
+   procedure Lemma_Double_Shift_Right (X : Double_Uns; S, S1 : Natural) is
+   begin
+      Lemma_Double_Shift_Right (X, Double_Uns (S), Double_Uns (S1));
+      pragma Assert (Shift_Right (Shift_Right (X, S), S1)
+        = Shift_Right (Shift_Right (X, S), Natural (Double_Uns (S1))));
+      pragma Assert (Shift_Right (X, S + S1)
+        = Shift_Right (X, Natural (Double_Uns (S + S1))));
+   end Lemma_Double_Shift_Right;
+
+   -----------------
+   -- Lemma_Hi_Lo --
+   -----------------
+
+   procedure Lemma_Hi_Lo (Xu : Double_Uns; Xhi, Xlo : Single_Uns) is
+   begin
+      pragma Assert (Double_Uns (Xhi) = Xu / Double_Uns'(2 ** Single_Size));
+      pragma Assert (Double_Uns (Xlo) = Xu mod 2 ** Single_Size);
+   end Lemma_Hi_Lo;
+
+   -------------------
+   -- Lemma_Hi_Lo_3 --
+   -------------------
+
+   procedure Lemma_Hi_Lo_3 (Xu : Double_Uns; Xhi, Xlo : Single_Uns) is
+   begin
+      Lemma_Hi_Lo (Xu, Xhi, Xlo);
+   end Lemma_Hi_Lo_3;
+
+   ------------------------------
+   -- Lemma_Mult_Decomposition --
+   ------------------------------
+
+   procedure Lemma_Mult_Decomposition
+     (Mult               : Big_Integer;
+      Xu, Yu             : Double_Uns;
+      Xhi, Xlo, Yhi, Ylo : Single_Uns)
+   is
+   begin
+      Lemma_Hi_Lo (Xu, Xhi, Xlo);
+      Lemma_Hi_Lo (Yu, Yhi, Ylo);
+
+      pragma Assert
+        (Mult = (Big_2xx32 * Big (Double_Uns (Xhi)) + Big (Double_Uns (Xlo))) *
+                (Big_2xx32 * Big (Double_Uns (Yhi)) + Big (Double_Uns (Ylo))));
+      pragma Assert (Mult =
+        Big_2xx32 * Big_2xx32 * Big (Double_Uns (Xhi)) * Big (Double_Uns (Yhi))
+                  + Big_2xx32 * Big (Double_Uns (Xhi)) * Big (Double_Uns (Ylo))
+                  + Big_2xx32 * Big (Double_Uns (Xlo)) * Big (Double_Uns (Yhi))
+                            + Big (Double_Uns (Xlo)) * Big (Double_Uns (Ylo)));
+      Lemma_Deep_Mult_Commutation (Big_2xx32 * Big_2xx32, Xhi, Yhi);
+      Lemma_Deep_Mult_Commutation (Big_2xx32, Xhi, Ylo);
+      Lemma_Deep_Mult_Commutation (Big_2xx32, Xlo, Yhi);
+      Lemma_Mult_Commutation (Xlo, Ylo);
+      pragma Assert (Mult =
+                       Big_2xx32 * Big_2xx32 * Big (Double_Uns'(Xhi * Yhi))
+                                 + Big_2xx32 * Big (Double_Uns'(Xhi * Ylo))
+                                 + Big_2xx32 * Big (Double_Uns'(Xlo * Yhi))
+                                             + Big (Double_Uns'(Xlo * Ylo)));
+   end Lemma_Mult_Decomposition;
+
+   -------------------
+   -- Lemma_Neg_Div --
+   -------------------
+
+   procedure Lemma_Neg_Div (X, Y : Big_Integer) is
+   begin
+      pragma Assert ((-X) / (-Y) = -(X / (-Y)));
+      pragma Assert (X / (-Y) = -(X / Y));
+   end Lemma_Neg_Div;
+
+   -----------------------
+   -- Lemma_Powers_Of_2 --
+   -----------------------
+
+   procedure Lemma_Powers_Of_2 (M, N : Natural) is
+   begin
+      if M + N < Double_Size then
+         pragma Assert (Double_Uns'(2**M) * Double_Uns'(2**N)
+                        = Double_Uns'(2**(M + N)));
+      end if;
+
+      Lemma_Powers_Of_2_Commutation (M);
+      Lemma_Powers_Of_2_Commutation (N);
+      Lemma_Powers_Of_2_Commutation (M + N);
+
+      if M + N < Double_Size then
+         pragma Assert (Big (Double_Uns'(2))**M * Big (Double_Uns'(2))**N
+                        = Big (Double_Uns'(2))**(M + N));
+         Lemma_Powers_Of_2_Increasing (M + N, Double_Size);
+         Lemma_Mult_Commutation (2 ** M, 2 ** N, 2 ** (M + N));
+      else
+         pragma Assert (Big (Double_Uns'(2))**M * Big (Double_Uns'(2))**N
+                        = Big (Double_Uns'(2))**(M + N));
+      end if;
+   end Lemma_Powers_Of_2;
+
+   -----------------------------------
+   -- Lemma_Powers_Of_2_Commutation --
+   -----------------------------------
+
+   procedure Lemma_Powers_Of_2_Commutation (M : Natural) is
+   begin
+      if M > 0 then
+         Lemma_Powers_Of_2_Commutation (M - 1);
+         pragma Assert (Big (Double_Uns'(2))**(M - 1) = Big_2xx (M - 1));
+         pragma Assert (Big (Double_Uns'(2))**M = Big_2xx (M - 1) * 2);
+         if M < Double_Size then
+            Lemma_Powers_Of_2_Increasing (M - 1, Double_Size - 1);
+            Lemma_Bounded_Powers_Of_2_Increasing (M - 1, Double_Size - 1);
+            pragma Assert (Double_Uns'(2 ** (M - 1)) * 2 = Double_Uns'(2**M));
+            Lemma_Mult_Commutation
+              (Double_Uns'(2 ** (M - 1)), 2, Double_Uns'(2**M));
+            pragma Assert (Big (Double_Uns'(2))**M = Big_2xx (M));
+         end if;
+      end if;
+   end Lemma_Powers_Of_2_Commutation;
+
+   ----------------------------------
+   -- Lemma_Powers_Of_2_Increasing --
+   ----------------------------------
+
+   procedure Lemma_Powers_Of_2_Increasing (M, N : Natural) is
+   begin
+      if M + 1 < N then
+         Lemma_Powers_Of_2_Increasing (M + 1, N);
+      end if;
+   end Lemma_Powers_Of_2_Increasing;
+
+   -------------------
+   -- Lemma_Rem_Abs --
+   -------------------
+
+   procedure Lemma_Rem_Abs (X, Y : Big_Integer) is
+   begin
+      Lemma_Neg_Rem (X, Y);
+   end Lemma_Rem_Abs;
+
+   -----------------------
+   -- Lemma_Shift_Right --
+   -----------------------
+
+   procedure Lemma_Shift_Right (X : Double_Uns; Shift : Natural) is
+      XX : Double_Uns := X;
+   begin
+      for J in 1 .. Shift loop
+         XX := Shift_Right (XX, 1);
+         Lemma_Double_Shift_Right (X, J - 1, 1);
+         pragma Loop_Invariant (XX = Shift_Right (X, J));
+         pragma Loop_Invariant (XX = X / Double_Uns'(2) ** J);
+      end loop;
+   end Lemma_Shift_Right;
+
+   ------------------------------
+   -- Lemma_Shift_Without_Drop --
+   ------------------------------
+
+   procedure Lemma_Shift_Without_Drop
+     (X, Y  : Double_Uns;
+      Mask  : Single_Uns;
+      Shift : Natural)
+   is
+      pragma Unreferenced (Mask);
+
+      procedure Lemma_Bound
+      with
+        Pre  => Shift <= Single_Size
+          and then X <= 2**Single_Size
+            * Double_Uns'(2**(Single_Size - Shift) - 1)
+            + Single_Uns'(2**Single_Size - 1),
+        Post => X <= 2**(Double_Size - Shift) - 1;
+
+      procedure Lemma_Exp_Pos (N : Integer)
+      with
+        Pre  => N in 0 .. Double_Size - 1,
+        Post => Double_Uns'(2**N) > 0;
+
+      -----------------------------
+      -- Local lemma null bodies --
+      -----------------------------
+
+      procedure Lemma_Bound is null;
+      procedure Lemma_Exp_Pos (N : Integer) is null;
+
+   --  Start of processing for Lemma_Shift_Without_Drop
+
+   begin
+      if Shift = 0 then
+         pragma Assert (Big (Y) = Big_2xx (Shift) * Big (X));
+         return;
+      end if;
+
+      Lemma_Bound;
+      Lemma_Exp_Pos (Double_Size - Shift);
+      pragma Assert (X < 2**(Double_Size - Shift));
+      pragma Assert (Big (X) < Big_2xx (Double_Size - Shift));
+      pragma Assert (Y = 2**Shift * X);
+      pragma Assert (Big_2xx (Shift) * Big (X)
+                     < Big_2xx (Shift) * Big_2xx (Double_Size - Shift));
+      Lemma_Powers_Of_2 (Shift, Double_Size - Shift);
+      Lemma_Mult_Commutation (2**Shift, X, Y);
+      pragma Assert (Big (Y) = Big_2xx (Shift) * Big (X));
+   end Lemma_Shift_Without_Drop;
+
+   -------------------------------
    -- Multiply_With_Ovflo_Check --
    -------------------------------
 
@@ -292,9 +1349,148 @@ package body System.Arith_Double is
 
       T1, T2 : Double_Uns;
 
+      --  Local ghost variables
+
+      Mult : constant Big_Integer := abs (Big (X) * Big (Y)) with Ghost;
+
+      --  Local lemmas
+
+      procedure Prove_Both_Too_Large
+      with
+        Ghost,
+        Pre  => Xhi /= 0
+          and then Yhi /= 0
+          and then Mult =
+            Big_2xx32 * Big_2xx32 * (Big (Double_Uns'(Xhi * Yhi)))
+                      + Big_2xx32 * (Big (Double_Uns'(Xhi * Ylo)))
+                      + Big_2xx32 * (Big (Double_Uns'(Xlo * Yhi)))
+                                  + (Big (Double_Uns'(Xlo * Ylo))),
+        Post => not In_Double_Int_Range (Big (X) * Big (Y));
+
+      procedure Prove_Final_Decomposition
+      with
+        Ghost,
+        Pre  => In_Double_Int_Range (Big (X) * Big (Y))
+          and then Mult = Big_2xx32 * Big (T2) + Big (Double_Uns (Lo (T1)))
+          and then Hi (T2) = 0,
+        Post => Mult = Big (Lo (T2) & Lo (T1));
+
+      procedure Prove_Neg_Int
+      with
+        Ghost,
+        Pre  => In_Double_Int_Range (Big (X) * Big (Y))
+          and then Mult = Big (T2)
+          and then ((X >= 0 and then Y < 0) or else (X < 0 and then Y >= 0)),
+        Post => To_Neg_Int (T2) = X * Y;
+
+      procedure Prove_Pos_Int
+      with
+        Ghost,
+        Pre  => In_Double_Int_Range (Big (X) * Big (Y))
+          and then Mult = Big (T2)
+          and then ((X >= 0 and then Y >= 0) or else (X < 0 and then Y < 0)),
+        Post => To_Pos_Int (T2) = X * Y;
+
+      procedure Prove_Result_Too_Large
+      with
+        Ghost,
+        Pre  => Mult = Big_2xx32 * Big (T2) + Big (Double_Uns (Lo (T1)))
+          and then Hi (T2) /= 0,
+        Post => not In_Double_Int_Range (Big (X) * Big (Y));
+
+      procedure Prove_Too_Large
+      with
+        Ghost,
+        Pre  => abs (Big (X) * Big (Y)) >= Big_2xx64,
+        Post => not In_Double_Int_Range (Big (X) * Big (Y));
+
+      --------------------------
+      -- Prove_Both_Too_Large --
+      --------------------------
+
+      procedure Prove_Both_Too_Large is
+      begin
+         pragma Assert
+           (Mult >= Big_2xx32 * Big_2xx32 * Big (Double_Uns'(Xhi * Yhi)));
+         pragma Assert (Double_Uns (Xhi) * Double_Uns (Yhi) >= 1);
+         pragma Assert (Mult >= Big_2xx32 * Big_2xx32);
+         Prove_Too_Large;
+      end Prove_Both_Too_Large;
+
+      -------------------------------
+      -- Prove_Final_Decomposition --
+      -------------------------------
+
+      procedure Prove_Final_Decomposition is
+      begin
+         Lemma_Hi_Lo (T2, Hi (T2), Lo (T2));
+         pragma Assert (Mult = Big_2xx32 * Big (Double_Uns (Lo (T2)))
+                                         + Big (Double_Uns (Lo (T1))));
+         pragma Assert (Mult <= Big_2xx63);
+         Lemma_Mult_Commutation (X, Y);
+         pragma Assert (Mult = abs (Big (X * Y)));
+         Lemma_Word_Commutation (Lo (T2));
+         pragma Assert (Mult = Big (Double_Uns'(2 ** Single_Size)
+                          * Double_Uns (Lo (T2)))
+                          + Big (Double_Uns (Lo (T1))));
+         Lemma_Add_Commutation (Double_Uns'(2 ** Single_Size)
+                                  * Double_Uns (Lo (T2)),
+                                Lo (T1));
+         pragma Assert (Mult = Big (Double_Uns'(2 ** Single_Size)
+                          * Double_Uns (Lo (T2)) + Lo (T1)));
+         pragma Assert (Lo (T2) & Lo (T1) = Double_Uns'(2 ** Single_Size)
+                          * Double_Uns (Lo (T2)) + Lo (T1));
+      end Prove_Final_Decomposition;
+
+      -------------------
+      -- Prove_Neg_Int --
+      -------------------
+
+      procedure Prove_Neg_Int is
+      begin
+         pragma Assert (X * Y <= 0);
+         pragma Assert (Mult = -Big (X * Y));
+      end Prove_Neg_Int;
+
+      -------------------
+      -- Prove_Pos_Int --
+      -------------------
+
+      procedure Prove_Pos_Int is
+      begin
+         pragma Assert (X * Y >= 0);
+         pragma Assert (Mult = Big (X * Y));
+      end Prove_Pos_Int;
+
+      ----------------------------
+      -- Prove_Result_Too_Large --
+      ----------------------------
+
+      procedure Prove_Result_Too_Large is
+      begin
+         pragma Assert (Mult >= Big_2xx32 * Big (T2));
+         Lemma_Hi_Lo (T2, Hi (T2), Lo (T2));
+         pragma Assert
+           (Mult >= Big_2xx32 * Big_2xx32 * Big (Double_Uns (Hi (T2))));
+         pragma Assert (Double_Uns (Hi (T2)) >= 1);
+         pragma Assert (Mult >= Big_2xx32 * Big_2xx32);
+         Prove_Too_Large;
+      end Prove_Result_Too_Large;
+
+      ---------------------
+      -- Prove_Too_Large --
+      ---------------------
+
+      procedure Prove_Too_Large is null;
+
+   --  Start of processing for Multiply_With_Ovflo_Check
+
    begin
+      Lemma_Mult_Decomposition (Mult, Xu, Yu, Xhi, Xlo, Yhi, Ylo);
+
       if Xhi /= 0 then
          if Yhi /= 0 then
+            Prove_Both_Too_Large;
             Raise_Error;
          else
             T2 := Xhi * Ylo;
@@ -311,28 +1507,50 @@ package body System.Arith_Double is
       --  result from the upper halves of the input values.
 
       T1 := Xlo * Ylo;
+
+      pragma Assert (Big (T2) = Big (Double_Uns'(Xhi * Ylo))
+                              + Big (Double_Uns'(Xlo * Yhi)));
+      Lemma_Mult_Distribution (Big_2xx32, Big (Double_Uns'(Xhi * Ylo)),
+                                          Big (Double_Uns'(Xlo * Yhi)));
+      pragma Assert (Mult = Big_2xx32 * Big (T2) + Big (T1));
+      Lemma_Add_Commutation (T2, Hi (T1));
+      pragma Assert
+        (Big (T2 + Hi (T1)) = Big (T2) + Big (Double_Uns (Hi (T1))));
+
       T2 := T2 + Hi (T1);
 
+      Lemma_Hi_Lo (T1, Hi (T1), Lo (T1));
+      pragma Assert (Mult = Big_2xx32 * Big (T2) + Big (Double_Uns (Lo (T1))));
+
       if Hi (T2) /= 0 then
+         Prove_Result_Too_Large;
          Raise_Error;
       end if;
 
+      Prove_Final_Decomposition;
+
       T2 := Lo (T2) & Lo (T1);
 
+      pragma Assert (Mult = Big (T2));
+
       if X >= 0 then
          if Y >= 0 then
+            Prove_Pos_Int;
             return To_Pos_Int (T2);
             pragma Annotate (CodePeer, Intentional, "precondition",
                              "Intentional Unsigned->Signed conversion");
          else
+            Prove_Neg_Int;
             return To_Neg_Int (T2);
          end if;
       else -- X < 0
          if Y < 0 then
+            Prove_Pos_Int;
             return To_Pos_Int (T2);
             pragma Annotate (CodePeer, Intentional, "precondition",
                              "Intentional Unsigned->Signed conversion");
          else
+            Prove_Neg_Int;
             return To_Neg_Int (T2);
          end if;
       end if;
@@ -346,6 +1564,9 @@ package body System.Arith_Double is
    procedure Raise_Error is
    begin
       raise Constraint_Error with "Double arithmetic overflow";
+      pragma Annotate
+        (GNATprove, Intentional, "exception might be raised",
+         "Procedure Raise_Error is called to signal input errors");
    end Raise_Error;
 
    -------------------
@@ -369,10 +1590,10 @@ package body System.Arith_Double is
       Zhi : Single_Uns := Hi (Zu);
       Zlo : Single_Uns := Lo (Zu);
 
-      D : array (1 .. 4) of Single_Uns;
+      D : array (1 .. 4) of Single_Uns with Relaxed_Initialization;
       --  The dividend, four digits (D(1) is high order)
 
-      Qd : array (1 .. 2) of Single_Uns;
+      Qd : array (1 .. 2) of Single_Uns with Relaxed_Initialization;
       --  The quotient digits, two digits (Qd(1) is high order)
 
       S1, S2, S3 : Single_Uns;
@@ -397,56 +1618,527 @@ package body System.Arith_Double is
       T1, T2, T3 : Double_Uns;
       --  Temporary values
 
+      --  Local ghost variables
+
+      Mult  : constant Big_Integer := abs (Big (X) * Big (Y)) with Ghost;
+      Quot  : Big_Integer with Ghost;
+      Big_R : Big_Integer with Ghost;
+      Big_Q : Big_Integer with Ghost;
+
+      --  Local lemmas
+
+      function Is_Division_By_Zero_Case return Boolean is (Z = 0) with Ghost;
+
+      procedure Prove_Dividend_Scaling
+      with
+        Ghost,
+        Pre  => D'Initialized
+          and then Scale <= Single_Size
+          and then Mult =
+            Big_2xx32 * Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (1)))
+                      + Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (2)))
+                                  + Big_2xx32 * Big (Double_Uns (D (3)))
+                                              + Big (Double_Uns (D (4)))
+          and then Big (D (1) & D (2)) * Big_2xx (Scale) < Big_2xx64
+          and then T1 = Shift_Left (D (1) & D (2), Scale)
+          and then T2 = Shift_Left (Double_Uns (D (3)), Scale)
+          and then T3 = Shift_Left (Double_Uns (D (4)), Scale),
+        Post => Mult * Big_2xx (Scale) =
+          Big_2xx32 * Big_2xx32 * Big_2xx32 * Big (Double_Uns (Hi (T1)))
+                    + Big_2xx32 * Big_2xx32 * Big (Double_Uns (Lo (T1) or
+                                                                      Hi (T2)))
+                                + Big_2xx32 * Big (Double_Uns (Lo (T2) or
+                                                                      Hi (T3)))
+                                            + Big (Double_Uns (Lo (T3)));
+      --  Proves the scaling of the 4-digit dividend actually multiplies it by
+      --  2**Scale.
+
+      procedure Prove_Multiplication (Q : Single_Uns)
+      with
+        Ghost,
+        Pre  => T1 = Q * Lo (Zu)
+          and then T2 = Q * Hi (Zu)
+          and then S3 = Lo (T1)
+          and then T3 = Hi (T1) + Lo (T2)
+          and then S2 = Lo (T3)
+          and then S1 = Hi (T3) + Hi (T2),
+        Post => Big3 (S1, S2, S3) = Big (Double_Uns (Q)) * Big (Zu);
+      --  Proves correctness of the multiplication of divisor by quotient to
+      --  compute amount to subtract.
+
+      procedure Prove_Overflow
+      with
+        Ghost,
+        Pre  => Z /= 0 and then Mult >= Big_2xx64 * Big (Double_Uns'(abs Z)),
+        Post => not In_Double_Int_Range (Big (X) * Big (Y) / Big (Z));
+      --  Proves overflow case when the quotient has at least 3 digits
+
+      procedure Prove_Qd_Calculation_Part_1 (J : Integer)
+      with
+        Ghost,
+        Pre  => J in 1 .. 2
+          and then D'Initialized
+          and then D (J) < Zhi
+          and then Hi (Zu) = Zhi
+          and then Qd (J)'Initialized
+          and then Qd (J) = Lo ((D (J) & D (J + 1)) / Zhi),
+        Post => Big (Double_Uns (Qd (J))) >=
+          Big3 (D (J), D (J + 1), D (J + 2)) / Big (Zu);
+      --  When dividing 3 digits by 2 digits, proves the initial calculation
+      --  of the quotient given by dividing the first 2 digits of the dividend
+      --  by the first digit of the divisor is not an underestimate (so
+      --  readjusting down works).
+
+      procedure Prove_Rescaling
+      with
+        Ghost,
+        Pre  => Scale <= Single_Size
+          and then Z /= 0
+          and then Mult * Big_2xx (Scale) = Big (Zu) * Big (Qu) + Big (Ru)
+          and then Big (Ru) < Big (Zu)
+          and then Big (Zu) = Big (Double_Uns'(abs Z)) * Big_2xx (Scale)
+          and then Quot = Big (X) * Big (Y) / Big (Z)
+          and then Big_R = Big (X) * Big (Y) rem Big (Z),
+        Post => abs Quot = Big (Qu)
+          and then abs Big_R = Big (Shift_Right (Ru, Scale));
+      --  Proves scaling back only the remainder is the right thing to do after
+      --  computing the scaled division.
+
+      procedure Prove_Rounding_Case
+      with
+        Ghost,
+        Pre  => Z /= 0
+          and then Quot = Big (X) * Big (Y) / Big (Z)
+          and then Big_R = Big (X) * Big (Y) rem Big (Z)
+          and then Big_Q =
+            Round_Quotient (Big (X) * Big (Y), Big (Z), Quot, Big_R)
+          and then Big (Ru) = abs Big_R
+          and then Big (Zu) = Big (Double_Uns'(abs Z)),
+        Post => abs Big_Q =
+          (if Ru > (Zu - Double_Uns'(1)) / Double_Uns'(2)
+           then abs Quot + 1
+           else abs Quot);
+      --  Proves correctness of the rounding of the unsigned quotient
+
+      procedure Prove_Sign_R
+      with
+        Ghost,
+        Pre  => Z /= 0 and then Big_R = Big (X) * Big (Y) rem Big (Z),
+        Post => In_Double_Int_Range (Big_R);
+
+      procedure Prove_Signs
+      with
+        Ghost,
+        Pre  => Z /= 0
+          and then Quot = Big (X) * Big (Y) / Big (Z)
+          and then Big_R = Big (X) * Big (Y) rem Big (Z)
+          and then Big_Q =
+            (if Round then
+               Round_Quotient (Big (X) * Big (Y), Big (Z), Quot, Big_R)
+             else Quot)
+          and then Big (Ru) = abs Big_R
+          and then Big (Qu) = abs Big_Q
+          and then In_Double_Int_Range (Big_Q)
+          and then In_Double_Int_Range (Big_R)
+          and then R =
+            (if (X >= 0) = (Y >= 0) then To_Pos_Int (Ru) else To_Neg_Int (Ru))
+          and then Q =
+            (if ((X >= 0) = (Y >= 0)) = (Z >= 0) then To_Pos_Int (Qu)
+             else To_Neg_Int (Qu)),  --  need to ensure To_Pos_Int precondition
+        Post => Big (R) = Big_R and then Big (Q) = Big_Q;
+      --  Proves final signs match the intended result after the unsigned
+      --  division is done.
+
+      procedure Prove_Z_Low
+      with
+        Ghost,
+        Pre  => Z /= 0
+          and then D'Initialized
+          and then Hi (abs Z) = 0
+          and then Lo (abs Z) = Zlo
+          and then Mult = Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (2)))
+                                    + Big_2xx32 * Big (Double_Uns (D (3)))
+                                                + Big (Double_Uns (D (4)))
+          and then D (2) < Zlo
+          and then Quot = (Big (X) * Big (Y)) / Big (Z)
+          and then Big_R = (Big (X) * Big (Y)) rem Big (Z)
+          and then T1 = D (2) & D (3)
+          and then T2 = Lo (T1 rem Zlo) & D (4)
+          and then Qu = Lo (T1 / Zlo) & Lo (T2 / Zlo)
+          and then Ru = T2 rem Zlo,
+        Post => Big (Qu) = abs Quot
+          and then Big (Ru) = abs Big_R;
+      --  Proves the case where the divisor is only one digit
+
+      ----------------------------
+      -- Prove_Dividend_Scaling --
+      ----------------------------
+
+      procedure Prove_Dividend_Scaling is
+      begin
+         Lemma_Hi_Lo (D (1) & D (2), D (1), D (2));
+         pragma Assert (Mult * Big_2xx (Scale) =
+           Big_2xx32 * Big_2xx32 * Big_2xx (Scale) * Big (D (1) & D (2))
+                     + Big_2xx32 * Big_2xx (Scale) * Big (Double_Uns (D (3)))
+                                 + Big_2xx (Scale) * Big (Double_Uns (D (4))));
+         pragma Assert (Big_2xx (Scale) > 0);
+         Lemma_Lt_Mult (Big (Double_Uns (D (3))), Big_2xx32,
+                        Big_2xx (Scale), Big_2xx64);
+         Lemma_Lt_Mult (Big (Double_Uns (D (4))), Big_2xx32,
+                        Big_2xx (Scale), Big_2xx64);
+         Lemma_Mult_Commutation (2 ** Scale, D (1) & D (2), T1);
+         declare
+            Big_D12 : constant Big_Integer :=
+              Big_2xx (Scale) * Big (D (1) & D (2));
+            Big_T1  : constant Big_Integer := Big (T1);
+         begin
+            pragma Assert (Big_D12 = Big_T1);
+            pragma Assert (Big_2xx32 * Big_2xx32 * Big_D12
+                           = Big_2xx32 * Big_2xx32 * Big_T1);
+         end;
+         Lemma_Mult_Commutation (2 ** Scale, Double_Uns (D (3)), T2);
+         declare
+            Big_D3 : constant Big_Integer :=
+              Big_2xx (Scale) * Big (Double_Uns (D (3)));
+            Big_T2 : constant Big_Integer := Big (T2);
+         begin
+            pragma Assert (Big_D3 = Big_T2);
+            pragma Assert (Big_2xx32 * Big_D3 = Big_2xx32 * Big_T2);
+         end;
+         Lemma_Mult_Commutation (2 ** Scale, Double_Uns (D (4)), T3);
+         declare
+            Big_D4 : constant Big_Integer :=
+              Big_2xx (Scale) * Big (Double_Uns (D (4)));
+            Big_T3 : constant Big_Integer := Big (T3);
+         begin
+            pragma Assert (Big_D4 = Big_T3);
+         end;
+         pragma Assert (Mult * Big_2xx (Scale) =
+           Big_2xx32 * Big_2xx32 * Big (T1) + Big_2xx32 * Big (T2) + Big (T3));
+         Lemma_Hi_Lo (T1, Hi (T1), Lo (T1));
+         Lemma_Hi_Lo (T2, Hi (T2), Lo (T2));
+         Lemma_Hi_Lo (T3, Hi (T3), Lo (T3));
+         pragma Assert (Double_Uns (Lo (T1) or Hi (T2)) =
+                          Double_Uns (Lo (T1)) + Double_Uns (Hi (T2)));
+         pragma Assert (Double_Uns (Lo (T2) or Hi (T3)) =
+                          Double_Uns (Lo (T2)) + Double_Uns (Hi (T3)));
+      end Prove_Dividend_Scaling;
+
+      --------------------------
+      -- Prove_Multiplication --
+      --------------------------
+
+      procedure Prove_Multiplication (Q : Single_Uns) is
+      begin
+         Lemma_Hi_Lo (Zu, Hi (Zu), Lo (Zu));
+         Lemma_Hi_Lo (T1, Hi (T1), S3);
+         Lemma_Hi_Lo (T2, Hi (T2), Lo (T2));
+         Lemma_Hi_Lo (T3, Hi (T3), S2);
+         pragma Assert (Big (Double_Uns (Q)) * Big (Zu) =
+                          Big_2xx32 * Big_2xx32 * Big (Double_Uns (Hi (T2)))
+                        + Big_2xx32 * Big_2xx32 * Big (Double_Uns (Hi (T3)))
+                                    + Big_2xx32 * Big (Double_Uns (S2))
+                                                + Big (Double_Uns (S3)));
+         pragma Assert (Double_Uns (Hi (T3)) + Hi (T2) = Double_Uns (S1));
+         Lemma_Add_Commutation (Double_Uns (Hi (T3)), Hi (T2));
+         pragma Assert
+           (Big (Double_Uns (Hi (T3))) + Big (Double_Uns (Hi (T2))) =
+              Big (Double_Uns (S1)));
+      end Prove_Multiplication;
+
+      --------------------
+      -- Prove_Overflow --
+      --------------------
+
+      procedure Prove_Overflow is
+      begin
+         Lemma_Div_Ge (Mult, Big_2xx64, Big (Double_Uns'(abs Z)));
+         Lemma_Abs_Commutation (Z);
+         Lemma_Abs_Div_Commutation (Big (X) * Big (Y), Big (Z));
+      end Prove_Overflow;
+
+      ---------------------------------
+      -- Prove_Qd_Calculation_Part_1 --
+      ---------------------------------
+
+      procedure Prove_Qd_Calculation_Part_1 (J : Integer) is
+      begin
+         Lemma_Hi_Lo (D (J) & D (J + 1), D (J), D (J + 1));
+         Lemma_Lt_Commutation (Double_Uns (D (J)), Double_Uns (Zhi));
+         Lemma_Gt_Mult (Big (Double_Uns (Zhi)),
+                        Big (Double_Uns (D (J))) + 1,
+                        Big_2xx32, Big (D (J) & D (J + 1)));
+         Lemma_Div_Lt
+           (Big (D (J) & D (J + 1)), Big_2xx32, Big (Double_Uns (Zhi)));
+         Lemma_Div_Commutation (D (J) & D (J + 1), Double_Uns (Zhi));
+         Lemma_Lo_Is_Ident ((D (J) & D (J + 1)) / Zhi);
+         Lemma_Div_Definition (D (J) & D (J + 1), Zhi, Double_Uns (Qd (J)),
+                               (D (J) & D (J + 1)) rem Zhi);
+         Lemma_Lt_Commutation
+           ((D (J) & D (J + 1)) rem Zhi, Double_Uns (Zhi));
+         Lemma_Gt_Mult
+           ((Big (Double_Uns (Qd (J))) + 1) * Big (Double_Uns (Zhi)),
+            Big (D (J) & D (J + 1)) + 1, Big_2xx32,
+            Big3 (D (J), D (J + 1), D (J + 2)));
+         Lemma_Hi_Lo (Zu, Zhi, Lo (Zu));
+         Lemma_Gt_Mult (Big (Zu), Big_2xx32 * Big (Double_Uns (Zhi)),
+                        Big (Double_Uns (Qd (J))) + 1,
+                        Big3 (D (J), D (J + 1), D (J + 2)));
+         Lemma_Div_Lt (Big3 (D (J), D (J + 1), D (J + 2)),
+                       Big (Double_Uns (Qd (J))) + 1, Big (Zu));
+      end Prove_Qd_Calculation_Part_1;
+
+      ---------------------
+      -- Prove_Rescaling --
+      ---------------------
+
+      procedure Prove_Rescaling is
+      begin
+         Lemma_Div_Lt (Big (Ru), Big (Double_Uns'(abs Z)), Big_2xx (Scale));
+         Lemma_Div_Eq (Mult, Big (Double_Uns'(abs Z)) * Big (Qu),
+                       Big_2xx (Scale), Big (Ru));
+         Lemma_Rev_Div_Definition (Mult, Big (Double_Uns'(abs Z)),
+                                   Big (Qu), Big (Ru) / Big_2xx (Scale));
+         Lemma_Abs_Div_Commutation (Big (X) * Big (Y), Big (Z));
+         Lemma_Abs_Rem_Commutation (Big (X) * Big (Y), Big (Z));
+         Lemma_Abs_Commutation (Z);
+         Lemma_Shift_Right (Ru, Scale);
+      end Prove_Rescaling;
+
+      -------------------------
+      -- Prove_Rounding_Case --
+      -------------------------
+
+      procedure Prove_Rounding_Case is
+      begin
+         if Same_Sign (Big (X) * Big (Y), Big (Z)) then
+            null;
+         end if;
+      end Prove_Rounding_Case;
+
+      ------------------
+      -- Prove_Sign_R --
+      ------------------
+
+      procedure Prove_Sign_R is
+      begin
+         pragma Assert (In_Double_Int_Range (Big (Z)));
+      end Prove_Sign_R;
+
+      -----------------
+      -- Prove_Signs --
+      -----------------
+
+      procedure Prove_Signs is null;
+
+      -----------------
+      -- Prove_Z_Low --
+      -----------------
+
+      procedure Prove_Z_Low is
+      begin
+         Lemma_Hi_Lo (T1, D (2), D (3));
+         Lemma_Add_Commutation (Double_Uns (D (2)), 1);
+         pragma Assert
+           (Big (Double_Uns (D (2))) + 1 <= Big (Double_Uns (Zlo)));
+         Lemma_Div_Definition (T1, Zlo, T1 / Zlo, T1 rem Zlo);
+         pragma Assert (Double_Uns (Lo (T1 rem Zlo)) = T1 rem Zlo);
+         Lemma_Hi_Lo (T2, Lo (T1 rem Zlo), D (4));
+         pragma Assert (T1 rem Zlo + Double_Uns'(1) <= Double_Uns (Zlo));
+         Lemma_Add_Commutation (T1 rem Zlo, 1);
+         pragma Assert (Big (T1 rem Zlo) + 1 <= Big (Double_Uns (Zlo)));
+         Lemma_Div_Definition (T2, Zlo, T2 / Zlo, Ru);
+         pragma Assert
+           (Mult = Big (Double_Uns (Zlo)) *
+                     (Big_2xx32 * Big (T1 / Zlo) + Big (T2 / Zlo)) + Big (Ru));
+         Lemma_Div_Lt (Big (T1), Big_2xx32, Big (Double_Uns (Zlo)));
+         Lemma_Div_Commutation (T1, Double_Uns (Zlo));
+         Lemma_Lo_Is_Ident (T1 / Zlo);
+         Lemma_Div_Lt (Big (T2), Big_2xx32, Big (Double_Uns (Zlo)));
+         Lemma_Div_Commutation (T2, Double_Uns (Zlo));
+         Lemma_Lo_Is_Ident (T2 / Zlo);
+         Lemma_Hi_Lo (Qu, Lo (T1 / Zlo), Lo (T2 / Zlo));
+         Lemma_Substitution (Mult, Big (Double_Uns (Zlo)),
+                             Big_2xx32 * Big (T1 / Zlo) + Big (T2 / Zlo),
+                             Big (Qu), Big (Ru));
+         Lemma_Lt_Commutation (Ru, Double_Uns (Zlo));
+         Lemma_Rev_Div_Definition
+           (Mult, Big (Double_Uns (Zlo)), Big (Qu), Big (Ru));
+         pragma Assert (Double_Uns (Zlo) = abs Z);
+         Lemma_Abs_Commutation (Z);
+         Lemma_Abs_Div_Commutation (Big (X) * Big (Y), Big (Z));
+         Lemma_Abs_Rem_Commutation (Big (X) * Big (Y), Big (Z));
+      end Prove_Z_Low;
+
+   --  Start of processing for Scaled_Divide
+
    begin
+      if Z = 0 then
+         pragma Assert (Is_Division_By_Zero_Case);
+         Raise_Error;
+         pragma Annotate
+           (GNATprove, Intentional, "call to nonreturning subprogram",
+            "Constraint_Error is raised in case of division by zero");
+      end if;
+
+      Quot := Big (X) * Big (Y) / Big (Z);
+      Big_R := Big (X) * Big (Y) rem Big (Z);
+      if Round then
+         Big_Q := Round_Quotient (Big (X) * Big (Y), Big (Z), Quot, Big_R);
+      else
+         Big_Q := Quot;
+      end if;
+
       --  First do the multiplication, giving the four digit dividend
 
+      Lemma_Abs_Mult_Commutation (Big (X), Big (Y));
+      Lemma_Abs_Commutation (X);
+      Lemma_Abs_Commutation (Y);
+      Lemma_Mult_Decomposition (Mult, Xu, Yu, Xhi, Xlo, Yhi, Ylo);
+
       T1 := Xlo * Ylo;
       D (4) := Lo (T1);
       D (3) := Hi (T1);
 
+      Lemma_Hi_Lo (T1, D (3), D (4));
+
       if Yhi /= 0 then
          T1 := Xlo * Yhi;
+
+         Lemma_Hi_Lo (T1, Hi (T1), Lo (T1));
+
          T2 := D (3) + Lo (T1);
+
+         Lemma_Mult_Distribution (Big_2xx32,
+                                  Big (Double_Uns (D (3))),
+                                  Big (Double_Uns (Lo (T1))));
+         Lemma_Hi_Lo (T2, Hi (T2), Lo (T2));
+
          D (3) := Lo (T2);
          D (2) := Hi (T1) + Hi (T2);
 
+         pragma Assert (Double_Uns (Hi (T1)) + Hi (T2) = Double_Uns (D (2)));
+         Lemma_Add_Commutation (Double_Uns (Hi (T1)), Hi (T2));
+         pragma Assert
+           (Big (Double_Uns (Hi (T1))) + Big (Double_Uns (Hi (T2))) =
+              Big (Double_Uns (D (2))));
+
          if Xhi /= 0 then
             T1 := Xhi * Ylo;
+
+            Lemma_Hi_Lo (T1, Hi (T1), Lo (T1));
+
             T2 := D (3) + Lo (T1);
+
+            Lemma_Hi_Lo (T2, Hi (T2), Lo (T2));
+
             D (3) := Lo (T2);
             T3 := D (2) + Hi (T1);
+
+            Lemma_Add_Commutation (T3, Hi (T2));
+
             T3 := T3 + Hi (T2);
-            D (2) := Lo (T3);
-            D (1) := Hi (T3);
+            T2 := Double_Uns'(Xhi * Yhi);
 
-            T1 := (D (1) & D (2)) + Double_Uns'(Xhi * Yhi);
-            D (1) := Hi (T1);
+            Lemma_Hi_Lo (T2, Hi (T2), Lo (T2));
+            Lemma_Add_Commutation (T3, Lo (T2));
+
+            T1 := T3 + Lo (T2);
             D (2) := Lo (T1);
 
+            Lemma_Hi_Lo (T1, Hi (T1), D (2));
+
+            D (1) := Hi (T2) + Hi (T1);
+
+            pragma Assert
+              (Double_Uns (Hi (T2)) + Hi (T1) = Double_Uns (D (1)));
+            Lemma_Add_Commutation (Double_Uns (Hi (T2)), Hi (T1));
+            pragma Assert
+              (Big (Double_Uns (Hi (T2))) + Big (Double_Uns (Hi (T1))) =
+                   Big (Double_Uns (D (1))));
+
+            pragma Assert (Mult =
+              Big_2xx32 * Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (1)))
+                        + Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (2)))
+                                    + Big_2xx32 * Big (Double_Uns (D (3)))
+                                                + Big (Double_Uns (D (4))));
+
          else
             D (1) := 0;
          end if;
 
+         pragma Assert (Mult =
+           Big_2xx32 * Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (1)))
+                     + Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (2)))
+                                 + Big_2xx32 * Big (Double_Uns (D (3)))
+                                             + Big (Double_Uns (D (4))));
+
       else
          if Xhi /= 0 then
             T1 := Xhi * Ylo;
+
+            Lemma_Hi_Lo (T1, Hi (T1), Lo (T1));
+
             T2 := D (3) + Lo (T1);
+
+            Lemma_Mult_Distribution (Big_2xx32,
+                                     Big (Double_Uns (D (3))),
+                                     Big (Double_Uns (Lo (T1))));
+            Lemma_Hi_Lo (T2, Hi (T2), Lo (T2));
+
             D (3) := Lo (T2);
             D (2) := Hi (T1) + Hi (T2);
 
+            pragma Assert
+              (Double_Uns (Hi (T1)) + Hi (T2) = Double_Uns (D (2)));
+            Lemma_Add_Commutation (Double_Uns (Hi (T1)), Hi (T2));
+            pragma Assert
+              (Big (Double_Uns (Hi (T1))) + Big (Double_Uns (Hi (T2))) =
+                 Big (Double_Uns (D (2))));
+            pragma Assert
+              (Mult = Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (2)))
+                                + Big_2xx32 * Big (Double_Uns (D (3)))
+                                            + Big (Double_Uns (D (4))));
          else
             D (2) := 0;
+
+            pragma Assert
+              (Mult = Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (2)))
+                                + Big_2xx32 * Big (Double_Uns (D (3)))
+                                            + Big (Double_Uns (D (4))));
          end if;
 
          D (1) := 0;
       end if;
 
+      pragma Assert
+        (Mult = Big_2xx32 * Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (1)))
+                          + Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (2)))
+                                      + Big_2xx32 * Big (Double_Uns (D (3)))
+                                                  + Big (Double_Uns (D (4))));
+
       --  Now it is time for the dreaded multiple precision division. First an
       --  easy case, check for the simple case of a one digit divisor.
 
       if Zhi = 0 then
          if D (1) /= 0 or else D (2) >= Zlo then
+            if D (1) > 0 then
+               pragma Assert
+                 (Mult >= Big_2xx32 * Big_2xx32 * Big_2xx32
+                          * Big (Double_Uns (D (1))));
+               pragma Assert (Mult >= Big_2xx64 * Big_2xx32);
+               Lemma_Ge_Commutation (2 ** Single_Size, Zu);
+               pragma Assert (Mult >= Big_2xx64 * Big (Zu));
+            else
+               Lemma_Ge_Commutation (Double_Uns (D (2)), Zu);
+               pragma Assert (Mult >= Big_2xx64 * Big (Zu));
+            end if;
+
+            Prove_Overflow;
             Raise_Error;
+            pragma Annotate
+              (GNATprove, Intentional, "call to nonreturning subprogram",
+               "Constraint_Error is raised in case of overflow");
 
          --  Here we are dividing at most three digits by one digit
 
@@ -456,12 +2148,20 @@ package body System.Arith_Double is
 
             Qu := Lo (T1 / Zlo) & Lo (T2 / Zlo);
             Ru := T2 rem Zlo;
+
+            Prove_Z_Low;
          end if;
 
       --  If divisor is double digit and dividend is too large, raise error
 
       elsif (D (1) & D (2)) >= Zu then
+         Lemma_Hi_Lo (D (1) & D (2), D (1), D (2));
+         Lemma_Ge_Commutation (D (1) & D (2), Zu);
+         Prove_Overflow;
          Raise_Error;
+         pragma Annotate
+           (GNATprove, Intentional, "call to nonreturning subprogram",
+            "Constraint_Error is raised in case of overflow");
 
       --  This is the complex case where we definitely have a double digit
       --  divisor and a dividend of at least three digits. We use the classical
@@ -473,88 +2173,293 @@ package body System.Arith_Double is
          --  First normalize the divisor so that it has the leading bit on.
          --  We do this by finding the appropriate left shift amount.
 
-         Shift := Single_Size / 2;
-         Mask  := Shift_Left (2 ** (Single_Size / 2) - 1, Shift);
+         Shift := Single_Size;
+         Mask  := Single_Uns'Last;
          Scale := 0;
 
-         while Shift /= 0 loop
-            if (Hi (Zu) and Mask) = 0 then
-               Scale := Scale + Shift;
-               Zu := Shift_Left (Zu, Shift);
-            end if;
-
-            Shift := Shift / 2;
-            Mask := Shift_Left (Mask, Shift);
+         pragma Assert (Big_2xx (Scale) = 1);
+
+         while Shift > 1 loop
+            pragma Loop_Invariant (Scale <= Single_Size - Shift);
+            pragma Loop_Invariant ((Hi (Zu) and Mask) /= 0);
+            pragma Loop_Invariant
+              (Mask = Shift_Left (Single_Uns'Last, Single_Size - Shift));
+            pragma Loop_Invariant (Zu = Shift_Left (abs Z, Scale));
+            pragma Loop_Invariant (Big (Zu) =
+              Big (Double_Uns'(abs Z)) * Big_2xx (Scale));
+            pragma Loop_Invariant (Shift mod 2 = 0);
+            pragma Annotate
+              (GNATprove, False_Positive, "loop invariant",
+               "Shift actually is a power of 2");
+            --  Note : this scaling algorithm only works when Single_Size is a
+            --  power of 2.
+
+            declare
+               Shift_Prev : constant Natural := Shift with Ghost;
+               Mask_Prev  : constant Single_Uns := Mask with Ghost;
+               Zu_Prev    : constant Double_Uns := Zu with Ghost;
+
+               procedure Prove_Shifting
+               with
+                 Ghost,
+                 Pre  => Shift <= Single_Size / 2
+                   and then Zu = Shift_Left (Zu_Prev, Shift)
+                   and then Mask_Prev =
+                     Shift_Left (Single_Uns'Last, Single_Size - 2 * Shift)
+                   and then Mask =
+                     Shift_Left (Single_Uns'Last, Single_Size - Shift)
+                   and then (Hi (Zu_Prev) and Mask_Prev and not Mask) /= 0,
+                 Post => (Hi (Zu) and Mask) /= 0;
+
+               --------------------
+               -- Prove_Shifting --
+               --------------------
+
+               procedure Prove_Shifting is null;
+
+            begin
+               Shift := Shift / 2;
+
+               pragma Assert (Shift_Prev = 2 * Shift);
+
+               Mask := Shift_Left (Mask, Shift);
+
+               Lemma_Double_Shift
+                 (Single_Uns'Last, Single_Size - Shift_Prev, Shift);
+
+               if (Hi (Zu) and Mask) = 0 then
+                  Zu := Shift_Left (Zu, Shift);
+
+                  Prove_Shifting;
+                  pragma Assert (Big (Zu_Prev) =
+                    Big (Double_Uns'(abs Z)) * Big_2xx (Scale));
+                  Lemma_Shift_Without_Drop (Zu_Prev, Zu, Mask, Shift);
+                  Lemma_Substitution
+                    (Big (Zu), Big_2xx (Shift),
+                     Big (Zu_Prev), Big (Double_Uns'(abs Z)) * Big_2xx (Scale),
+                     0);
+                  Lemma_Powers_Of_2 (Shift, Scale);
+                  Lemma_Substitution
+                    (Big (Zu), Big (Double_Uns'(abs Z)),
+                     Big_2xx (Shift) * Big_2xx (Scale),
+                     Big_2xx (Shift + Scale), 0);
+                  Lemma_Double_Shift (abs Z, Scale, Shift);
+
+                  Scale := Scale + Shift;
+
+                  pragma Assert
+                    (Big (Zu) = Big (Double_Uns'(abs Z)) * Big_2xx (Scale));
+               end if;
+            end;
          end loop;
 
          Zhi := Hi (Zu);
          Zlo := Lo (Zu);
 
-         pragma Assert (Zhi /= 0);
-         --  We have Hi(Zu)/=0 before normalization. The sequence of Shift_Left
-         --  operations results in the leading bit of Zu being 1 by moving the
-         --  leftmost 1-bit in Zu to leading position, thus Zhi=Hi(Zu)/=0 here.
+         pragma Assert (Shift = 1);
+         pragma Assert (Mask = Shift_Left (Single_Uns'Last, Single_Size - 1));
+         pragma Assert ((Zhi and Mask) /= 0);
+         pragma Assert (Zhi >= 2 ** (Single_Size - 1));
+         pragma Assert (Big (Zu) = Big (Double_Uns'(abs Z)) * Big_2xx (Scale));
+         --  We have Hi (Zu) /= 0 before normalization. The sequence of
+         --  Shift_Left operations results in the leading bit of Zu being 1 by
+         --  moving the leftmost 1-bit in Zu to leading position, thus
+         --  Zhi = Hi (Zu) >= 2 ** (Single_Size - 1) here.
 
          --  Note that when we scale up the dividend, it still fits in four
          --  digits, since we already tested for overflow, and scaling does
          --  not change the invariant that (D (1) & D (2)) < Zu.
 
+         Lemma_Lt_Commutation (D (1) & D (2), abs Z);
+         Lemma_Lt_Mult (Big (D (1) & D (2)),
+                        Big (Double_Uns'(abs Z)), Big_2xx (Scale),
+                        Big_2xx64);
+
          T1 := Shift_Left (D (1) & D (2), Scale);
+         T2 := Shift_Left (Double_Uns (D (3)), Scale);
+         T3 := Shift_Left (Double_Uns (D (4)), Scale);
+
+         Prove_Dividend_Scaling;
+
          D (1) := Hi (T1);
-         T2 := Shift_Left (0 & D (3), Scale);
          D (2) := Lo (T1) or Hi (T2);
-         T3 := Shift_Left (0 & D (4), Scale);
          D (3) := Lo (T2) or Hi (T3);
          D (4) := Lo (T3);
 
-         --  Loop to compute quotient digits, runs twice for Qd(1) and Qd(2)
-
-         for J in 0 .. 1 loop
-
-            --  Compute next quotient digit. We have to divide three digits by
-            --  two digits. We estimate the quotient by dividing the leading
-            --  two digits by the leading digit. Given the scaling we did above
-            --  which ensured the first bit of the divisor is set, this gives
-            --  an estimate of the quotient that is at most two too high.
-
-            Qd (J + 1) := (if D (J + 1) = Zhi
-                           then 2 ** Single_Size - 1
-                           else Lo ((D (J + 1) & D (J + 2)) / Zhi));
-
-            --  Compute amount to subtract
-
-            T1 := Qd (J + 1) * Zlo;
-            T2 := Qd (J + 1) * Zhi;
-            S3 := Lo (T1);
-            T1 := Hi (T1) + Lo (T2);
-            S2 := Lo (T1);
-            S1 := Hi (T1) + Hi (T2);
-
-            --  Adjust quotient digit if it was too high
-
-            --  We use the version of the algorithm in the 2nd Edition of
-            --  "The Art of Computer Programming". This had a bug not
-            --  discovered till 1995, see Vol 2 errata:
-            --     http://www-cs-faculty.stanford.edu/~uno/err2-2e.ps.gz.
-            --  Under rare circumstances the expression in the test could
-            --  overflow. This version was further corrected in 2005, see
-            --  Vol 2 errata:
-            --     http://www-cs-faculty.stanford.edu/~uno/all2-pre.ps.gz.
-            --  This implementation is not impacted by these bugs, due to the
-            --  use of a word-size comparison done in function Le3 instead of
-            --  a comparison on two-word integer quantities in the original
-            --  algorithm.
-
-            loop
-               exit when Le3 (S1, S2, S3, D (J + 1), D (J + 2), D (J + 3));
-               Qd (J + 1) := Qd (J + 1) - 1;
-               Sub3 (S1, S2, S3, 0, Zhi, Zlo);
+         pragma Assert (Mult * Big_2xx (Scale) =
+           Big_2xx32 * Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (1)))
+                     + Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (2)))
+                                 + Big_2xx32 * Big (Double_Uns (D (3)))
+                                             + Big (Double_Uns (D (4))));
+         Lemma_Substitution (Big_2xx64 * Big (Zu), Big_2xx64, Big (Zu),
+                             Big (Double_Uns'(abs Z)) * Big_2xx (Scale), 0);
+         Lemma_Lt_Mult (Mult, Big_2xx64 * Big (Double_Uns'(abs Z)),
+                        Big_2xx (Scale), Big_2xx64 * Big (Zu));
+         Lemma_Div_Lt (Mult * Big_2xx (Scale), Big (Zu), Big_2xx64);
+         Lemma_Substitution (Mult * Big_2xx (Scale), Big_2xx32,
+                             Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (1)))
+                                       + Big_2xx32 * Big (Double_Uns (D (2)))
+                                                   + Big (Double_Uns (D (3))),
+                             Big3 (D (1), D (2), D (3)),
+                             Big (Double_Uns (D (4))));
+
+         --  Loop to compute quotient digits, runs twice for Qd (1) and Qd (2)
+
+         declare
+            Qd1 : Single_Uns := 0 with Ghost;
+         begin
+            for J in 1 .. 2 loop
+               Lemma_Hi_Lo (D (J) & D (J + 1), D (J), D (J + 1));
+               pragma Assert (Big (D (J) & D (J + 1)) < Big (Zu));
+
+               --  Compute next quotient digit. We have to divide three digits
+               --  by two digits. We estimate the quotient by dividing the
+               --  leading two digits by the leading digit. Given the scaling
+               --  we did above which ensured the first bit of the divisor is
+               --  set, this gives an estimate of the quotient that is at most
+               --  two too high.
+
+               if D (J) > Zhi then
+                  Lemma_Lt_Commutation (Zu, D (J) & D (J + 1));
+                  pragma Assert (False);
+
+               elsif D (J) = Zhi then
+                  Qd (J) := Single_Uns'Last;
+
+                  Lemma_Gt_Mult (Big (Zu), Big (D (J) & D (J + 1)) + 1,
+                                 Big_2xx32,
+                                 Big3 (D (J), D (J + 1), D (J + 2)));
+                  Lemma_Div_Lt
+                    (Big3 (D (J), D (J + 1), D (J + 2)), Big_2xx32, Big (Zu));
+
+               else
+                  Qd (J) := Lo ((D (J) & D (J + 1)) / Zhi);
+
+                  Prove_Qd_Calculation_Part_1 (J);
+               end if;
+
+               Lemma_Gt_Mult
+                 (Big (Double_Uns (Qd (J))),
+                  Big3 (D (J), D (J + 1), D (J + 2)) / Big (Zu),
+                  Big (Zu), Big3 (D (J), D (J + 1), D (J + 2)) - Big (Zu));
+
+               --  Compute amount to subtract
+
+               T1 := Qd (J) * Zlo;
+               T2 := Qd (J) * Zhi;
+               S3 := Lo (T1);
+               T3 := Hi (T1) + Lo (T2);
+               S2 := Lo (T3);
+               S1 := Hi (T3) + Hi (T2);
+
+               Prove_Multiplication (Qd (J));
+
+               --  Adjust quotient digit if it was too high
+
+               --  We use the version of the algorithm in the 2nd Edition
+               --  of "The Art of Computer Programming". This had a bug not
+               --  discovered till 1995, see Vol 2 errata:
+               --     http://www-cs-faculty.stanford.edu/~uno/err2-2e.ps.gz.
+               --  Under rare circumstances the expression in the test could
+               --  overflow. This version was further corrected in 2005, see
+               --  Vol 2 errata:
+               --     http://www-cs-faculty.stanford.edu/~uno/all2-pre.ps.gz.
+               --  This implementation is not impacted by these bugs, due
+               --  to the use of a word-size comparison done in function Le3
+               --  instead of a comparison on two-word integer quantities in
+               --  the original algorithm.
+
+               Lemma_Hi_Lo_3 (Zu, Zhi, Zlo);
+
+               while not Le3 (S1, S2, S3, D (J), D (J + 1), D (J + 2)) loop
+                  pragma Loop_Invariant (Qd (J)'Initialized);
+                  pragma Loop_Invariant
+                    (Big3 (S1, S2, S3) = Big (Double_Uns (Qd (J))) * Big (Zu));
+                  pragma Assert (Big3 (S1, S2, S3) > 0);
+                  if Qd (J) = 0 then
+                     pragma Assert (Big3 (S1, S2, S3) = 0);
+                     pragma Assert (False);
+                  end if;
+                  Lemma_Ge_Commutation (Double_Uns (Qd (J)), 1);
+                  Lemma_Ge_Mult
+                    (Big (Double_Uns (Qd (J))), 1, Big (Zu), Big (Zu));
+
+                  Sub3 (S1, S2, S3, 0, Zhi, Zlo);
+
+                  pragma Assert
+                    (Big3 (S1, S2, S3) >
+                       Big3 (D (J), D (J + 1), D (J + 2)) - Big (Zu));
+                  Lemma_Subtract_Commutation (Double_Uns (Qd (J)), 1);
+                  Lemma_Substitution (Big3 (S1, S2, S3), Big (Zu),
+                                      Big (Double_Uns (Qd (J))) - 1,
+                                      Big (Double_Uns (Qd (J) - 1)), 0);
+
+                  Qd (J) := Qd (J) - 1;
+
+                  pragma Assert
+                    (Big3 (S1, S2, S3) = Big (Double_Uns (Qd (J))) * Big (Zu));
+               end loop;
+
+               --  Now subtract S1&S2&S3 from D1&D2&D3 ready for next step
+
+               pragma Assert
+                 (Big3 (S1, S2, S3) = Big (Double_Uns (Qd (J))) * Big (Zu));
+               pragma Assert (Big3 (S1, S2, S3) >
+                                Big3 (D (J), D (J + 1), D (J + 2)) - Big (Zu));
+               Inline_Le3 (S1, S2, S3, D (J), D (J + 1), D (J + 2));
+
+               Sub3 (D (J), D (J + 1), D (J + 2), S1, S2, S3);
+
+               pragma Assert (Big3 (D (J), D (J + 1), D (J + 2)) < Big (Zu));
+               if D (J) > 0 then
+                  pragma Assert (Big_2xx32 * Big_2xx32 = Big_2xx64);
+                  pragma Assert (Big3 (D (J), D (J + 1), D (J + 2)) =
+                                   Big_2xx32 * Big_2xx32
+                                   * Big (Double_Uns (D (J)))
+                                 + Big_2xx32 * Big (Double_Uns (D (J + 1)))
+                                             + Big (Double_Uns (D (J + 2))));
+                  pragma Assert (Big3 (D (J), D (J + 1), D (J + 2)) =
+                                   Big_2xx64 * Big (Double_Uns (D (J)))
+                                 + Big_2xx32 * Big (Double_Uns (D (J + 1)))
+                                             + Big (Double_Uns (D (J + 2))));
+                  pragma Assert (Big3 (D (J), D (J + 1), D (J + 2)) >=
+                                   Big_2xx64 * Big (Double_Uns (D (J))));
+                  Lemma_Ge_Commutation (Double_Uns (D (J)), Double_Uns'(1));
+                  pragma Assert
+                    (Big3 (D (J), D (J + 1), D (J + 2)) >= Big_2xx64);
+                  pragma Assert (False);
+               end if;
+
+               if J = 1 then
+                  Qd1 := Qd (1);
+                  pragma Assert
+                    (Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (1))) = 0);
+                  pragma Assert
+                    (Mult * Big_2xx (Scale) =
+                       Big_2xx32 * Big3 (S1, S2, S3)
+                     + Big3 (D (2), D (3), D (4)));
+                  Lemma_Substitution (Mult * Big_2xx (Scale), Big_2xx32,
+                                      Big3 (S1, S2, S3),
+                                      Big (Double_Uns (Qd1)) * Big (Zu),
+                                      Big3 (D (2), D (3), D (4)));
+               else
+                  pragma Assert (Qd1 = Qd (1));
+                  pragma Assert
+                    (Big_2xx32 * Big_2xx32 * Big (Double_Uns (D (2))) = 0);
+                  pragma Assert
+                    (Mult * Big_2xx (Scale) =
+                       Big_2xx32 * Big (Double_Uns (Qd (1))) * Big (Zu)
+                     + Big3 (S1, S2, S3)
+                     + Big3 (D (2), D (3), D (4)));
+                  pragma Assert
+                    (Mult * Big_2xx (Scale) =
+                       Big_2xx32 * Big (Double_Uns (Qd (1))) * Big (Zu)
+                                 + Big (Double_Uns (Qd (2))) * Big (Zu)
+                     + Big_2xx32 * Big (Double_Uns (D (3)))
+                                 + Big (Double_Uns (D (4))));
+               end if;
             end loop;
-
-            --  Now subtract S1&S2&S3 from D1&D2&D3 ready for next step
-
-            Sub3 (D (J + 1), D (J + 2), D (J + 3), S1, S2, S3);
-         end loop;
+         end;
 
          --  The two quotient digits are now set, and the remainder of the
          --  scaled division is in D3&D4. To get the remainder for the
@@ -564,38 +2469,93 @@ package body System.Arith_Double is
          --  for rounding below.
 
          Qu := Qd (1) & Qd (2);
-         Ru := Shift_Right (D (3) & D (4), Scale);
+         Ru := D (3) & D (4);
+
+         pragma Assert
+           (Mult * Big_2xx (Scale) =
+              Big_2xx32 * Big (Double_Uns (Qd (1))) * Big (Zu)
+                        + Big (Double_Uns (Qd (2))) * Big (Zu)
+            + Big_2xx32 * Big (Double_Uns (D (3)))
+                        + Big (Double_Uns (D (4))));
+         Lemma_Hi_Lo (Qu, Qd (1), Qd (2));
+         Lemma_Hi_Lo (Ru, D (3), D (4));
+         Lemma_Substitution
+           (Mult * Big_2xx (Scale), Big (Zu),
+            Big_2xx32 * Big (Double_Uns (Qd (1))) + Big (Double_Uns (Qd (2))),
+            Big (Qu), Big (Ru));
+         Prove_Rescaling;
+
+         Ru := Shift_Right (Ru, Scale);
+
+         Lemma_Shift_Right (Zu, Scale);
+
          Zu := Shift_Right (Zu, Scale);
+
+         Lemma_Simplify (Big (Double_Uns'(abs Z)), Big_2xx (Scale));
       end if;
 
+      pragma Assert (Big (Ru) = abs Big_R);
+      pragma Assert (Big (Qu) = abs Quot);
+      pragma Assert (Big (Zu) = Big (Double_Uns'(abs Z)));
+
       --  Deal with rounding case
 
-      if Round and then Ru > (Zu - Double_Uns'(1)) / Double_Uns'(2) then
+      if Round then
+         Prove_Rounding_Case;
 
-         --  Protect against wrapping around when rounding, by signaling
-         --  an overflow when the quotient is too large.
+         if Ru > (Zu - Double_Uns'(1)) / Double_Uns'(2) then
+            pragma Assert (abs Big_Q = Big (Qu) + 1);
 
-         if Qu = Double_Uns'Last then
-            Raise_Error;
-         end if;
+            --  Protect against wrapping around when rounding, by signaling
+            --  an overflow when the quotient is too large.
+
+            if Qu = Double_Uns'Last then
+               pragma Assert (abs Big_Q = Big_2xx64);
+               Raise_Error;
+               pragma Annotate
+                 (GNATprove, Intentional, "call to nonreturning subprogram",
+                  "Constraint_Error is raised in case of overflow");
+            end if;
+
+            Lemma_Add_One (Qu);
 
-         Qu := Qu + Double_Uns'(1);
+            Qu := Qu + Double_Uns'(1);
+         end if;
       end if;
 
+      pragma Assert (Big (Qu) = abs Big_Q);
+
       --  Set final signs (RM 4.5.5(27-30))
 
       --  Case of dividend (X * Y) sign positive
 
       if (X >= 0 and then Y >= 0) or else (X < 0 and then Y < 0) then
          R := To_Pos_Int (Ru);
+         pragma Annotate
+           (GNATprove, Intentional, "precondition",
+            "Constraint_Error is raised in case of overflow");
+
          Q := (if Z > 0 then To_Pos_Int (Qu) else To_Neg_Int (Qu));
+         pragma Annotate
+           (GNATprove, Intentional, "precondition",
+            "Constraint_Error is raised in case of overflow");
 
       --  Case of dividend (X * Y) sign negative
 
       else
          R := To_Neg_Int (Ru);
+         pragma Annotate
+           (GNATprove, Intentional, "precondition",
+            "Constraint_Error is raised in case of overflow");
+
          Q := (if Z > 0 then To_Neg_Int (Qu) else To_Pos_Int (Qu));
+         pragma Annotate
+           (GNATprove, Intentional, "precondition",
+            "Constraint_Error is raised in case of overflow");
       end if;
+
+      Prove_Sign_R;
+      Prove_Signs;
    end Scaled_Divide;
 
    ----------
@@ -603,23 +2563,178 @@ package body System.Arith_Double is
    ----------
 
    procedure Sub3 (X1, X2, X3 : in out Single_Uns; Y1, Y2, Y3 : Single_Uns) is
+
+      --  Local ghost variables
+
+      XX1 : constant Single_Uns := X1 with Ghost;
+      XX2 : constant Single_Uns := X2 with Ghost;
+      XX3 : constant Single_Uns := X3 with Ghost;
+
+      --  Local lemmas
+
+      procedure Lemma_Add3_No_Carry (X1, X2, X3, Y1, Y2, Y3 : Single_Uns)
+      with
+        Ghost,
+        Pre  => X1 <= Single_Uns'Last - Y1
+          and then X2 <= Single_Uns'Last - Y2
+          and then X3 <= Single_Uns'Last - Y3,
+        Post => Big3 (X1 + Y1, X2 + Y2, X3 + Y3)
+              = Big3 (X1, X2, X3) + Big3 (Y1, Y2, Y3);
+
+      procedure Lemma_Ge_Expand (X1, X2, X3, Y1, Y2, Y3 : Single_Uns)
+      with
+        Ghost,
+        Pre  => Big3 (X1, X2, X3) >= Big3 (Y1, Y2, Y3),
+        Post => X1 > Y1
+          or else (X1 = Y1 and then X2 > Y2)
+          or else (X1 = Y1 and then X2 = Y2 and then X3 >= Y3);
+
+      procedure Lemma_Sub3_No_Carry (X1, X2, X3, Y1, Y2, Y3 : Single_Uns)
+      with
+        Ghost,
+        Pre  => X1 >= Y1 and then X2 >= Y2 and then X3 >= Y3,
+        Post => Big3 (X1 - Y1, X2 - Y2, X3 - Y3)
+              = Big3 (X1, X2, X3) - Big3 (Y1, Y2, Y3);
+
+      procedure Lemma_Sub3_With_Carry2 (X1, X2, X3, Y2 : Single_Uns)
+      with
+        Ghost,
+        Pre  => X2 < Y2,
+        Post => Big3 (X1, X2 - Y2, X3)
+              = Big3 (X1, X2, X3) + Big3 (1, 0, 0) - Big3 (0, Y2, 0);
+
+      procedure Lemma_Sub3_With_Carry3 (X1, X2, X3, Y3 : Single_Uns)
+      with
+        Ghost,
+        Pre  => X3 < Y3,
+        Post => Big3 (X1, X2, X3 - Y3)
+              = Big3 (X1, X2, X3) + Big3 (0, 1, 0) - Big3 (0, 0, Y3);
+
+      -------------------------
+      -- Lemma_Add3_No_Carry --
+      -------------------------
+
+      procedure Lemma_Add3_No_Carry (X1, X2, X3, Y1, Y2, Y3 : Single_Uns) is
+      begin
+         Lemma_Add_Commutation (Double_Uns (X1), Y1);
+         Lemma_Add_Commutation (Double_Uns (X2), Y2);
+         Lemma_Add_Commutation (Double_Uns (X3), Y3);
+         pragma Assert (Double_Uns (Single_Uns'(X1 + Y1))
+                        = Double_Uns (X1) + Double_Uns (Y1));
+         pragma Assert (Double_Uns (Single_Uns'(X2 + Y2))
+                        = Double_Uns (X2) + Double_Uns (Y2));
+         pragma Assert (Double_Uns (Single_Uns'(X3 + Y3))
+                        = Double_Uns (X3) + Double_Uns (Y3));
+      end Lemma_Add3_No_Carry;
+
+      ---------------------
+      -- Lemma_Ge_Expand --
+      ---------------------
+
+      procedure Lemma_Ge_Expand (X1, X2, X3, Y1, Y2, Y3 : Single_Uns) is null;
+
+      -------------------------
+      -- Lemma_Sub3_No_Carry --
+      -------------------------
+
+      procedure Lemma_Sub3_No_Carry (X1, X2, X3, Y1, Y2, Y3 : Single_Uns) is
+      begin
+         Lemma_Subtract_Commutation (Double_Uns (X1), Double_Uns (Y1));
+         Lemma_Subtract_Commutation (Double_Uns (X2), Double_Uns (Y2));
+         Lemma_Subtract_Commutation (Double_Uns (X3), Double_Uns (Y3));
+      end Lemma_Sub3_No_Carry;
+
+      ----------------------------
+      -- Lemma_Sub3_With_Carry2 --
+      ----------------------------
+
+      procedure Lemma_Sub3_With_Carry2 (X1, X2, X3, Y2 : Single_Uns) is
+         pragma Unreferenced (X1, X3);
+      begin
+         Lemma_Add_Commutation
+           (Double_Uns'(2 ** Single_Size) - Double_Uns (Y2), X2);
+         Lemma_Subtract_Commutation
+           (Double_Uns'(2 ** Single_Size), Double_Uns (Y2));
+      end Lemma_Sub3_With_Carry2;
+
+      ----------------------------
+      -- Lemma_Sub3_With_Carry3 --
+      ----------------------------
+
+      procedure Lemma_Sub3_With_Carry3 (X1, X2, X3, Y3 : Single_Uns) is
+         pragma Unreferenced (X1, X2);
+      begin
+         Lemma_Add_Commutation
+           (Double_Uns'(2 ** Single_Size) - Double_Uns (Y3), X3);
+         Lemma_Subtract_Commutation
+           (Double_Uns'(2 ** Single_Size), Double_Uns (Y3));
+      end Lemma_Sub3_With_Carry3;
+
+   --  Start of processing for Sub3
+
    begin
+      Lemma_Ge_Expand (X1, X2, X3, Y1, Y2, Y3);
+
       if Y3 > X3 then
          if X2 = 0 then
+            pragma Assert (X1 >= 1);
+            Lemma_Sub3_No_Carry (X1, X2, X3, 1, 0, 0);
+
             X1 := X1 - 1;
+
+            pragma Assert
+              (Big3 (X1, X2, X3) = Big3 (XX1, XX2, XX3) - Big3 (1, 0, 0));
+            pragma Assert
+              (Big3 (X1, X2, X3) = Big3 (XX1, XX2, XX3)
+               - Big3 (0, Single_Uns'Last, 0) - Big3 (0, 1, 0));
+            Lemma_Add3_No_Carry (X1, X2, X3, 0, Single_Uns'Last, 0);
+         else
+            Lemma_Sub3_No_Carry (X1, X2, X3, 0, 1, 0);
          end if;
 
          X2 := X2 - 1;
+
+         pragma Assert
+           (Big3 (X1, X2, X3) = Big3 (XX1, XX2, XX3) - Big3 (0, 1, 0));
+         Lemma_Sub3_With_Carry3 (X1, X2, X3, Y3);
+      else
+         Lemma_Sub3_No_Carry (X1, X2, X3, 0, 0, Y3);
       end if;
 
       X3 := X3 - Y3;
 
+      pragma Assert
+        (Big3 (X1, X2, X3) = Big3 (XX1, XX2, XX3) - Big3 (0, 0, Y3));
+
       if Y2 > X2 then
+         pragma Assert (X1 >= 1);
+         Lemma_Sub3_No_Carry (X1, X2, X3, 1, 0, 0);
+
          X1 := X1 - 1;
+
+         pragma Assert
+           (Big3 (X1, X2, X3) = Big3 (XX1, XX2, XX3)
+            - Big3 (0, 0, Y3) - Big3 (1, 0, 0));
+         Lemma_Sub3_With_Carry2 (X1, X2, X3, Y2);
+      else
+         Lemma_Sub3_No_Carry (X1, X2, X3, 0, Y2, 0);
       end if;
 
       X2 := X2 - Y2;
+
+      pragma Assert
+        (Big3 (X1, X2, X3) = Big3 (XX1, XX2, XX3) - Big3 (0, Y2, Y3));
+      pragma Assert (X1 >= Y1);
+      Lemma_Sub3_No_Carry (X1, Y2, X3, Y1, 0, 0);
+
       X1 := X1 - Y1;
+
+      pragma Assert
+        (Big3 (X1, X2, X3) = Big3 (XX1, XX2, XX3)
+         - Big3 (0, Y2, Y3) - Big3 (Y1, 0, 0));
+      Lemma_Add3_No_Carry (0, Y2, Y3, Y1, 0, 0);
+      pragma Assert
+        (Big3 (X1, X2, X3) = Big3 (XX1, XX2, XX3) - Big3 (Y1, Y2, Y3));
    end Sub3;
 
    -------------------------------
@@ -629,18 +2744,126 @@ package body System.Arith_Double is
    function Subtract_With_Ovflo_Check (X, Y : Double_Int) return Double_Int is
       R : constant Double_Int := To_Int (To_Uns (X) - To_Uns (Y));
 
+      --  Local lemmas
+
+      procedure Prove_Negative_X
+      with
+        Ghost,
+        Pre  => X < 0 and then (Y <= 0 or else R < 0),
+        Post => R = X - Y;
+
+      procedure Prove_Non_Negative_X
+      with
+        Ghost,
+        Pre  => X >= 0 and then (Y > 0 or else R >= 0),
+        Post => R = X - Y;
+
+      procedure Prove_Overflow_Case
+      with
+        Ghost,
+        Pre  =>
+          (if X >= 0 then Y <= 0 and then R < 0
+                     else Y > 0 and then R >= 0),
+        Post => not In_Double_Int_Range (Big (X) - Big (Y));
+
+      ----------------------
+      -- Prove_Negative_X --
+      ----------------------
+
+      procedure Prove_Negative_X is
+      begin
+         if X = Double_Int'First then
+            if Y = Double_Int'First or else Y > 0 then
+               null;
+            else
+               pragma Assert
+                 (To_Uns (X) - To_Uns (Y) =
+                    2 ** (Double_Size - 1) + Double_Uns (-Y));
+            end if;
+
+         elsif Y >= 0 or else Y = Double_Int'First then
+            null;
+
+         else
+            pragma Assert
+              (To_Uns (X) - To_Uns (Y) = -Double_Uns (-X) + Double_Uns (-Y));
+         end if;
+      end Prove_Negative_X;
+
+      --------------------------
+      -- Prove_Non_Negative_X --
+      --------------------------
+
+      procedure Prove_Non_Negative_X is
+      begin
+         if Y > 0 then
+            declare
+               Ru : constant Double_Uns := To_Uns (X) - To_Uns (Y);
+            begin
+               pragma Assert (Ru = Double_Uns (X) - Double_Uns (Y));
+               if Ru < 2 ** (Double_Size - 1) then  --  R >= 0
+                  Lemma_Subtract_Double_Uns (X => Y, Y => X);
+                  pragma Assert (Ru = Double_Uns (X - Y));
+
+               elsif Ru = 2 ** (Double_Size - 1) then
+                  pragma Assert (Double_Uns (Y) < 2 ** (Double_Size - 1));
+                  pragma Assert (False);
+
+               else
+                  pragma Assert
+                    (R = -Double_Int (-(Double_Uns (X) - Double_Uns (Y))));
+                  pragma Assert
+                    (R = -Double_Int (-Double_Uns (X) + Double_Uns (Y)));
+                  pragma Assert
+                    (R = -Double_Int (Double_Uns (Y) - Double_Uns (X)));
+               end if;
+            end;
+
+         elsif Y = Double_Int'First then
+            pragma Assert
+              (To_Uns (X) - To_Uns (Y) =
+                 Double_Uns (X) - 2 ** (Double_Size - 1));
+            pragma Assert (False);
+
+         else
+            pragma Assert
+              (To_Uns (X) - To_Uns (Y) = Double_Uns (X) + Double_Uns (-Y));
+         end if;
+      end Prove_Non_Negative_X;
+
+      -------------------------
+      -- Prove_Overflow_Case --
+      -------------------------
+
+      procedure Prove_Overflow_Case is
+      begin
+         if X >= 0 and then Y /= Double_Int'First then
+            pragma Assert
+              (To_Uns (X) - To_Uns (Y) = Double_Uns (X) + Double_Uns (-Y));
+
+         elsif X < 0 and then X /= Double_Int'First then
+            pragma Assert
+              (To_Uns (X) - To_Uns (Y) = -Double_Uns (-X) - Double_Uns (Y));
+         end if;
+      end Prove_Overflow_Case;
+
+   --  Start of processing for Subtract_With_Ovflo_Check
+
    begin
       if X >= 0 then
          if Y > 0 or else R >= 0 then
+            Prove_Non_Negative_X;
             return R;
          end if;
 
       else -- X < 0
          if Y <= 0 or else R < 0 then
+            Prove_Negative_X;
             return R;
          end if;
       end if;
 
+      Prove_Overflow_Case;
       Raise_Error;
    end Subtract_With_Ovflo_Check;
 
diff --git a/gcc/ada/libgnat/s-aridou.ads b/gcc/ada/libgnat/s-aridou.ads
index 0df27ca..7c32f7f 100644
--- a/gcc/ada/libgnat/s-aridou.ads
+++ b/gcc/ada/libgnat/s-aridou.ads
@@ -33,6 +33,9 @@
 --  signed integer values in cases where either overflow checking is required,
 --  or intermediate results are longer than the result type.
 
+with Ada.Numerics.Big_Numbers.Big_Integers_Ghost;
+use Ada.Numerics.Big_Numbers.Big_Integers_Ghost;
+
 generic
 
    type Double_Int is range <>;
@@ -50,27 +53,93 @@ generic
    with function Shift_Left (A : Single_Uns; B : Natural) return Single_Uns
      is <>;
 
-package System.Arith_Double is
-   pragma Pure;
+package System.Arith_Double
+  with Pure, SPARK_Mode
+is
+   --  Preconditions in this unit are meant for analysis only, not for run-time
+   --  checking, so that the expected exceptions are raised. This is enforced
+   --  by setting the corresponding assertion policy to Ignore. Postconditions
+   --  and contract cases should not be executed at runtime as well, in order
+   --  not to slow down the execution of these functions.
+
+   pragma Assertion_Policy (Pre            => Ignore,
+                            Post           => Ignore,
+                            Contract_Cases => Ignore,
+                            Ghost          => Ignore);
+
+   package Signed_Conversion is new Signed_Conversions (Int => Double_Int);
+
+   function Big (Arg : Double_Int) return Big_Integer is
+     (Signed_Conversion.To_Big_Integer (Arg))
+   with Ghost;
+
+   package Unsigned_Conversion is new Unsigned_Conversions (Int => Double_Uns);
 
-   function Add_With_Ovflo_Check (X, Y : Double_Int) return Double_Int;
+   function Big (Arg : Double_Uns) return Big_Integer is
+     (Unsigned_Conversion.To_Big_Integer (Arg))
+   with Ghost;
+
+   function In_Double_Int_Range (Arg : Big_Integer) return Boolean is
+     (In_Range (Arg, Big (Double_Int'First), Big (Double_Int'Last)))
+   with Ghost;
+
+   function Add_With_Ovflo_Check (X, Y : Double_Int) return Double_Int
+   with
+     Pre  => In_Double_Int_Range (Big (X) + Big (Y)),
+     Post => Add_With_Ovflo_Check'Result = X + Y;
    --  Raises Constraint_Error if sum of operands overflows Double_Int,
    --  otherwise returns the signed integer sum.
 
-   function Subtract_With_Ovflo_Check (X, Y : Double_Int) return Double_Int;
+   function Subtract_With_Ovflo_Check (X, Y : Double_Int) return Double_Int
+   with
+     Pre  => In_Double_Int_Range (Big (X) - Big (Y)),
+     Post => Subtract_With_Ovflo_Check'Result = X - Y;
    --  Raises Constraint_Error if difference of operands overflows Double_Int,
    --  otherwise returns the signed integer difference.
 
-   function Multiply_With_Ovflo_Check (X, Y : Double_Int) return Double_Int;
+   function Multiply_With_Ovflo_Check (X, Y : Double_Int) return Double_Int
+   with
+     Pre  => In_Double_Int_Range (Big (X) * Big (Y)),
+     Post => Multiply_With_Ovflo_Check'Result = X * Y;
    pragma Convention (C, Multiply_With_Ovflo_Check);
    --  Raises Constraint_Error if product of operands overflows Double_Int,
    --  otherwise returns the signed integer product. Gigi may also call this
    --  routine directly.
 
+   function Same_Sign (X, Y : Big_Integer) return Boolean is
+     (X = Big (Double_Int'(0))
+        or else Y = Big (Double_Int'(0))
+        or else (X < Big (Double_Int'(0))) = (Y < Big (Double_Int'(0))))
+   with Ghost;
+
+   function Round_Quotient (X, Y, Q, R : Big_Integer) return Big_Integer is
+     (if abs R > (abs Y - Big (Double_Int'(1))) / Big (Double_Int'(2)) then
+       (if Same_Sign (X, Y) then Q + Big (Double_Int'(1))
+        else Q - Big (Double_Int'(1)))
+      else
+        Q)
+   with
+     Ghost,
+     Pre => Y /= 0 and then Q = X / Y and then R = X rem Y;
+
    procedure Scaled_Divide
      (X, Y, Z : Double_Int;
       Q, R    : out Double_Int;
-      Round   : Boolean);
+      Round   : Boolean)
+   with
+     Pre  => Z /= 0
+       and then In_Double_Int_Range
+         (if Round then Round_Quotient (Big (X) * Big (Y), Big (Z),
+                                        Big (X) * Big (Y) / Big (Z),
+                                        Big (X) * Big (Y) rem Big (Z))
+          else Big (X) * Big (Y) / Big (Z)),
+     Post => Big (R) = Big (X) * Big (Y) rem Big (Z)
+       and then
+         (if Round then
+            Big (Q) = Round_Quotient (Big (X) * Big (Y), Big (Z),
+                                      Big (X) * Big (Y) / Big (Z), Big (R))
+          else
+            Big (Q) = Big (X) * Big (Y) / Big (Z));
    --  Performs the division of (X * Y) / Z, storing the quotient in Q
    --  and the remainder in R. Constraint_Error is raised if Z is zero,
    --  or if the quotient does not fit in Double_Int. Round indicates if
@@ -82,7 +151,22 @@ package System.Arith_Double is
    procedure Double_Divide
      (X, Y, Z : Double_Int;
       Q, R    : out Double_Int;
-      Round   : Boolean);
+      Round   : Boolean)
+   with
+     Pre  => Y /= 0
+       and then Z /= 0
+       and then In_Double_Int_Range
+         (if Round then Round_Quotient (Big (X), Big (Y) * Big (Z),
+                                        Big (X) / (Big (Y) * Big (Z)),
+                                        Big (X) rem (Big (Y) * Big (Z)))
+          else Big (X) / (Big (Y) * Big (Z))),
+     Post => Big (R) = Big (X) rem (Big (Y) * Big (Z))
+       and then
+         (if Round then
+            Big (Q) = Round_Quotient (Big (X), Big (Y) * Big (Z),
+                                      Big (X) / (Big (Y) * Big (Z)), Big (R))
+          else
+            Big (Q) = Big (X) / (Big (Y) * Big (Z)));
    --  Performs the division X / (Y * Z), storing the quotient in Q and
    --  the remainder in R. Constraint_Error is raised if Y or Z is zero,
    --  or if the quotient does not fit in Double_Int. Round indicates if the
diff --git a/gcc/ada/libgnat/s-arit128.ads b/gcc/ada/libgnat/s-arit128.ads
index fa6fedc..6213cfb 100644
--- a/gcc/ada/libgnat/s-arit128.ads
+++ b/gcc/ada/libgnat/s-arit128.ads
@@ -81,11 +81,4 @@ package System.Arith_128 is
    --  then Q is the rounded quotient. The remainder R is not affected by the
    --  setting of the Round flag.
 
-private
-   --  Make it callable from strub contexts.
-   --  There is a matching setting in trans.c,
-   --  for calls issued by Gigi.
-   pragma Machine_Attribute (Multiply_With_Ovflo_Check128,
-                             "strub", "callable");
-
 end System.Arith_128;
diff --git a/gcc/ada/libgnat/s-arit64.adb b/gcc/ada/libgnat/s-arit64.adb
index 2f24a70..23b76d9 100644
--- a/gcc/ada/libgnat/s-arit64.adb
+++ b/gcc/ada/libgnat/s-arit64.adb
@@ -31,7 +31,9 @@
 
 with System.Arith_Double;
 
-package body System.Arith_64 is
+package body System.Arith_64
+  with SPARK_Mode
+is
 
    subtype Uns64 is Interfaces.Unsigned_64;
    subtype Uns32 is Interfaces.Unsigned_32;
diff --git a/gcc/ada/libgnat/s-arit64.ads b/gcc/ada/libgnat/s-arit64.ads
index 68d2149..fbfd0f6 100644
--- a/gcc/ada/libgnat/s-arit64.ads
+++ b/gcc/ada/libgnat/s-arit64.ads
@@ -36,31 +36,103 @@
 pragma Restrictions (No_Elaboration_Code);
 --  Allow direct call from gigi generated code
 
+--  Preconditions in this unit are meant for analysis only, not for run-time
+--  checking, so that the expected exceptions are raised. This is enforced
+--  by setting the corresponding assertion policy to Ignore. Postconditions
+--  and contract cases should not be executed at runtime as well, in order
+--  not to slow down the execution of these functions.
+
+pragma Assertion_Policy (Pre            => Ignore,
+                         Post           => Ignore,
+                         Contract_Cases => Ignore,
+                         Ghost          => Ignore);
+
+with Ada.Numerics.Big_Numbers.Big_Integers_Ghost;
 with Interfaces;
 
-package System.Arith_64 is
-   pragma Pure;
+package System.Arith_64
+  with Pure, SPARK_Mode
+is
+   use type Ada.Numerics.Big_Numbers.Big_Integers_Ghost.Big_Integer;
+   use type Interfaces.Integer_64;
 
    subtype Int64 is Interfaces.Integer_64;
 
-   function Add_With_Ovflo_Check64 (X, Y : Int64) return Int64;
+   subtype Big_Integer is
+     Ada.Numerics.Big_Numbers.Big_Integers_Ghost.Big_Integer
+     with Ghost;
+
+   package Signed_Conversion is new
+     Ada.Numerics.Big_Numbers.Big_Integers_Ghost.Signed_Conversions
+     (Int => Int64);
+
+   function Big (Arg : Int64) return Big_Integer is
+     (Signed_Conversion.To_Big_Integer (Arg))
+   with Ghost;
+
+   function In_Int64_Range (Arg : Big_Integer) return Boolean is
+     (Ada.Numerics.Big_Numbers.Big_Integers_Ghost.In_Range
+       (Arg, Big (Int64'First), Big (Int64'Last)))
+   with Ghost;
+
+   function Add_With_Ovflo_Check64 (X, Y : Int64) return Int64
+   with
+     Pre  => In_Int64_Range (Big (X) + Big (Y)),
+     Post => Add_With_Ovflo_Check64'Result = X + Y;
    --  Raises Constraint_Error if sum of operands overflows 64 bits,
    --  otherwise returns the 64-bit signed integer sum.
 
-   function Subtract_With_Ovflo_Check64 (X, Y : Int64) return Int64;
+   function Subtract_With_Ovflo_Check64 (X, Y : Int64) return Int64
+   with
+     Pre  => In_Int64_Range (Big (X) - Big (Y)),
+     Post => Subtract_With_Ovflo_Check64'Result = X - Y;
    --  Raises Constraint_Error if difference of operands overflows 64
    --  bits, otherwise returns the 64-bit signed integer difference.
 
-   function Multiply_With_Ovflo_Check64 (X, Y : Int64) return Int64;
+   function Multiply_With_Ovflo_Check64 (X, Y : Int64) return Int64
+   with
+     Pure_Function,
+     Pre  => In_Int64_Range (Big (X) * Big (Y)),
+     Post => Multiply_With_Ovflo_Check64'Result = X * Y;
    pragma Export (C, Multiply_With_Ovflo_Check64, "__gnat_mulv64");
    --  Raises Constraint_Error if product of operands overflows 64
    --  bits, otherwise returns the 64-bit signed integer product.
    --  Gigi may also call this routine directly.
 
+   function Same_Sign (X, Y : Big_Integer) return Boolean is
+     (X = Big (Int64'(0))
+        or else Y = Big (Int64'(0))
+        or else (X < Big (Int64'(0))) = (Y < Big (Int64'(0))))
+   with Ghost;
+
+   function Round_Quotient (X, Y, Q, R : Big_Integer) return Big_Integer is
+     (if abs R > (abs Y - Big (Int64'(1))) / Big (Int64'(2)) then
+       (if Same_Sign (X, Y) then Q + Big (Int64'(1))
+        else Q - Big (Int64'(1)))
+      else
+        Q)
+   with
+     Ghost,
+     Pre => Y /= 0 and then Q = X / Y and then R = X rem Y;
+
    procedure Scaled_Divide64
      (X, Y, Z : Int64;
       Q, R    : out Int64;
-      Round   : Boolean);
+      Round   : Boolean)
+   with
+     Pre  => Z /= 0
+       and then In_Int64_Range
+         (if Round then Round_Quotient (Big (X) * Big (Y), Big (Z),
+                                        Big (X) * Big (Y) / Big (Z),
+                                        Big (X) * Big (Y) rem Big (Z))
+          else Big (X) * Big (Y) / Big (Z)),
+     Post => Big (R) = Big (X) * Big (Y) rem Big (Z)
+       and then
+         (if Round then
+            Big (Q) = Round_Quotient (Big (X) * Big (Y), Big (Z),
+                                      Big (X) * Big (Y) / Big (Z), Big (R))
+          else
+            Big (Q) = Big (X) * Big (Y) / Big (Z));
    --  Performs the division of (X * Y) / Z, storing the quotient in Q
    --  and the remainder in R. Constraint_Error is raised if Z is zero,
    --  or if the quotient does not fit in 64 bits. Round indicates if
@@ -78,7 +150,22 @@ package System.Arith_64 is
    procedure Double_Divide64
      (X, Y, Z : Int64;
       Q, R    : out Int64;
-      Round   : Boolean);
+      Round   : Boolean)
+   with
+     Pre  => Y /= 0
+       and then Z /= 0
+       and then In_Int64_Range
+         (if Round then Round_Quotient (Big (X), Big (Y) * Big (Z),
+                                        Big (X) / (Big (Y) * Big (Z)),
+                                        Big (X) rem (Big (Y) * Big (Z)))
+          else Big (X) / (Big (Y) * Big (Z))),
+     Post => Big (R) = Big (X) rem (Big (Y) * Big (Z))
+       and then
+         (if Round then
+            Big (Q) = Round_Quotient (Big (X), Big (Y) * Big (Z),
+                                      Big (X) / (Big (Y) * Big (Z)), Big (R))
+          else
+            Big (Q) = Big (X) / (Big (Y) * Big (Z)));
    --  Performs the division X / (Y * Z), storing the quotient in Q and
    --  the remainder in R. Constraint_Error is raised if Y or Z is zero,
    --  or if the quotient does not fit in 64 bits. Round indicates if the
@@ -93,11 +180,4 @@ package System.Arith_64 is
       Round   : Boolean) renames Double_Divide64;
    --  Renamed procedure to preserve compatibility with earlier versions
 
-private
-   --  Make it callable from strub contexts.
-   --  There is a matching setting in trans.c,
-   --  for calls issued by Gigi.
-   pragma Machine_Attribute (Multiply_With_Ovflo_Check64,
-                             "strub", "callable");
-
 end System.Arith_64;
diff --git a/gcc/ada/libgnat/s-atocou.ads b/gcc/ada/libgnat/s-atocou.ads
index 9488b6d..9b25cba 100644
--- a/gcc/ada/libgnat/s-atocou.ads
+++ b/gcc/ada/libgnat/s-atocou.ads
@@ -31,6 +31,8 @@
 
 --  This package provides atomic counter on platforms where it is supported:
 --    - all Alpha platforms
+--    - all AARCH64 platforms
+--    - all ARM platforms
 --    - all ia64 platforms
 --    - all PowerPC platforms
 --    - all SPARC V9 platforms
diff --git a/gcc/ada/libgnat/s-regexp.adb b/gcc/ada/libgnat/s-regexp.adb
index 51ff39c..8f1940a 100644
--- a/gcc/ada/libgnat/s-regexp.adb
+++ b/gcc/ada/libgnat/s-regexp.adb
@@ -122,7 +122,7 @@ package body System.Regexp is
    is
       S : String := Pattern;
       --  The pattern which is really compiled (when the pattern is case
-      --  insensitive, we convert this string to lower-cases
+      --  insensitive, we convert this string to lower-cases).
 
       Map : Mapping := (others => 0);
       --  Mapping between characters and columns in the tables
@@ -209,8 +209,59 @@ package body System.Regexp is
          --  The last occurrence of an opening parenthesis, if Glob=False,
          --  or the last occurrence of an opening curly brace, if Glob=True.
 
+         procedure Find_Close_Bracket;
+         --  Go through the pattern to find a closing bracket. Raise an
+         --  exception if none is found.
+
          procedure Raise_Exception_If_No_More_Chars (K : Integer := 0);
-         --  If no more characters are raised, call Raise_Exception
+         --  If J + K > S'Last then call Raise_Exception
+
+         ------------------------
+         -- Find_Close_Bracket --
+         ------------------------
+
+         procedure Find_Close_Bracket is
+            Possible_Range_Start : Boolean := True;
+            --  Set True everywhere a range character '-' can occur
+
+         begin
+            loop
+               exit when S (J) = Close_Bracket;
+
+               Raise_Exception_If_No_More_Chars (1);
+               --  The current character is not a close_bracket, thus it should
+               --  be followed by at least one more char. If not, no close
+               --  bracket is present and the pattern is ill-formed.
+
+               if S (J) = '-' and then S (J + 1) /= Close_Bracket then
+                  if not Possible_Range_Start then
+                     Raise_Exception
+                        ("No mix of ranges is allowed in "
+                        & "regular expression", J);
+                  end if;
+
+                  J := J + 1;
+                  Raise_Exception_If_No_More_Chars (1);
+
+                  Possible_Range_Start := False;
+                  --  Range cannot be followed by '-' character,
+                  --  except as last character in the set.
+
+               else
+                  Possible_Range_Start := True;
+               end if;
+
+               if S (J) = '\' then
+                  J := J + 1;
+                  Raise_Exception_If_No_More_Chars (1);
+                  --  We ignore the next character and need to check we have
+                  --  one more available character. This is necessary for
+                  --  the erroneous [\] pattern which stands for [\]] or [\\].
+               end if;
+
+               J := J + 1;
+            end loop;
+         end Find_Close_Bracket;
 
          --------------------------------------
          -- Raise_Exception_If_No_More_Chars --
@@ -240,63 +291,23 @@ package body System.Regexp is
                      end if;
                   end if;
 
-                  --  The first character never has a special meaning
-
+                  --  Characters ']' and '-' are meant as literals when first
+                  --  in the list.  As such, they have no special meaning and
+                  --  we pass them.
                   if S (J) = ']' or else S (J) = '-' then
                      J := J + 1;
                      Raise_Exception_If_No_More_Chars;
                   end if;
 
-                  --  The set of characters cannot be empty
-
                   if S (J) = ']' then
+                     --  ??? This message is misleading since the check forbids
+                     --  the sets []] and [-] but not the empty set [].
                      Raise_Exception
                        ("Set of characters cannot be empty in regular "
                           & "expression", J);
                   end if;
 
-                  declare
-                     Possible_Range_Start : Boolean := True;
-                     --  Set True everywhere a range character '-' can occur
-
-                  begin
-                     loop
-                        exit when S (J) = Close_Bracket;
-
-                        --  The current character should be followed by a
-                        --  closing bracket.
-
-                        Raise_Exception_If_No_More_Chars (1);
-
-                        if S (J) = '-'
-                          and then S (J + 1) /= Close_Bracket
-                        then
-                           if not Possible_Range_Start then
-                              Raise_Exception
-                                ("No mix of ranges is allowed in "
-                                   & "regular expression", J);
-                           end if;
-
-                           J := J + 1;
-                           Raise_Exception_If_No_More_Chars;
-
-                           --  Range cannot be followed by '-' character,
-                           --  except as last character in the set.
-
-                           Possible_Range_Start := False;
-
-                        else
-                           Possible_Range_Start := True;
-                        end if;
-
-                        if S (J) = '\' then
-                           J := J + 1;
-                           Raise_Exception_If_No_More_Chars;
-                        end if;
-
-                        J := J + 1;
-                     end loop;
-                  end;
+                  Find_Close_Bracket;
 
                   --  A closing bracket can end an elmt or term
 
@@ -431,6 +442,9 @@ package body System.Regexp is
                            & "applied to a term in regular expression", J);
                      end if;
 
+                     --  A second term must follow
+                     Raise_Exception_If_No_More_Chars (K => 1);
+
                      Past_Elmt := False;
                      Past_Term := False;
                   end if;
diff --git a/gcc/ada/libgnat/s-secsta.ads b/gcc/ada/libgnat/s-secsta.ads
index 6648c23..7d6b1b9 100644
--- a/gcc/ada/libgnat/s-secsta.ads
+++ b/gcc/ada/libgnat/s-secsta.ads
@@ -440,9 +440,4 @@ private
    function Get_Stack_Info (Stack : SS_Stack_Ptr) return Stack_Info;
    --  Obtain the information attributes of secondary stack Stack
 
-   pragma Machine_Attribute (SS_Allocate, "strub", "callable");
-   pragma Machine_Attribute (SS_Mark, "strub", "callable");
-   pragma Machine_Attribute (SS_Release, "strub", "callable");
-   --  Enable these to be called from within strub contexts.
-
 end System.Secondary_Stack;
diff --git a/gcc/ada/libgnat/s-widthu.adb b/gcc/ada/libgnat/s-widthu.adb
index e0e4d17..79a0074 100644
--- a/gcc/ada/libgnat/s-widthu.adb
+++ b/gcc/ada/libgnat/s-widthu.adb
@@ -29,8 +29,8 @@
 --                                                                          --
 ------------------------------------------------------------------------------
 
-with Ada.Numerics.Big_Numbers.Big_Integers;
-use Ada.Numerics.Big_Numbers.Big_Integers;
+with Ada.Numerics.Big_Numbers.Big_Integers_Ghost;
+use Ada.Numerics.Big_Numbers.Big_Integers_Ghost;
 
 function System.Width_U (Lo, Hi : Uns) return Natural is
 
@@ -134,10 +134,13 @@ begin
          W := W + 1;
          Pow := Pow * 10;
 
-         pragma Loop_Variant (Decreases => T);
          pragma Loop_Invariant (W in 3 .. Max_W + 3);
          pragma Loop_Invariant (Pow = Big_10 ** (W - 2));
          pragma Loop_Invariant (Big (T) = Big (T_Init) / Pow);
+         pragma Loop_Variant (Decreases => T);
+         pragma Annotate
+           (CodePeer, False_Positive,
+            "validity check", "confusion on generated code");
       end loop;
 
       declare
diff --git a/gcc/ada/libgnat/system-rtems.ads b/gcc/ada/libgnat/system-rtems.ads
index c083cb5..77423d3 100644
--- a/gcc/ada/libgnat/system-rtems.ads
+++ b/gcc/ada/libgnat/system-rtems.ads
@@ -109,15 +109,13 @@ package System is
    --             hardware priority levels.  Protected Object ceilings can
    --             override these values.
    --  245        is used by the Interrupt_Manager task
-   --  0          is reserved for the RTEMS IDLE task and really should not
-   --             be accessible from Ada but GNAT initializes
-   --             Current_Priority to 0 so it must be valid
+   --  0          is reserved for the RTEMS IDLE task
 
    Max_Priority           : constant Positive := 244;
    Max_Interrupt_Priority : constant Positive := 254;
 
-   subtype Any_Priority       is Integer      range   0 .. 254;
-   subtype Priority           is Any_Priority range   0 .. 244;
+   subtype Any_Priority       is Integer      range   1 .. 254;
+   subtype Priority           is Any_Priority range   1 .. 244;
    subtype Interrupt_Priority is Any_Priority range 245 .. 254;
 
    Default_Priority : constant Priority := 122;
diff --git a/gcc/ada/namet-sp.adb b/gcc/ada/namet-sp.adb
index bc145ff..f10373f 100644
--- a/gcc/ada/namet-sp.adb
+++ b/gcc/ada/namet-sp.adb
@@ -23,6 +23,8 @@
 --                                                                          --
 ------------------------------------------------------------------------------
 
+with Aspects;
+with Snames;
 with System.WCh_Cnv; use System.WCh_Cnv;
 
 with GNAT.UTF_32_Spelling_Checker;
@@ -44,6 +46,44 @@ package body Namet.Sp is
    --  either Name_Buffer or Name_Len. The result is in Result (1 .. Length).
    --  The caller must ensure that the result buffer is long enough.
 
+   ------------------------
+   -- Aspect_Spell_Check --
+   ------------------------
+
+   function Aspect_Spell_Check (Name : Name_Id) return Boolean is
+     (Aspect_Spell_Check (Name) /= No_Name);
+
+   function Aspect_Spell_Check (Name : Name_Id) return Name_Id is
+      use Aspects;
+   begin
+      for J in Aspect_Id_Exclude_No_Aspect loop
+         if Is_Bad_Spelling_Of (Name, Aspect_Names (J)) then
+            return Aspect_Names (J);
+         end if;
+      end loop;
+
+      return No_Name;
+   end Aspect_Spell_Check;
+
+   ---------------------------
+   -- Attribute_Spell_Check --
+   ---------------------------
+
+   function Attribute_Spell_Check (N : Name_Id) return Boolean is
+     (Attribute_Spell_Check (N) /= No_Name);
+
+   function Attribute_Spell_Check (N : Name_Id) return Name_Id is
+      use Snames;
+   begin
+      for J in First_Attribute_Name .. Last_Attribute_Name loop
+         if Is_Bad_Spelling_Of (N, J) then
+            return J;
+         end if;
+      end loop;
+
+      return No_Name;
+   end Attribute_Spell_Check;
+
    ----------------------------
    -- Get_Name_String_UTF_32 --
    ----------------------------
diff --git a/gcc/ada/namet-sp.ads b/gcc/ada/namet-sp.ads
index 2953aa7..23dbd2b 100644
--- a/gcc/ada/namet-sp.ads
+++ b/gcc/ada/namet-sp.ads
@@ -31,6 +31,20 @@
 
 package Namet.Sp is
 
+   function Aspect_Spell_Check (Name : Name_Id) return Boolean;
+   --  Returns True, if Name is a misspelling of some aspect name
+
+   function Aspect_Spell_Check (Name : Name_Id) return Name_Id;
+   --  Returns a possible correction, if Name is a misspelling of some aspect
+   --  name. If not, return No_Name.
+
+   function Attribute_Spell_Check (N : Name_Id) return Boolean;
+   --  Returns True, if Name is a misspelling of some attribute name
+
+   function Attribute_Spell_Check (N : Name_Id) return Name_Id;
+   --  Returns a possible correction, if Name is a misspelling of some
+   --  attribute name. If not, return No_Name.
+
    function Is_Bad_Spelling_Of (Found, Expect : Name_Id) return Boolean;
    --  Compares two identifier names from the names table, and returns True if
    --  Found is a plausible misspelling of Expect. This function properly deals
diff --git a/gcc/ada/par-ch13.adb b/gcc/ada/par-ch13.adb
index 616d398..227696a 100644
--- a/gcc/ada/par-ch13.adb
+++ b/gcc/ada/par-ch13.adb
@@ -47,28 +47,10 @@ package body Ch13 is
       Scan_State : Saved_Scan_State;
       Result     : Boolean;
 
-      function Possible_Misspelled_Aspect return Boolean;
-      --  Returns True, if Token_Name is a misspelling of some aspect name
-
       function With_Present return Boolean;
       --  Returns True if WITH is present, indicating presence of aspect
       --  specifications. Also allows incorrect use of WHEN in place of WITH.
 
-      --------------------------------
-      -- Possible_Misspelled_Aspect --
-      --------------------------------
-
-      function Possible_Misspelled_Aspect return Boolean is
-      begin
-         for J in Aspect_Id_Exclude_No_Aspect loop
-            if Is_Bad_Spelling_Of (Token_Name, Aspect_Names (J)) then
-               return True;
-            end if;
-         end loop;
-
-         return False;
-      end Possible_Misspelled_Aspect;
-
       ------------------
       -- With_Present --
       ------------------
@@ -89,7 +71,7 @@ package body Ch13 is
                Scan; -- past WHEN
 
                if Token = Tok_Identifier
-                 and then Get_Aspect_Id (Token_Name) /= No_Aspect
+                 and then Is_Aspect_Id (Token_Name)
                then
                   Error_Msg_SC ("WHEN should be WITH");
                   Restore_Scan_State (Scan_State);
@@ -149,8 +131,8 @@ package body Ch13 is
       --  specification is ill-formed.
 
       elsif not Strict then
-         if Get_Aspect_Id (Token_Name) /= No_Aspect
-           or else Possible_Misspelled_Aspect
+         if Is_Aspect_Id (Token_Name)
+           or else Aspect_Spell_Check (Token_Name)
          then
             Result := True;
          else
@@ -164,7 +146,7 @@ package body Ch13 is
       --  is still an aspect specification so we give an appropriate message.
 
       else
-         if Get_Aspect_Id (Token_Name) = No_Aspect then
+         if not Is_Aspect_Id (Token_Name) then
             Result := False;
 
          else
@@ -271,21 +253,10 @@ package body Ch13 is
             begin
                Check_Restriction (Msg_Issued, No_Unrecognized_Aspects, Aspect);
                if not Msg_Issued then
-                  Error_Msg_Warn := not Debug_Flag_2;
-                  Error_Msg_N
-                    ("<<& is not a valid aspect identifier", Token_Node);
-                  OK := False;
+                  Bad_Aspect (Token_Node, Token_Name, not Debug_Flag_2);
 
-                  --  Check bad spelling
+                  OK := False;
 
-                  for J in Aspect_Id_Exclude_No_Aspect loop
-                     if Is_Bad_Spelling_Of (Token_Name, Aspect_Names (J)) then
-                        Error_Msg_Name_1 := Aspect_Names (J);
-                        Error_Msg_N -- CODEFIX
-                          ("\<<possible misspelling of%", Token_Node);
-                        exit;
-                     end if;
-                  end loop;
                end if;
             end;
 
@@ -456,7 +427,7 @@ package body Ch13 is
                            --         Aspect => ...
 
                            if Token = Tok_Identifier
-                             and then Get_Aspect_Id (Token_Name) /= No_Aspect
+                             and then Is_Aspect_Id (Token_Name)
                            then
                               Restore_Scan_State (Scan_State);
 
@@ -588,7 +559,7 @@ package body Ch13 is
          --  and proceed to the next aspect.
 
          elsif Token = Tok_Identifier
-           and then Get_Aspect_Id (Token_Name) /= No_Aspect
+           and then Is_Aspect_Id (Token_Name)
          then
             declare
                Scan_State : Saved_Scan_State;
@@ -626,7 +597,7 @@ package body Ch13 is
                Scan; -- past semicolon
 
                if Token = Tok_Identifier
-                 and then Get_Aspect_Id (Token_Name) /= No_Aspect
+                 and then Is_Aspect_Id (Token_Name)
                then
                   Scan; -- past identifier
 
diff --git a/gcc/ada/par-ch4.adb b/gcc/ada/par-ch4.adb
index b6cc1a0..545d83b 100644
--- a/gcc/ada/par-ch4.adb
+++ b/gcc/ada/par-ch4.adb
@@ -2892,8 +2892,10 @@ package body Ch4 is
                if Token_Is_At_Start_Of_Line
                  and then not
                    (Ada_Version >= Ada_2012
-                     and then Style_Check_Indentation /= 0
-                     and then Start_Column rem Style_Check_Indentation /= 0)
+                      and then
+                        (Style_Check_Indentation = 0
+                           or else
+                             Start_Column rem Style_Check_Indentation /= 0))
                then
                   Error_Msg_AP ("missing operand");
                   return Error;
diff --git a/gcc/ada/par-sync.adb b/gcc/ada/par-sync.adb
index 4ad4627..05188a7 100644
--- a/gcc/ada/par-sync.adb
+++ b/gcc/ada/par-sync.adb
@@ -172,7 +172,7 @@ package body Sync is
                --  current malformed aspect has been successfully skipped.
 
                if Token = Tok_Identifier
-                 and then Get_Aspect_Id (Token_Name) /= No_Aspect
+                 and then Is_Aspect_Id (Token_Name)
                then
                   Restore_Scan_State (Scan_State);
                   exit;
diff --git a/gcc/ada/par-util.adb b/gcc/ada/par-util.adb
index f4179b9..1d7283c 100644
--- a/gcc/ada/par-util.adb
+++ b/gcc/ada/par-util.adb
@@ -816,7 +816,7 @@ package body Util is
             C : constant Entity_Id := Current_Entity (N);
          begin
             if Present (C) and then Sloc (C) = Standard_Location then
-               Error_Msg_N ("redefinition of entity& in Standard?K?", N);
+               Error_Msg_N ("redefinition of entity& in Standard?.k?", N);
             end if;
          end;
       end if;
diff --git a/gcc/ada/repinfo.adb b/gcc/ada/repinfo.adb
index 83d9681..75af649 100644
--- a/gcc/ada/repinfo.adb
+++ b/gcc/ada/repinfo.adb
@@ -38,6 +38,7 @@ with Output;         use Output;
 with Osint.C;        use Osint.C;
 with Sem_Aux;        use Sem_Aux;
 with Sem_Eval;       use Sem_Eval;
+with Sem_Util;
 with Sinfo;          use Sinfo;
 with Sinfo.Nodes;    use Sinfo.Nodes;
 with Sinfo.Utils;    use Sinfo.Utils;
@@ -426,11 +427,14 @@ package body Repinfo is
          end if;
 
       --  Alignment is not always set for task, protected, and class-wide
-      --  types.
+      --  types. Representation aspects are not computed for types in a
+      --  generic unit.
 
       else
          pragma Assert
-           (Is_Concurrent_Type (Ent) or else Is_Class_Wide_Type (Ent));
+           (Is_Concurrent_Type (Ent) or else
+              Is_Class_Wide_Type (Ent) or else
+              Sem_Util.In_Generic_Scope (Ent));
       end if;
    end List_Common_Type_Info;
 
@@ -571,7 +575,7 @@ package body Repinfo is
                --  as for some Java bindings and for generic instances).
 
                if Ekind (E) = E_Package then
-                  if No (Renamed_Object (E)) then
+                  if No (Renamed_Entity (E)) then
                      List_Entities (E, Bytes_Big_Endian);
                   end if;
 
@@ -902,6 +906,13 @@ package body Repinfo is
 
    procedure List_Object_Info (Ent : Entity_Id) is
    begin
+      --  The information has not been computed in a generic unit, so don't try
+      --  to print it.
+
+      if Sem_Util.In_Generic_Scope (Ent) then
+         return;
+      end if;
+
       Write_Separator;
 
       if List_Representation_Info_To_JSON then
@@ -1112,7 +1123,7 @@ package body Repinfo is
          Npos  : constant Uint := Normalized_Position (Ent);
          Fbit  : constant Uint := Normalized_First_Bit (Ent);
          Spos  : Uint;
-         Sbit  : Uint;
+         Sbit  : Uint := No_Uint;
          Lbit  : Uint;
 
       begin
@@ -1176,13 +1187,17 @@ package body Repinfo is
             Write_Str (" range  ");
          end if;
 
-         Sbit := Starting_First_Bit + Fbit;
+         if Known_Static_Normalized_First_Bit (Ent) then
+            Sbit := Starting_First_Bit + Fbit;
 
-         if Sbit >= SSU then
-            Sbit := Sbit - SSU;
-         end if;
+            if Sbit >= SSU then
+               Sbit := Sbit - SSU;
+            end if;
 
-         UI_Write (Sbit, Decimal);
+            UI_Write (Sbit, Decimal);
+         else
+            Write_Unknown_Val;
+         end if;
 
          if List_Representation_Info_To_JSON then
             Write_Line (", ");
@@ -2120,7 +2135,7 @@ package body Repinfo is
 
    function Rep_Value (Val : Node_Ref_Or_Val; D : Discrim_List) return Uint is
 
-      function B (Val : Boolean) return Uint;
+      function B (Val : Boolean) return Ubool;
       --  Returns Uint_0 for False, Uint_1 for True
 
       function T (Val : Node_Ref_Or_Val) return Boolean;
@@ -2141,7 +2156,7 @@ package body Repinfo is
       -- B --
       -------
 
-      function B (Val : Boolean) return Uint is
+      function B (Val : Boolean) return Ubool is
       begin
          if Val then
             return Uint_1;
diff --git a/gcc/ada/repinfo.ads b/gcc/ada/repinfo.ads
index 606bba4..d07208e 100644
--- a/gcc/ada/repinfo.ads
+++ b/gcc/ada/repinfo.ads
@@ -118,12 +118,12 @@ package Repinfo is
    --  this field is done only in -gnatR3 mode, and in other modes, the value
    --  is set to Uint_Minus_1.
 
-   subtype Node_Ref is Uint;
+   subtype Node_Ref is Unegative;
    --  Subtype used for negative Uint values used to represent nodes
 
    subtype Node_Ref_Or_Val is Uint;
-   --  Subtype used for values that can either be a Node_Ref (negative)
-   --  or a value (non-negative)
+   --  Subtype used for values that can be a Node_Ref (negative) or a value
+   --  (non-negative) or No_Uint.
 
    type TCode is range 0 .. 27;
    --  Type used on Ada side to represent DEFTREECODE values defined in
@@ -306,7 +306,7 @@ package Repinfo is
    --  In the case of components, if the location of the component is static,
    --  then all four fields (Component_Bit_Offset, Normalized_Position, Esize,
    --  and Normalized_First_Bit) are set to appropriate values. In the case of
-   --  a non-static component location, Component_Bit_Offset is not used and
+   --  a nonstatic component location, Component_Bit_Offset is not used and
    --  is left set to Unknown. Normalized_Position and Normalized_First_Bit
    --  are set appropriately.
 
diff --git a/gcc/ada/rtsfind.ads b/gcc/ada/rtsfind.ads
index 99f870a..bedea07 100644
--- a/gcc/ada/rtsfind.ads
+++ b/gcc/ada/rtsfind.ads
@@ -60,6 +60,12 @@ package Rtsfind is
    --  the compilation except in the presence of use clauses, which might
    --  result in unexpected ambiguities.
 
+   --  IMPORTANT NOTE: the specs of packages and procedures with'ed using
+   --  this mechanism must not contain private with clauses. This is because
+   --  the special context installation/removal for private with clauses
+   --  only works in a clean environment for compilation, and could lead
+   --  here to removing visibility over lib units in the calling context.
+
    --  NOTE: If RTU_Id is modified, the subtypes of RTU_Id in the package body
    --  might need to be modified. See Get_Unit_Name.
 
diff --git a/gcc/ada/scng.adb b/gcc/ada/scng.adb
index fb42e30b..fd184b5 100644
--- a/gcc/ada/scng.adb
+++ b/gcc/ada/scng.adb
@@ -322,6 +322,49 @@ package body Scng is
       --  Returns True if the scan pointer is pointing to the start of a wide
       --  character sequence, does not modify the scan pointer in any case.
 
+      procedure Check_Bidi (Code : Char_Code);
+      --  Give a warning if Code is a bidirectional character, which can cause
+      --  security vulnerabilities. See the following article:
+      --
+      --  @article{boucher_trojansource_2021,
+      --      title = {Trojan {Source}: {Invisible} {Vulnerabilities}},
+      --      author = {Nicholas Boucher and Ross Anderson},
+      --      year = {2021},
+      --      journal = {Preprint},
+      --      eprint = {2111.00169},
+      --      archivePrefix = {arXiv},
+      --      primaryClass = {cs.CR},
+      --      url = {https://arxiv.org/abs/2111.00169}
+      --  }
+
+      ----------------
+      -- Check_Bidi --
+      ----------------
+
+      type Bidi_Characters is
+        (LRE, RLE, LRO, RLO, LRI, RLI, FSI, PDF, PDI);
+      Bidi_Character_Codes : constant array (Bidi_Characters) of Char_Code :=
+        (LRE => 16#202A#,
+         RLE => 16#202B#,
+         LRO => 16#202D#,
+         RLO => 16#202E#,
+         LRI => 16#2066#,
+         RLI => 16#2067#,
+         FSI => 16#2068#,
+         PDF => 16#202C#,
+         PDI => 16#2069#);
+      --  Above are the bidirectional characters, along with their Unicode code
+      --  points.
+
+      procedure Check_Bidi (Code : Char_Code) is
+      begin
+         for Bidi_Code of Bidi_Character_Codes loop
+            if Code = Bidi_Code then
+               Error_Msg ("??bidirectional wide character", Wptr);
+            end if;
+         end loop;
+      end Check_Bidi;
+
       -----------------------
       -- Double_Char_Token --
       -----------------------
@@ -1070,6 +1113,8 @@ package body Scng is
                   if Err then
                      Error_Illegal_Wide_Character;
                      Code := Get_Char_Code (' ');
+                  else
+                     Check_Bidi (Code);
                   end if;
 
                   Accumulate_Checksum (Code);
@@ -1611,11 +1656,11 @@ package body Scng is
 
                   elsif Start_Of_Wide_Character then
                      declare
-                        Wptr : constant Source_Ptr := Scan_Ptr;
                         Code : Char_Code;
                         Err  : Boolean;
 
                      begin
+                        Wptr := Scan_Ptr;
                         Scan_Wide (Source, Scan_Ptr, Code, Err);
 
                         --  If not well formed wide character, then just skip
@@ -1629,6 +1674,8 @@ package body Scng is
                         elsif Is_UTF_32_Line_Terminator (UTF_32 (Code)) then
                            Scan_Ptr := Wptr;
                            exit;
+                        else
+                           Check_Bidi (Code);
                         end if;
                      end;
 
@@ -1736,7 +1783,6 @@ package body Scng is
                if Start_Of_Wide_Character then
                   Wptr := Scan_Ptr;
                   Scan_Wide (Source, Scan_Ptr, Code, Err);
-                  Accumulate_Checksum (Code);
 
                   if Err then
                      Error_Illegal_Wide_Character;
@@ -1752,8 +1798,12 @@ package body Scng is
                      Error_Msg -- CODEFIX
                        ("(Ada 2005) non-graphic character not permitted " &
                         "in character literal", Wptr);
+                  else
+                     Check_Bidi (Code);
                   end if;
 
+                  Accumulate_Checksum (Code);
+
                   if Source (Scan_Ptr) /= ''' then
                         Error_Msg_S ("missing apostrophe");
                   else
@@ -2701,52 +2751,13 @@ package body Scng is
 
       Tabs_Loop : loop
 
-         --  Inner loop scans past blanks as fast as possible, bumping Scan_Ptr
-         --  past the blanks and adjusting Start_Column to account for them.
-
-         Blanks_Loop : loop
-            if Source (Scan_Ptr) = ' ' then
-               if Source (Scan_Ptr + 1) = ' ' then
-                  if Source (Scan_Ptr + 2) = ' ' then
-                     if Source (Scan_Ptr + 3) = ' ' then
-                        if Source (Scan_Ptr + 4) = ' ' then
-                           if Source (Scan_Ptr + 5) = ' ' then
-                              if Source (Scan_Ptr + 6) = ' ' then
-                                 Scan_Ptr := Scan_Ptr + 7;
-                                 Start_Column := Start_Column + 7;
-                              else
-                                 Scan_Ptr := Scan_Ptr + 6;
-                                 Start_Column := Start_Column + 6;
-                                 exit Blanks_Loop;
-                              end if;
-                           else
-                              Scan_Ptr := Scan_Ptr + 5;
-                              Start_Column := Start_Column + 5;
-                              exit Blanks_Loop;
-                           end if;
-                        else
-                           Scan_Ptr := Scan_Ptr + 4;
-                           Start_Column := Start_Column + 4;
-                           exit Blanks_Loop;
-                        end if;
-                     else
-                        Scan_Ptr := Scan_Ptr + 3;
-                        Start_Column := Start_Column + 3;
-                        exit Blanks_Loop;
-                     end if;
-                  else
-                     Scan_Ptr := Scan_Ptr + 2;
-                     Start_Column := Start_Column + 2;
-                     exit Blanks_Loop;
-                  end if;
-               else
-                  Scan_Ptr := Scan_Ptr + 1;
-                  Start_Column := Start_Column + 1;
-                  exit Blanks_Loop;
-               end if;
-            else
-               exit Blanks_Loop;
-            end if;
+         --  Inner loop scans past blanks, bumping Scan_Ptr past the blanks and
+         --  adjusting Start_Column to account for them.
+
+         Blanks_Loop :
+         while Source (Scan_Ptr) = ' ' loop
+            Scan_Ptr := Scan_Ptr + 1;
+            Start_Column := Start_Column + 1;
          end loop Blanks_Loop;
 
          --  Outer loop keeps going only if a horizontal tab follows
@@ -2771,7 +2782,7 @@ package body Scng is
 
    exception
       when Constraint_Error =>
-         return Start_Column;
+         return Column_Number'Last;
    end Set_Start_Column;
 
 end Scng;
diff --git a/gcc/ada/sem_attr.adb b/gcc/ada/sem_attr.adb
index f2bb12d..2575ddf 100644
--- a/gcc/ada/sem_attr.adb
+++ b/gcc/ada/sem_attr.adb
@@ -74,6 +74,7 @@ with Sinfo.Utils;    use Sinfo.Utils;
 with Sinput;         use Sinput;
 with System;
 with Stringt;        use Stringt;
+with Strub;          use Strub;
 with Style;
 with Stylesw;        use Stylesw;
 with Targparm;       use Targparm;
@@ -2854,7 +2855,7 @@ package body Sem_Attr is
          if Bad_Unordered_Enumeration_Reference (N, P_Base_Type) then
             Error_Msg_Sloc := Sloc (P_Base_Type);
             Error_Msg_NE
-              ("comparison on unordered enumeration type& declared#?U?",
+              ("comparison on unordered enumeration type& declared#?.u?",
                N, P_Base_Type);
          end if;
       end Min_Max;
@@ -11294,6 +11295,27 @@ package body Sem_Attr is
                Resolve (P);
             end if;
 
+            --  Refuse to compute access to variables and constants when that
+            --  would drop the strub mode associated with them, unless they're
+            --  unchecked conversions. We don't have to do this when the types
+            --  of the data objects are annotated: then the access type
+            --  designates the annotated type, and there's no loss. Only when
+            --  the variable is annotated directly that the pragma gets
+            --  attached to the variable, rather than to its type, and then,
+            --  expressing an access-to-annotated-type type to hold the 'Access
+            --  result is not possible without resorting to that very annotated
+            --  type.
+
+            if Attr_Id /= Attribute_Unchecked_Access
+              and then Comes_From_Source (N)
+              and then Is_Entity_Name (P)
+              and then Explicit_Strub_Mode (Entity (P)) = Enabled
+              and then
+                Explicit_Strub_Mode (Designated_Type (Btyp)) = Unspecified
+            then
+               Error_Msg_F ("target access type drops `strub` mode from &", P);
+            end if;
+
             --  X'Access is illegal if X denotes a constant and the access type
             --  is access-to-variable. Same for 'Unchecked_Access. The rule
             --  does not apply to 'Unrestricted_Access. If the reference is a
diff --git a/gcc/ada/sem_aux.adb b/gcc/ada/sem_aux.adb
index dcced7e..e1bcf53 100644
--- a/gcc/ada/sem_aux.adb
+++ b/gcc/ada/sem_aux.adb
@@ -336,10 +336,18 @@ package body Sem_Aux is
 
    function First_Subtype (Typ : Entity_Id) return Entity_Id is
       B   : constant Entity_Id := Base_Type (Typ);
-      F   : constant Node_Id   := Freeze_Node (B);
+      F   : Node_Id := Freeze_Node (B);
       Ent : Entity_Id;
 
    begin
+      --  The freeze node of a ghost type might have been rewritten in a null
+      --  statement by the time gigi calls First_Subtype on the corresponding
+      --  type.
+
+      if Nkind (F) = N_Null_Statement then
+         F := Original_Node (F);
+      end if;
+
       --  If the base type has no freeze node, it is a type in Standard, and
       --  always acts as its own first subtype, except where it is one of the
       --  predefined integer types. If the type is formal, it is also a first
diff --git a/gcc/ada/sem_case.adb b/gcc/ada/sem_case.adb
index 31f14d5..1bd2670 100644
--- a/gcc/ada/sem_case.adb
+++ b/gcc/ada/sem_case.adb
@@ -106,10 +106,26 @@ package body Sem_Case is
 
    package Composite_Case_Ops is
 
+      function Box_Value_Required (Subtyp : Entity_Id) return Boolean;
+      --  If result is True, then the only allowed value (in a choice
+      --  aggregate) for a component of this (sub)type is a box. This rule
+      --  means that such a component can be ignored in case alternative
+      --  selection. This in turn implies that it is ok if the component
+      --  type doesn't meet the usual restrictions, such as not being an
+      --  access/task/protected type, since nobody is going to look
+      --  at it.
+
       function Choice_Count (Alternatives : List_Id) return Nat;
       --  The sum of the number of choices for each alternative in the given
       --  list.
 
+      function Normalized_Case_Expr_Type
+        (Case_Statement : Node_Id) return Entity_Id;
+      --  Usually returns the Etype of the selector expression of the
+      --  case statement. However, in the case of a constrained composite
+      --  subtype with a nonstatic constraint, returns the unconstrained
+      --  base type.
+
       function Scalar_Part_Count (Subtyp : Entity_Id) return Nat;
       --  Given the composite type Subtyp of a case selector, returns the
       --  number of scalar parts in an object of this type. This is the
@@ -119,13 +135,6 @@ package body Sem_Case is
          function Array_Choice_Length (Choice : Node_Id) return Nat;
          --  Given a choice expression of an array type, returns its length.
 
-         function Normalized_Case_Expr_Type
-           (Case_Statement : Node_Id) return Entity_Id;
-         --  Usually returns the Etype of the selector expression of the
-         --  case statement. However, in the case of a constrained array
-         --  subtype with a nonstatic constraint, returns the unconstrained
-         --  array base type.
-
          function Unconstrained_Array_Effective_Length
            (Array_Type : Entity_Id; Case_Statement : Node_Id) return Nat;
          --  If the nominal subtype of the case selector is unconstrained,
@@ -1164,6 +1173,54 @@ package body Sem_Case is
          return UI_To_Int (Len);
       end Static_Array_Length;
 
+      ------------------------
+      -- Box_Value_Required --
+      ------------------------
+
+      function Box_Value_Required (Subtyp : Entity_Id) return Boolean is
+         --  Some of these restrictions will be relaxed eventually, but best
+         --  to initially err in the direction of being too restrictive.
+      begin
+         if Has_Predicates (Subtyp) then
+            return True;
+         elsif Is_Discrete_Type (Subtyp) then
+            if not Is_Static_Subtype (Subtyp) then
+               return True;
+            elsif Is_Enumeration_Type (Subtyp)
+               and then Has_Enumeration_Rep_Clause (Subtyp)
+               --  Maybe enumeration rep clauses can be ignored here?
+            then
+               return True;
+            end if;
+         elsif Is_Array_Type (Subtyp) then
+            if Number_Dimensions (Subtyp) /= 1 then
+               return True;
+            elsif not Is_Constrained (Subtyp) then
+               if not Is_Static_Subtype (Etype (First_Index (Subtyp))) then
+                  return True;
+               end if;
+            elsif not Is_OK_Static_Range (First_Index (Subtyp)) then
+               return True;
+            end if;
+         elsif Is_Record_Type (Subtyp) then
+            if Has_Discriminants (Subtyp)
+              and then Is_Constrained (Subtyp)
+              and then not Has_Static_Discriminant_Constraint (Subtyp)
+            then
+               --  Perhaps treat differently the case where Subtyp is the
+               --  subtype of the top-level selector expression, as opposed
+               --  to the subtype of some subcomponent thereof.
+               return True;
+            end if;
+         else
+            --  Return True for any type that is not a discrete type,
+            --  a record type, or an array type.
+            return True;
+         end if;
+
+         return False;
+      end Box_Value_Required;
+
       ------------------
       -- Choice_Count --
       ------------------
@@ -1179,13 +1236,45 @@ package body Sem_Case is
          return Result;
       end Choice_Count;
 
+      -------------------------------
+      -- Normalized_Case_Expr_Type --
+      -------------------------------
+
+      function Normalized_Case_Expr_Type
+        (Case_Statement : Node_Id) return Entity_Id
+      is
+         Unnormalized : constant Entity_Id :=
+           Etype (Expression (Case_Statement));
+
+         Is_Dynamically_Constrained_Array : constant Boolean :=
+           Is_Array_Type (Unnormalized)
+             and then Is_Constrained (Unnormalized)
+             and then not Has_Static_Array_Bounds (Unnormalized);
+
+         Is_Dynamically_Constrained_Record : constant Boolean :=
+           Is_Record_Type (Unnormalized)
+             and then Has_Discriminants (Unnormalized)
+             and then Is_Constrained (Unnormalized)
+             and then not Has_Static_Discriminant_Constraint (Unnormalized);
+      begin
+         if Is_Dynamically_Constrained_Array
+           or Is_Dynamically_Constrained_Record
+         then
+            return Base_Type (Unnormalized);
+         else
+            return Unnormalized;
+         end if;
+      end Normalized_Case_Expr_Type;
+
       -----------------------
       -- Scalar_Part_Count --
       -----------------------
 
       function Scalar_Part_Count (Subtyp : Entity_Id) return Nat is
       begin
-         if Is_Scalar_Type (Subtyp) then
+         if Box_Value_Required (Subtyp) then
+            return 0; -- component does not participate in case selection
+         elsif Is_Scalar_Type (Subtyp) then
             return 1;
          elsif Is_Array_Type (Subtyp) then
             return Static_Array_Length (Subtyp)
@@ -1203,8 +1292,8 @@ package body Sem_Case is
                return Result;
             end;
          else
-            pragma Assert (False);
-            raise Program_Error;
+            pragma Assert (Serious_Errors_Detected > 0);
+            return 0;
          end if;
       end Scalar_Part_Count;
 
@@ -1255,29 +1344,9 @@ package body Sem_Case is
             return 0;
          end Array_Choice_Length;
 
-         -------------------------------
-         -- Normalized_Case_Expr_Type --
-         -------------------------------
-
-         function Normalized_Case_Expr_Type
-           (Case_Statement : Node_Id) return Entity_Id
-         is
-            Unnormalized : constant Entity_Id :=
-              Etype (Expression (Case_Statement));
-         begin
-            if Is_Array_Type (Unnormalized)
-              and then Is_Constrained (Unnormalized)
-              and then not Has_Static_Array_Bounds (Unnormalized)
-            then
-               return Base_Type (Unnormalized);
-            else
-               return Unnormalized;
-            end if;
-         end Normalized_Case_Expr_Type;
-
-      ------------------------------------------
+         ------------------------------------------
          -- Unconstrained_Array_Effective_Length --
-      ------------------------------------------
+         ------------------------------------------
 
          function Unconstrained_Array_Effective_Length
            (Array_Type : Entity_Id; Case_Statement : Node_Id) return Nat
@@ -1374,6 +1443,10 @@ package body Sem_Case is
 
             procedure Traverse_Discrete_Parts (Subtyp : Entity_Id) is
             begin
+               if Box_Value_Required (Subtyp) then
+                  return;
+               end if;
+
                if Is_Discrete_Type (Subtyp) then
                   Update_Result
                     ((Low  => Expr_Value (Type_Low_Bound (Subtyp)),
@@ -1668,13 +1741,32 @@ package body Sem_Case is
                               end loop;
                            end;
 
-                           if Box_Present (Comp_Assoc) then
-                              --  Box matches all values
-                              Update_Result_For_Full_Coverage
-                                (Etype (First (Choices (Comp_Assoc))));
-                           else
-                              Traverse_Choice (Expression (Comp_Assoc));
-                           end if;
+                           declare
+                              Comp_Type : constant Entity_Id :=
+                                Etype (First (Choices (Comp_Assoc)));
+                           begin
+                              if Box_Value_Required (Comp_Type) then
+                                 --  This component is not allowed to
+                                 --  influence which alternative is
+                                 --  chosen; case choice must be box.
+                                 --
+                                 --  For example, component might be
+                                 --  of a real type or of an access type
+                                 --  or of a non-static discrete subtype.
+                                 if not Box_Present (Comp_Assoc) then
+                                    Error_Msg_N
+                                      ("Non-box case choice component value" &
+                                         " of unsupported type/subtype",
+                                       Expression (Comp_Assoc));
+                                 end if;
+                              elsif Box_Present (Comp_Assoc) then
+                                 --  Box matches all values
+                                 Update_Result_For_Full_Coverage
+                                   (Etype (First (Choices (Comp_Assoc))));
+                              else
+                                 Traverse_Choice (Expression (Comp_Assoc));
+                              end if;
+                           end;
 
                            if Binding_Chars (Comp_Assoc) /= No_Name
                            then
@@ -1702,9 +1794,19 @@ package body Sem_Case is
                              Next_Component_Or_Discriminant (Comp_From_Type);
                         end loop;
 
-                        pragma Assert
-                          (Nat (Next_Part - Saved_Next_Part)
-                           = Scalar_Part_Count (Etype (Expr)));
+                        declare
+                           Expr_Type : Entity_Id := Etype (Expr);
+                        begin
+                           if Has_Discriminants (Expr_Type) then
+                              --  Avoid nonstatic choice expr types,
+                              --  for which Scalar_Part_Count returns 0.
+                              Expr_Type := Base_Type (Expr_Type);
+                           end if;
+
+                           pragma Assert
+                             (Nat (Next_Part - Saved_Next_Part)
+                               = Scalar_Part_Count (Expr_Type));
+                        end;
                      end;
                   elsif Is_Array_Type (Etype (Expr)) then
                      if Is_Non_Empty_List (Component_Associations (Expr)) then
@@ -3256,108 +3358,14 @@ package body Sem_Case is
          -----------------------------------
 
          procedure Check_Composite_Case_Selector is
-            --  Some of these restrictions will be relaxed eventually, but best
-            --  to initially err in the direction of being too restrictive.
-
-            procedure Check_Component_Subtype (Subtyp : Entity_Id);
-            --  Recursively traverse subcomponent types to perform checks.
-
-            -----------------------------
-            -- Check_Component_Subtype --
-            -----------------------------
-
-            procedure Check_Component_Subtype (Subtyp : Entity_Id) is
-            begin
-               if Has_Predicates (Subtyp) then
-                  Error_Msg_N
-                     ("subtype of case selector (or subcomponent thereof) " &
-                      "has predicate", N);
-               elsif Is_Discrete_Type (Subtyp) then
-                  if not Is_Static_Subtype (Subtyp) then
-                     Error_Msg_N
-                       ("discrete subtype of selector subcomponent is not " &
-                        "a static subtype", N);
-                  elsif Is_Enumeration_Type (Subtyp)
-                    and then Has_Enumeration_Rep_Clause (Subtyp)
-                  then
-                     Error_Msg_N
-                       ("enumeration type of selector subcomponent has " &
-                        "an enumeration representation clause", N);
-                  end if;
-               elsif Is_Array_Type (Subtyp) then
-                  if Number_Dimensions (Subtyp) /= 1 then
-                     Error_Msg_N
-                       ("dimensionality of array type of case selector (or " &
-                        "subcomponent thereof) is greater than 1", N);
-
-                  elsif not Is_Constrained (Subtyp) then
-                     if not Is_Static_Subtype
-                              (Etype (First_Index (Subtyp)))
-                     then
-                        Error_Msg_N
-                          ("Unconstrained array subtype of case selector" &
-                           " has nonstatic index subtype", N);
-                     end if;
-
-                  elsif not Is_OK_Static_Range (First_Index (Subtyp)) then
-                     Error_Msg_N
-                       ("array subtype of case selector (or " &
-                        "subcomponent thereof) has nonstatic constraint", N);
-                  end if;
-                  Check_Component_Subtype (Component_Type (Subtyp));
-               elsif Is_Record_Type (Subtyp) then
-
-                  if Has_Discriminants (Subtyp)
-                    and then Is_Constrained (Subtyp)
-                    and then not Has_Static_Discriminant_Constraint (Subtyp)
-                  then
-                     --  We are only disallowing nonstatic constraints for
-                     --  subcomponent subtypes, not for the subtype of the
-                     --  expression we are casing on. This test could be
-                     --  implemented via an Is_Recursive_Call parameter if
-                     --  that seems preferable.
-
-                     if Subtyp /= Check_Choices.Subtyp then
-                        Error_Msg_N
-                          ("constrained discriminated subtype of case " &
-                           "selector subcomponent has nonstatic " &
-                           "constraint", N);
-                     end if;
-                  end if;
-
-                  declare
-                     Comp : Entity_Id :=
-                       First_Component_Or_Discriminant (Base_Type (Subtyp));
-                  begin
-                     while Present (Comp) loop
-                        Check_Component_Subtype (Etype (Comp));
-                        Next_Component_Or_Discriminant (Comp);
-                     end loop;
-                  end;
-               else
-                  Error_Msg_N
-                    ("type of case selector (or subcomponent thereof) is " &
-                     "not a discrete type, a record type, or an array type",
-                     N);
-               end if;
-            end Check_Component_Subtype;
-
          begin
             if not Is_Composite_Type (Subtyp) then
                Error_Msg_N
                  ("case selector type neither discrete nor composite", N);
-
             elsif Is_Limited_Type (Subtyp) then
                Error_Msg_N ("case selector type is limited", N);
-
             elsif Is_Class_Wide_Type (Subtyp) then
                Error_Msg_N ("case selector type is class-wide", N);
-
-            elsif Needs_Finalization (Subtyp) then
-               Error_Msg_N ("case selector type requires finalization", N);
-
-            else
-               Check_Component_Subtype (Subtyp);
             end if;
          end Check_Composite_Case_Selector;
 
diff --git a/gcc/ada/sem_ch10.adb b/gcc/ada/sem_ch10.adb
index 5b54e7d..6305bdb 100644
--- a/gcc/ada/sem_ch10.adb
+++ b/gcc/ada/sem_ch10.adb
@@ -341,8 +341,8 @@ package body Sem_Ch10 is
             function Same_Unit (N : Node_Id; P : Entity_Id) return Boolean is
             begin
                return Entity (N) = P
-                 or else (Present (Renamed_Object (P))
-                           and then Entity (N) = Renamed_Object (P));
+                 or else (Present (Renamed_Entity (P))
+                           and then Entity (N) = Renamed_Entity (P));
             end Same_Unit;
 
          --  Start of processing for Process_Body_Clauses
@@ -1102,14 +1102,14 @@ package body Sem_Ch10 is
                   then
                      Style_Check := False;
 
-                     if Present (Renamed_Object (Nam)) then
+                     if Present (Renamed_Entity (Nam)) then
                         Un :=
                           Load_Unit
                             (Load_Name  =>
                                Get_Body_Name
                                  (Get_Unit_Name
                                    (Unit_Declaration_Node
-                                     (Renamed_Object (Nam)))),
+                                     (Renamed_Entity (Nam)))),
                              Required   => False,
                              Subunit    => False,
                              Error_Node => N,
@@ -2870,7 +2870,7 @@ package body Sem_Ch10 is
             --  been analyzed.
 
             Analyze (Parent (Parent (Entity (Pref))));
-            pragma Assert (Renamed_Object (Entity (Pref)) = Par_Name);
+            pragma Assert (Renamed_Entity (Entity (Pref)) = Par_Name);
             Par_Name := Entity (Pref);
          end if;
 
@@ -4173,7 +4173,7 @@ package body Sem_Ch10 is
            ("parent unit must be package or generic package", Lib_Unit);
          raise Unrecoverable_Error;
 
-      elsif Present (Renamed_Object (P_Name)) then
+      elsif Present (Renamed_Entity (P_Name)) then
          Error_Msg_N ("parent unit cannot be a renaming", Lib_Unit);
          raise Unrecoverable_Error;
 
diff --git a/gcc/ada/sem_ch12.adb b/gcc/ada/sem_ch12.adb
index a62eb7c..d46d65c 100644
--- a/gcc/ada/sem_ch12.adb
+++ b/gcc/ada/sem_ch12.adb
@@ -2935,7 +2935,7 @@ package body Sem_Ch12 is
 
       --  Check for a formal package that is a package renaming
 
-      if Present (Renamed_Object (Gen_Unit)) then
+      if Present (Renamed_Entity (Gen_Unit)) then
 
          --  Indicate that unit is used, before replacing it with renamed
          --  entity for use below.
@@ -2945,7 +2945,7 @@ package body Sem_Ch12 is
             Generate_Reference  (Gen_Unit, N);
          end if;
 
-         Gen_Unit := Renamed_Object (Gen_Unit);
+         Gen_Unit := Renamed_Entity (Gen_Unit);
       end if;
 
       if Ekind (Gen_Unit) /= E_Generic_Package then
@@ -3117,7 +3117,7 @@ package body Sem_Ch12 is
          Set_Etype (Renaming_In_Par, Standard_Void_Type);
          Set_Scope (Renaming_In_Par, Parent_Instance);
          Set_Parent (Renaming_In_Par, Parent (Formal));
-         Set_Renamed_Object (Renaming_In_Par, Formal);
+         Set_Renamed_Entity (Renaming_In_Par, Formal);
          Append_Entity (Renaming_In_Par, Parent_Instance);
       end if;
 
@@ -4287,9 +4287,9 @@ package body Sem_Ch12 is
          Set_Is_Instantiated (Gen_Unit);
          Generate_Reference  (Gen_Unit, N);
 
-         if Present (Renamed_Object (Gen_Unit)) then
-            Set_Is_Instantiated (Renamed_Object (Gen_Unit));
-            Generate_Reference  (Renamed_Object (Gen_Unit), N);
+         if Present (Renamed_Entity (Gen_Unit)) then
+            Set_Is_Instantiated (Renamed_Entity (Gen_Unit));
+            Generate_Reference  (Renamed_Entity (Gen_Unit), N);
          end if;
       end if;
 
@@ -4312,10 +4312,10 @@ package body Sem_Ch12 is
 
       --  If generic is a renaming, get original generic unit
 
-      if Present (Renamed_Object (Gen_Unit))
-        and then Ekind (Renamed_Object (Gen_Unit)) = E_Generic_Package
+      if Present (Renamed_Entity (Gen_Unit))
+        and then Ekind (Renamed_Entity (Gen_Unit)) = E_Generic_Package
       then
-         Gen_Unit := Renamed_Object (Gen_Unit);
+         Gen_Unit := Renamed_Entity (Gen_Unit);
       end if;
 
       --  Verify that there are no circular instantiations
@@ -5692,10 +5692,10 @@ package body Sem_Ch12 is
 
          --  If renaming, get original unit
 
-         if Present (Renamed_Object (Gen_Unit))
-           and then Is_Generic_Subprogram (Renamed_Object (Gen_Unit))
+         if Present (Renamed_Entity (Gen_Unit))
+           and then Is_Generic_Subprogram (Renamed_Entity (Gen_Unit))
          then
-            Gen_Unit := Renamed_Object (Gen_Unit);
+            Gen_Unit := Renamed_Entity (Gen_Unit);
             Set_Is_Instantiated (Gen_Unit);
             Generate_Reference  (Gen_Unit, N);
          end if;
@@ -6856,9 +6856,9 @@ package body Sem_Ch12 is
          elsif Ekind (E1) = E_Package then
             Check_Mismatch
               (Ekind (E1) /= Ekind (E2)
-                or else (Present (Renamed_Object (E2))
-                          and then Renamed_Object (E1) /=
-                                     Renamed_Object (E2)));
+                or else (Present (Renamed_Entity (E2))
+                          and then Renamed_Entity (E1) /=
+                                     Renamed_Entity (E2)));
 
          elsif Is_Overloadable (E1) then
             --  Verify that the actual subprograms match. Note that actuals
@@ -6929,7 +6929,7 @@ package body Sem_Ch12 is
       E := First_Entity (P_Id);
       while Present (E) loop
          if Ekind (E) = E_Package then
-            if Renamed_Object (E) = P_Id then
+            if Renamed_Entity (E) = P_Id then
                exit;
 
             elsif Nkind (Parent (E)) /= N_Package_Renaming_Declaration then
@@ -7128,7 +7128,7 @@ package body Sem_Ch12 is
             --  formal part are also visible. Otherwise, ignore the entity
             --  created to validate the actuals.
 
-            if Renamed_Object (E) = Instance then
+            if Renamed_Entity (E) = Instance then
                exit;
 
             elsif Nkind (Parent (E)) /= N_Package_Renaming_Declaration then
@@ -7144,10 +7144,10 @@ package body Sem_Ch12 is
               and then not Is_Generic_Formal (E)
             then
                if Box_Present (Parent (Associated_Formal_Package (E))) then
-                  Check_Generic_Actuals (Renamed_Object (E), True);
+                  Check_Generic_Actuals (Renamed_Entity (E), True);
 
                else
-                  Check_Generic_Actuals (Renamed_Object (E), False);
+                  Check_Generic_Actuals (Renamed_Entity (E), False);
                end if;
 
                Set_Is_Hidden (E, False);
@@ -7380,9 +7380,9 @@ package body Sem_Ch12 is
          Inst_Par := Entity (Prefix (Gen_Id));
 
          if Ekind (Inst_Par) = E_Package
-           and then Present (Renamed_Object (Inst_Par))
+           and then Present (Renamed_Entity (Inst_Par))
          then
-            Inst_Par := Renamed_Object (Inst_Par);
+            Inst_Par := Renamed_Entity (Inst_Par);
          end if;
 
          if Ekind (Inst_Par) = E_Package then
@@ -7584,7 +7584,8 @@ package body Sem_Ch12 is
 
                E := First_Entity (Entity (Prefix (Gen_Id)));
                while Present (E) loop
-                  if Present (Renamed_Entity (E))
+                  if not Is_Object (E)
+                    and then Present (Renamed_Entity (E))
                     and then
                       Renamed_Entity (E) = Renamed_Entity (Entity (Gen_Id))
                   then
@@ -7621,8 +7622,8 @@ package body Sem_Ch12 is
 
             if Is_Generic_Unit (E)
               and then Nkind (Parent (E)) in N_Generic_Renaming_Declaration
-              and then Is_Child_Unit (Renamed_Object (E))
-              and then Is_Generic_Unit (Scope (Renamed_Object (E)))
+              and then Is_Child_Unit (Renamed_Entity (E))
+              and then Is_Generic_Unit (Scope (Renamed_Entity (E)))
               and then Nkind (Name (Parent (E))) = N_Expanded_Name
             then
                Rewrite (Gen_Id, New_Copy_Tree (Name (Parent (E))));
@@ -8691,10 +8692,10 @@ package body Sem_Ch12 is
             if Ekind (E1) = E_Package
               and then Nkind (Parent (E1)) = N_Package_Renaming_Declaration
             then
-               if Renamed_Object (E1) = Pack then
+               if Renamed_Entity (E1) = Pack then
                   return True;
 
-               elsif E1 = P or else Renamed_Object (E1) = P then
+               elsif E1 = P or else Renamed_Entity (E1) = P then
                   return False;
 
                elsif Is_Actual_Of_Previous_Formal (E1) then
@@ -8744,10 +8745,10 @@ package body Sem_Ch12 is
             then
                null;
 
-            elsif Renamed_Object (E) = Par then
+            elsif Renamed_Entity (E) = Par then
                return False;
 
-            elsif Renamed_Object (E) = Pack then
+            elsif Renamed_Entity (E) = Pack then
                return True;
 
             elsif Is_Actual_Of_Previous_Formal (E) then
@@ -10079,7 +10080,7 @@ package body Sem_Ch12 is
          then
             --  If this is the renaming for the parent instance, done
 
-            if Renamed_Object (E) = Par then
+            if Renamed_Entity (E) = Par then
                exit;
 
             --  The visibility of a formal of an enclosing generic is already
@@ -10089,7 +10090,7 @@ package body Sem_Ch12 is
                null;
 
             elsif Present (Associated_Formal_Package (E)) then
-               Check_Generic_Actuals (Renamed_Object (E), True);
+               Check_Generic_Actuals (Renamed_Entity (E), True);
                Set_Is_Hidden (E, False);
 
                --  Find formal package in generic unit that corresponds to
@@ -10699,8 +10700,8 @@ package body Sem_Ch12 is
          --  The actual may be a renamed package, or an outer generic formal
          --  package whose instantiation is converted into a renaming.
 
-         if Present (Renamed_Object (Actual_Pack)) then
-            Actual_Pack := Renamed_Object (Actual_Pack);
+         if Present (Renamed_Entity (Actual_Pack)) then
+            Actual_Pack := Renamed_Entity (Actual_Pack);
          end if;
 
          if Nkind (Analyzed_Formal) = N_Formal_Package_Declaration then
@@ -11798,7 +11799,7 @@ package body Sem_Ch12 is
 
                while Present (Actual) loop
                   exit when Ekind (Actual) = E_Package
-                    and then Present (Renamed_Object (Actual));
+                    and then Present (Renamed_Entity (Actual));
 
                   if Chars (Actual) = Chars (Formal)
                     and then not Is_Scalar_Type (Actual)
@@ -14791,7 +14792,7 @@ package body Sem_Ch12 is
                   Set_Instance_Of (Base_Type (E1), Base_Type (E2));
                end if;
 
-               if Ekind (E1) = E_Package and then No (Renamed_Object (E1)) then
+               if Ekind (E1) = E_Package and then No (Renamed_Entity (E1)) then
                   Map_Formal_Package_Entities (E1, E2);
                end if;
             end if;
@@ -15347,11 +15348,11 @@ package body Sem_Ch12 is
       ---------------------------
 
       procedure Restore_Nested_Formal (Formal : Entity_Id) is
+         pragma Assert (Ekind (Formal) = E_Package);
          Ent : Entity_Id;
-
       begin
-         if Present (Renamed_Object (Formal))
-           and then Denotes_Formal_Package (Renamed_Object (Formal), True)
+         if Present (Renamed_Entity (Formal))
+           and then Denotes_Formal_Package (Renamed_Entity (Formal), True)
          then
             return;
 
@@ -15490,20 +15491,20 @@ package body Sem_Ch12 is
             --  visible on exit from the instance, and therefore nothing needs
             --  to be done either, except to keep it accessible.
 
-            if Is_Package and then Renamed_Object (E) = Pack_Id then
+            if Is_Package and then Renamed_Entity (E) = Pack_Id then
                exit;
 
             elsif Nkind (Parent (E)) /= N_Package_Renaming_Declaration then
                null;
 
             elsif
-              Denotes_Formal_Package (Renamed_Object (E), True, Pack_Id)
+              Denotes_Formal_Package (Renamed_Entity (E), True, Pack_Id)
             then
                Set_Is_Hidden (E, False);
 
             else
                declare
-                  Act_P : constant Entity_Id := Renamed_Object (E);
+                  Act_P : constant Entity_Id := Renamed_Entity (E);
                   Id    : Entity_Id;
 
                begin
@@ -15512,7 +15513,7 @@ package body Sem_Ch12 is
                     and then Id /= First_Private_Entity (Act_P)
                   loop
                      exit when Ekind (Id) = E_Package
-                                 and then Renamed_Object (Id) = Act_P;
+                                 and then Renamed_Entity (Id) = Act_P;
 
                      Set_Is_Hidden (Id, True);
                      Set_Is_Potentially_Use_Visible (Id, In_Use (Act_P));
diff --git a/gcc/ada/sem_ch13.adb b/gcc/ada/sem_ch13.adb
index 3374e8b..d63d60f 100644
--- a/gcc/ada/sem_ch13.adb
+++ b/gcc/ada/sem_ch13.adb
@@ -483,16 +483,16 @@ package body Sem_Ch13 is
                      if Warn_On_Reverse_Bit_Order then
                         Error_Msg_N
                           ("info: multi-byte field specified with "
-                           & "non-standard Bit_Order?V?", CC);
+                           & "non-standard Bit_Order?.v?", CC);
 
                         if Bytes_Big_Endian then
                            Error_Msg_N
                              ("\bytes are not reversed "
-                              & "(component is big-endian)?V?", CC);
+                              & "(component is big-endian)?.v?", CC);
                         else
                            Error_Msg_N
                              ("\bytes are not reversed "
-                              & "(component is little-endian)?V?", CC);
+                              & "(component is little-endian)?.v?", CC);
                         end if;
                      end if;
 
@@ -707,17 +707,18 @@ package body Sem_Ch13 is
                      Error_Msg_Uint_1 := MSS;
                      Error_Msg_N
                        ("info: reverse bit order in machine scalar of "
-                        & "length^?V?", First_Bit (CC));
+                        & "length^?.v?", First_Bit (CC));
                      Error_Msg_Uint_1 := NFB;
                      Error_Msg_Uint_2 := NLB;
 
                      if Bytes_Big_Endian then
                         Error_Msg_NE
-                          ("\big-endian range for component & is ^ .. ^?V?",
+                          ("\big-endian range for component & is ^ .. ^?.v?",
                            First_Bit (CC), Comp);
                      else
                         Error_Msg_NE
-                          ("\little-endian range for component & is ^ .. ^?V?",
+                          ("\little-endian range for component " &
+                           "& is ^ .. ^?.v?",
                            First_Bit (CC), Comp);
                      end if;
                   end if;
@@ -782,16 +783,16 @@ package body Sem_Ch13 is
                   then
                      Error_Msg_N
                        ("info: multi-byte field specified with non-standard "
-                        & "Bit_Order?V?", CLC);
+                        & "Bit_Order?.v?", CLC);
 
                      if Bytes_Big_Endian then
                         Error_Msg_N
                           ("\bytes are not reversed "
-                           & "(component is big-endian)?V?", CLC);
+                           & "(component is big-endian)?.v?", CLC);
                      else
                         Error_Msg_N
                           ("\bytes are not reversed "
-                           & "(component is little-endian)?V?", CLC);
+                           & "(component is little-endian)?.v?", CLC);
                      end if;
 
                   --  Do not allow non-contiguous field
@@ -815,13 +816,13 @@ package body Sem_Ch13 is
                   then
                      Error_Msg_N
                        ("info: Bit_Order clause does not affect byte "
-                        & "ordering?V?", Pos);
+                        & "ordering?.v?", Pos);
                      Error_Msg_Uint_1 :=
                        Intval (Pos) + Intval (FB) /
                        System_Storage_Unit;
                      Error_Msg_N
                        ("info: position normalized to ^ before bit order "
-                        & "interpreted?V?", Pos);
+                        & "interpreted?.v?", Pos);
                   end if;
 
                   --  Here is where we fix up the Component_Bit_Offset value
@@ -6248,7 +6249,7 @@ package body Sem_Ch13 is
 
       Check_Restriction_No_Use_Of_Attribute (N);
 
-      if Get_Aspect_Id (Chars (N)) /= No_Aspect then
+      if Is_Aspect_Id (Chars (N)) then
          --  6.1/3 No_Specification_of_Aspect: Identifies an aspect for which
          --    no aspect_specification, attribute_definition_clause, or pragma
          --    is given.
@@ -6911,7 +6912,7 @@ package body Sem_Ch13 is
                  and then RM_Size (Ctyp) /= Csize
                then
                   Error_Msg_NE
-                    ("component size overrides size clause for&?S?", N, Ctyp);
+                    ("component size overrides size clause for&?.s?", N, Ctyp);
                end if;
 
                Set_Has_Component_Size_Clause (Btype, True);
@@ -8809,7 +8810,7 @@ package body Sem_Ch13 is
                           and then RM_Size (Etype (Comp)) /= Esize (Comp)
                         then
                            Error_Msg_NE
-                             ("?S?component size overrides size clause for&",
+                             ("?.s?component size overrides size clause for&",
                               Component_Name (CC), Etype (Comp));
                         end if;
 
@@ -8918,7 +8919,7 @@ package body Sem_Ch13 is
                   then
                      Error_Msg_Sloc := Sloc (Comp);
                      Error_Msg_NE
-                       ("?C?no component clause given for & declared #",
+                       ("?.c?no component clause given for & declared #",
                         N, Comp);
                   end if;
 
@@ -8940,9 +8941,6 @@ package body Sem_Ch13 is
    is
       Loc : constant Source_Ptr := Sloc (Expr);
 
-      Non_Static : exception;
-      --  Raised if something non-static is found
-
       Btyp : constant Entity_Id := Base_Type (Typ);
 
       BLo : constant Uint := Expr_Value (Type_Low_Bound  (Btyp));
@@ -8995,7 +8993,7 @@ package body Sem_Ch13 is
 
       function Build_Val (V : Uint) return Node_Id;
       --  Return an analyzed N_Identifier node referencing this value, suitable
-      --  for use as an entry in the Static_Discrte_Predicate list. This node
+      --  for use as an entry in the Static_Discrete_Predicate list. This node
       --  is typed with the base type.
 
       function Build_Range (Lo : Uint; Hi : Uint) return Node_Id;
@@ -9003,9 +9001,13 @@ package body Sem_Ch13 is
       --  use as an entry in the Static_Discrete_Predicate list. This node is
       --  typed with the base type.
 
-      function Get_RList (Exp : Node_Id) return RList;
+      function Get_RList
+        (Exp    : Node_Id;
+         Static : access Boolean) return RList;
       --  This is a recursive routine that converts the given expression into a
       --  list of ranges, suitable for use in building the static predicate.
+      --  Static.all will be set to False if the expression is found to be non
+      --  static. Note that Static.all should be set to True by the caller.
 
       function Is_False (R : RList) return Boolean;
       pragma Inline (Is_False);
@@ -9033,18 +9035,23 @@ package body Sem_Ch13 is
       --  a static expression or static range, gets either the expression value
       --  or the high bound of the range.
 
-      function Membership_Entry (N : Node_Id) return RList;
+      function Membership_Entry
+        (N : Node_Id; Static : access Boolean) return RList;
       --  Given a single membership entry (range, value, or subtype), returns
-      --  the corresponding range list. Raises Static_Error if not static.
+      --  the corresponding range list. Set Static.all to False if not static.
 
-      function Membership_Entries (N : Node_Id) return RList;
+      function Membership_Entries
+        (N : Node_Id; Static : access Boolean) return RList;
       --  Given an element on an alternatives list of a membership operation,
       --  returns the range list corresponding to this entry and all following
       --  entries (i.e. returns the "or" of this list of values).
+      --  Set Static.all to False if not static.
 
-      function Stat_Pred (Typ : Entity_Id) return RList;
-      --  Given a type, if it has a static predicate, then return the predicate
-      --  as a range list, otherwise raise Non_Static.
+      function Stat_Pred
+        (Typ    : Entity_Id;
+         Static : access Boolean) return RList;
+      --  Given a type, if it has a static predicate, then set Result to the
+      --  predicate as a range list, otherwise set Static.all to False.
 
       -----------
       -- "and" --
@@ -9296,7 +9303,10 @@ package body Sem_Ch13 is
       -- Get_RList --
       ---------------
 
-      function Get_RList (Exp : Node_Id) return RList is
+      function Get_RList
+        (Exp    : Node_Id;
+         Static : access Boolean) return RList
+      is
          Op  : Node_Kind;
          Val : Uint;
 
@@ -9322,23 +9332,23 @@ package body Sem_Ch13 is
             when N_And_Then
                | N_Op_And
             =>
-               return Get_RList (Left_Opnd (Exp))
+               return Get_RList (Left_Opnd (Exp), Static)
                         and
-                      Get_RList (Right_Opnd (Exp));
+                      Get_RList (Right_Opnd (Exp), Static);
 
             --  Or
 
             when N_Op_Or
                | N_Or_Else
             =>
-               return Get_RList (Left_Opnd (Exp))
+               return Get_RList (Left_Opnd (Exp), Static)
                         or
-                      Get_RList (Right_Opnd (Exp));
+                      Get_RList (Right_Opnd (Exp), Static);
 
             --  Not
 
             when N_Op_Not =>
-               return not Get_RList (Right_Opnd (Exp));
+               return not Get_RList (Right_Opnd (Exp), Static);
 
                --  Comparisons of type with static value
 
@@ -9371,7 +9381,8 @@ package body Sem_Ch13 is
                --  Other cases are non-static
 
                else
-                  raise Non_Static;
+                  Static.all := False;
+                  return False_Range;
                end if;
 
                --  Construct range according to comparison operation
@@ -9403,26 +9414,30 @@ package body Sem_Ch13 is
 
             when N_In =>
                if not Is_Type_Ref (Left_Opnd (Exp)) then
-                  raise Non_Static;
+                  Static.all := False;
+                  return False_Range;
                end if;
 
                if Present (Right_Opnd (Exp)) then
-                  return Membership_Entry (Right_Opnd (Exp));
+                  return Membership_Entry (Right_Opnd (Exp), Static);
                else
-                  return Membership_Entries (First (Alternatives (Exp)));
+                  return Membership_Entries
+                           (First (Alternatives (Exp)), Static);
                end if;
 
             --  Negative membership (NOT IN)
 
             when N_Not_In =>
                if not Is_Type_Ref (Left_Opnd (Exp)) then
-                  raise Non_Static;
+                  Static.all := False;
+                  return False_Range;
                end if;
 
                if Present (Right_Opnd (Exp)) then
-                  return not Membership_Entry (Right_Opnd (Exp));
+                  return not Membership_Entry (Right_Opnd (Exp), Static);
                else
-                  return not Membership_Entries (First (Alternatives (Exp)));
+                  return not Membership_Entries
+                               (First (Alternatives (Exp)), Static);
                end if;
 
             --  Function call, may be call to static predicate
@@ -9436,19 +9451,20 @@ package body Sem_Ch13 is
                           or else
                         Is_Predicate_Function_M (Ent)
                      then
-                        return Stat_Pred (Etype (First_Formal (Ent)));
+                        return Stat_Pred (Etype (First_Formal (Ent)), Static);
                      end if;
                   end;
                end if;
 
                --  Other function call cases are non-static
 
-               raise Non_Static;
+               Static.all := False;
+               return False_Range;
 
             --  Qualified expression, dig out the expression
 
             when N_Qualified_Expression =>
-               return Get_RList (Expression (Exp));
+               return Get_RList (Expression (Exp), Static);
 
             when N_Case_Expression =>
                declare
@@ -9473,7 +9489,8 @@ package body Sem_Ch13 is
                      Dep := Expression (Alt);
 
                      if not Is_OK_Static_Expression (Dep) then
-                        raise Non_Static;
+                        Static.all := False;
+                        return False_Range;
 
                      elsif Is_True (Expr_Value (Dep)) then
                         Append_List_To (Choices,
@@ -9483,30 +9500,32 @@ package body Sem_Ch13 is
                      Next (Alt);
                   end loop;
 
-                  return Membership_Entries (First (Choices));
+                  return Membership_Entries (First (Choices), Static);
                end;
 
             --  Expression with actions: if no actions, dig out expression
 
             when N_Expression_With_Actions =>
                if Is_Empty_List (Actions (Exp)) then
-                  return Get_RList (Expression (Exp));
+                  return Get_RList (Expression (Exp), Static);
                else
-                  raise Non_Static;
+                  Static.all := False;
+                  return False_Range;
                end if;
 
             --  Xor operator
 
             when N_Op_Xor =>
-               return (Get_RList (Left_Opnd (Exp))
-                        and not Get_RList (Right_Opnd (Exp)))
-                 or   (Get_RList (Right_Opnd (Exp))
-                        and not Get_RList (Left_Opnd (Exp)));
+               return (Get_RList (Left_Opnd (Exp), Static)
+                        and not Get_RList (Right_Opnd (Exp), Static))
+                 or   (Get_RList (Right_Opnd (Exp), Static)
+                        and not Get_RList (Left_Opnd (Exp), Static));
 
             --  Any other node type is non-static
 
             when others =>
-               raise Non_Static;
+               Static.all := False;
+               return False_Range;
          end case;
       end Get_RList;
 
@@ -9573,12 +9592,14 @@ package body Sem_Ch13 is
       -- Membership_Entries --
       ------------------------
 
-      function Membership_Entries (N : Node_Id) return RList is
+      function Membership_Entries
+        (N : Node_Id; Static : access Boolean) return RList is
       begin
          if No (Next (N)) then
-            return Membership_Entry (N);
+            return Membership_Entry (N, Static);
          else
-            return Membership_Entry (N) or Membership_Entries (Next (N));
+            return Membership_Entry (N, Static)
+              or Membership_Entries (Next (N), Static);
          end if;
       end Membership_Entries;
 
@@ -9586,7 +9607,9 @@ package body Sem_Ch13 is
       -- Membership_Entry --
       ----------------------
 
-      function Membership_Entry (N : Node_Id) return RList is
+      function Membership_Entry
+        (N : Node_Id; Static : access Boolean) return RList
+      is
          Val : Uint;
          SLo : Uint;
          SHi : Uint;
@@ -9599,7 +9622,8 @@ package body Sem_Ch13 is
                  or else
                not Is_OK_Static_Expression (High_Bound (N))
             then
-               raise Non_Static;
+               Static.all := False;
+               return False_Range;
             else
                SLo := Expr_Value (Low_Bound  (N));
                SHi := Expr_Value (High_Bound (N));
@@ -9642,7 +9666,7 @@ package body Sem_Ch13 is
                --  If type has predicates, process them
 
                if Has_Predicates (Entity (N)) then
-                  return Stat_Pred (Entity (N));
+                  return Stat_Pred (Entity (N), Static);
 
                --  For static subtype without predicates, get range
 
@@ -9654,14 +9678,16 @@ package body Sem_Ch13 is
                --  Any other type makes us non-static
 
                else
-                  raise Non_Static;
+                  Static.all := False;
+                  return False_Range;
                end if;
 
             --  Any other kind of identifier in predicate (e.g. a non-static
             --  expression value) means this is not a static predicate.
 
             else
-               raise Non_Static;
+               Static.all := False;
+               return False_Range;
             end if;
          end if;
       end Membership_Entry;
@@ -9670,12 +9696,15 @@ package body Sem_Ch13 is
       -- Stat_Pred --
       ---------------
 
-      function Stat_Pred (Typ : Entity_Id) return RList is
+      function Stat_Pred
+        (Typ    : Entity_Id;
+         Static : access Boolean) return RList is
       begin
          --  Not static if type does not have static predicates
 
          if not Has_Static_Predicate (Typ) then
-            raise Non_Static;
+            Static.all := False;
+            return False_Range;
          end if;
 
          --  Otherwise we convert the predicate list to a range list
@@ -9716,12 +9745,19 @@ package body Sem_Ch13 is
       --  Analyze the expression to see if it is a static predicate
 
       declare
-         Ranges : constant RList := Get_RList (Expr);
+         Static : aliased Boolean := True;
+         Ranges : constant RList := Get_RList (Expr, Static'Access);
          --  Range list from expression if it is static
 
          Plist : List_Id;
 
       begin
+         --  If non-static, return doing nothing
+
+         if not Static then
+            return;
+         end if;
+
          --  Convert range list into a form for the static predicate. In the
          --  Ranges array, we just have raw ranges, these must be converted
          --  to properly typed and analyzed static expressions or range nodes.
@@ -9826,12 +9862,6 @@ package body Sem_Ch13 is
             end if;
          end;
       end;
-
-      --  If non-static, return doing nothing
-
-   exception
-      when Non_Static =>
-         return;
    end Build_Discrete_Static_Predicate;
 
    --------------------------------
@@ -10096,7 +10126,7 @@ package body Sem_Ch13 is
             then
                Error_Msg_Sloc := Sloc (Predicate_Function (T));
                Error_Msg_Node_2 := T;
-               Error_Msg_N ("info: & inherits predicate from & #?L?", Typ);
+               Error_Msg_N ("info: & inherits predicate from & #?.l?", Typ);
             end if;
          end if;
       end Add_Call;
@@ -11881,7 +11911,7 @@ package body Sem_Ch13 is
          Clause : Node_Id := First (Component_Clauses (N));
          Prev_Bit_Offset : Uint := Uint_0;
          OOO : constant String :=
-           "?component clause out of order with respect to declaration";
+           "?_r?component clause out of order with respect to declaration";
 
       begin
          --  Step Comp through components and Clause through component clauses,
@@ -11907,7 +11937,7 @@ package body Sem_Ch13 is
                  and then not Reverse_Storage_Order (Rectype)
                  and then Component_Bit_Offset (Comp) < Prev_Bit_Offset
                then
-                  Error_Msg_N ("?memory layout out of order", Clause);
+                  Error_Msg_N ("?_r?memory layout out of order", Clause);
                   exit;
                end if;
 
@@ -12147,7 +12177,7 @@ package body Sem_Ch13 is
 
                         if Warn and then Error_Msg_Uint_1 > 0 then
                            Error_Msg_NE
-                             ("?H?^-bit gap before component&",
+                             ("?.h?^-bit gap before component&",
                               Component_Name (Component_Clause (CEnt)),
                               CEnt);
                         end if;
@@ -12588,9 +12618,11 @@ package body Sem_Ch13 is
       end if;
 
       --  Skip the following warnings if overlap was detected; programmer
-      --  should fix the errors first.
+      --  should fix the errors first. Also skip the warnings for types in
+      --  generics, because their representation information is not fully
+      --  computed.
 
-      if not Overlap_Detected then
+      if not Overlap_Detected and then not In_Generic_Scope (Rectype) then
          --  Check for record holes (gaps)
 
          if Warn_On_Record_Holes then
@@ -13132,6 +13164,7 @@ package body Sem_Ch13 is
                   if Get_Aspect_Id (Ritem) in Aspect_CPU
                                             | Aspect_Dynamic_Predicate
                                             | Aspect_Predicate
+                                            | Aspect_Static_Predicate
                                             | Aspect_Priority
                   then
                     --  Retrieve the visibility to components and discriminants
@@ -13139,11 +13172,61 @@ package body Sem_Ch13 is
 
                      Push_Type (E);
                      Check_Aspect_At_Freeze_Point (Ritem);
+
+                     --  In the case of predicate aspects, there will be
+                     --  a corresponding Predicate pragma associated with
+                     --  the aspect, and the expression of the pragma also
+                     --  needs to be analyzed at this point, to ensure that
+                     --  Save_Global_References will capture global refs in
+                     --  expressions that occur in generic bodies, for proper
+                     --  later resolution of the pragma in instantiations.
+
+                     if Is_Type (E)
+                       and then Inside_A_Generic
+                       and then Has_Predicates (E)
+                       and then Present (Aspect_Rep_Item (Ritem))
+                     then
+                        declare
+                           Pragma_Args : constant List_Id :=
+                             Pragma_Argument_Associations
+                               (Aspect_Rep_Item (Ritem));
+                           Pragma_Expr : constant Node_Id :=
+                             Expression (Next (First (Pragma_Args)));
+                        begin
+                           if Present (Pragma_Expr) then
+                              Analyze_And_Resolve
+                                (Pragma_Expr, Standard_Boolean);
+                           end if;
+                        end;
+                     end if;
+
                      Pop_Type (E);
 
                   else
                      Check_Aspect_At_Freeze_Point (Ritem);
                   end if;
+
+               --  A pragma Predicate should be checked like one of the
+               --  corresponding aspects, wrt possible misuse of ghost
+               --  entities.
+
+               elsif Nkind (Ritem) = N_Pragma
+                 and then No (Corresponding_Aspect (Ritem))
+                 and then
+                   Get_Pragma_Id (Pragma_Name (Ritem)) = Pragma_Predicate
+               then
+                  --  Retrieve the visibility to components and discriminants
+                  --  in order to properly analyze the pragma.
+
+                  declare
+                     Arg : constant Node_Id :=
+                        Next (First (Pragma_Argument_Associations (Ritem)));
+                  begin
+                     Push_Type (E);
+                     Preanalyze_Spec_Expression
+                       (Expression (Arg), Standard_Boolean);
+                     Pop_Type (E);
+                  end;
                end if;
 
                Next_Rep_Item (Ritem);
@@ -16649,7 +16732,7 @@ package body Sem_Ch13 is
 
          if Warn_On_Biased_Representation then
             Error_Msg_NE
-              ("?B?" & Msg & " forces biased representation for&", N, E);
+              ("?.b?" & Msg & " forces biased representation for&", N, E);
          end if;
       end if;
    end Set_Biased;
diff --git a/gcc/ada/sem_ch3.adb b/gcc/ada/sem_ch3.adb
index c8d4ec1..569e019 100644
--- a/gcc/ada/sem_ch3.adb
+++ b/gcc/ada/sem_ch3.adb
@@ -79,6 +79,7 @@ with Sinfo.Nodes;    use Sinfo.Nodes;
 with Sinfo.Utils;    use Sinfo.Utils;
 with Sinput;         use Sinput;
 with Snames;         use Snames;
+with Strub;          use Strub;
 with Targparm;       use Targparm;
 with Tbuild;         use Tbuild;
 with Ttypes;         use Ttypes;
@@ -3506,6 +3507,15 @@ package body Sem_Ch3 is
          Set_Is_Tagged_Type (T, True);
          Set_No_Tagged_Streams_Pragma (T, No_Tagged_Streams);
          Make_Class_Wide_Type (T);
+      end if;
+
+      --  For tagged types, or when prefixed-call syntax is allowed for
+      --  untagged types, initialize the list of primitive operations to
+      --  an empty list.
+
+      if Tagged_Present (N)
+        or else Extensions_Allowed
+      then
          Set_Direct_Primitive_Operations (T, New_Elmt_List);
       end if;
 
@@ -5746,6 +5756,15 @@ package body Sem_Ch3 is
                   if Is_Tagged_Type (Id) then
                      Set_No_Tagged_Streams_Pragma
                        (Id, No_Tagged_Streams_Pragma (T));
+                  end if;
+
+                  --  For tagged types, or when prefixed-call syntax is allowed
+                  --  for untagged types, initialize the list of primitive
+                  --  operations to an empty list.
+
+                  if Is_Tagged_Type (Id)
+                    or else Extensions_Allowed
+                  then
                      Set_Direct_Primitive_Operations (Id, New_Elmt_List);
                   end if;
 
@@ -16047,6 +16066,8 @@ package body Sem_Ch3 is
          Set_Alias (New_Subp, Actual_Subp);
       end if;
 
+      Copy_Strub_Mode (New_Subp, Alias (New_Subp));
+
       --  Derived subprograms of a tagged type must inherit the convention
       --  of the parent subprogram (a requirement of AI-117). Derived
       --  subprograms of untagged types simply get convention Ada by default.
@@ -17190,8 +17211,12 @@ package body Sem_Ch3 is
          Set_Etype        (T, Any_Type);
          Set_Scalar_Range (T, Scalar_Range (Any_Type));
 
-         if Is_Tagged_Type (T)
-           and then Is_Record_Type (T)
+         --  For tagged types, or when prefixed-call syntax is allowed for
+         --  untagged types, initialize the list of primitive operations to
+         --  an empty list.
+
+         if (Is_Tagged_Type (T) and then Is_Record_Type (T))
+           or else Extensions_Allowed
          then
             Set_Direct_Primitive_Operations (T, New_Elmt_List);
          end if;
@@ -17233,10 +17258,46 @@ package body Sem_Ch3 is
         and then Is_Interface (Parent_Type)
       then
          declare
-            Iface               : Node_Id;
             Partial_View        : Entity_Id;
             Partial_View_Parent : Entity_Id;
-            New_Iface           : Node_Id;
+
+            function Reorder_Interfaces return Boolean;
+            --  Look for an interface in the full view's interface list that
+            --  matches the parent type of the partial view, and when found,
+            --  rewrite the full view's parent with the partial view's parent,
+            --  append the full view's original parent to the interface list,
+            --  recursively call Derived_Type_Definition on the full type, and
+            --  return True. If a match is not found, return False.
+            --  ??? This seems broken in the case of generic packages.
+
+            ------------------------
+            -- Reorder_Interfaces --
+            ------------------------
+
+            function Reorder_Interfaces return Boolean is
+               Iface     : Node_Id;
+               New_Iface : Node_Id;
+            begin
+               Iface := First (Interface_List (Def));
+               while Present (Iface) loop
+                  if Etype (Iface) = Etype (Partial_View) then
+                     Rewrite (Subtype_Indication (Def),
+                       New_Copy (Subtype_Indication (Parent (Partial_View))));
+
+                     New_Iface :=
+                       Make_Identifier (Sloc (N), Chars (Parent_Type));
+                     Append (New_Iface, Interface_List (Def));
+
+                     --  Analyze the transformed code
+
+                     Derived_Type_Declaration (T, N, Is_Completion);
+                     return True;
+                  end if;
+
+                  Next (Iface);
+               end loop;
+               return False;
+            end Reorder_Interfaces;
 
          begin
             --  Look for the associated private type declaration
@@ -17257,30 +17318,26 @@ package body Sem_Ch3 is
                then
                   null;
 
-               --  Traverse the list of interfaces of the full-view to look
-               --  for the parent of the partial-view and perform the tree
-               --  transformation.
+               --  Traverse the list of interfaces of the full view to look
+               --  for the parent of the partial view and reorder the
+               --  interfaces to match the order in the partial view,
+               --  if needed.
 
                else
-                  Iface := First (Interface_List (Def));
-                  while Present (Iface) loop
-                     if Etype (Iface) = Etype (Partial_View) then
-                        Rewrite (Subtype_Indication (Def),
-                          New_Copy (Subtype_Indication
-                                     (Parent (Partial_View))));
-
-                        New_Iface :=
-                          Make_Identifier (Sloc (N), Chars (Parent_Type));
-                        Append (New_Iface, Interface_List (Def));
 
-                        --  Analyze the transformed code
+                  if Reorder_Interfaces then
+                     --  Having the interfaces listed in any order is legal.
+                     --  However, the compiler does not properly handle
+                     --  different orders between partial and full views in
+                     --  generic units. We give a warning about the order
+                     --  mismatch, so the user can work around this problem.
 
-                        Derived_Type_Declaration (T, N, Is_Completion);
-                        return;
-                     end if;
+                     Error_Msg_N ("??full declaration does not respect " &
+                                  "partial declaration order", T);
+                     Error_Msg_N ("\??consider reordering", T);
 
-                     Next (Iface);
-                  end loop;
+                     return;
+                  end if;
                end if;
             end if;
          end;
@@ -19890,7 +19947,7 @@ package body Sem_Ch3 is
         and then Intval (Right_Opnd (Mod_Expr)) <= Uint_128
       then
          Error_Msg_N
-           ("suspicious MOD value, was '*'* intended'??M?", Mod_Expr);
+           ("suspicious MOD value, was '*'* intended'??.m?", Mod_Expr);
       end if;
 
       --  Proceed with analysis of mod expression
@@ -21301,7 +21358,7 @@ package body Sem_Ch3 is
                      goto Leave;
                   end;
 
-               --  For non-concurrent types, transfer explicit primitives, but
+               --  For nonconcurrent types, transfer explicit primitives, but
                --  omit those inherited from the parent of the private view
                --  since they will be re-inherited later on.
 
diff --git a/gcc/ada/sem_ch4.adb b/gcc/ada/sem_ch4.adb
index fecc060a..77c1b97 100644
--- a/gcc/ada/sem_ch4.adb
+++ b/gcc/ada/sem_ch4.adb
@@ -281,6 +281,19 @@ package body Sem_Ch4 is
    --  type is not directly visible. The routine uses this type to emit a more
    --  informative message.
 
+   function Has_Possible_Literal_Aspects (N : Node_Id) return Boolean;
+   --  Ada_2022: if an operand is a literal it may be subject to an
+   --  implicit conversion to a type for which a user-defined literal
+   --  function exists. During the first pass of type resolution we do
+   --  not know the context imposed on the literal, so we assume that
+   --  the literal type is a valid candidate and rely on the second pass
+   --  of resolution to find the type with the proper aspect. We only
+   --  add this interpretation if no other one was found, which may be
+   --  too restrictive but seems sufficient to handle most proper uses
+   --  of the new aspect. It is unclear whether a full implementation of
+   --  these aspects can be achieved without larger modifications to the
+   --  two-pass resolution algorithm.
+
    procedure Remove_Abstract_Operations (N : Node_Id);
    --  Ada 2005: implementation of AI-310. An abstract non-dispatching
    --  operation is not a candidate interpretation.
@@ -2963,10 +2976,7 @@ package body Sem_Ch4 is
 
       procedure Find_Interpretation;
       function Find_Interpretation return Boolean;
-      --  Routine and wrapper to find a matching interpretation in case
-      --  of overloading. The wrapper returns True iff a matching
-      --  interpretation is found. Beware, in absence of overloading,
-      --  using this function will break gnat's bootstrapping.
+      --  Routine and wrapper to find a matching interpretation
 
       procedure Try_One_Interp (T1 : Entity_Id);
       --  Routine to try one proposed interpretation. Note that the context
@@ -3078,11 +3088,16 @@ package body Sem_Ch4 is
 
       procedure Find_Interpretation is
       begin
-         Get_First_Interp (L, Index, It);
-         while Present (It.Typ) loop
-            Try_One_Interp (It.Typ);
-            Get_Next_Interp (Index, It);
-         end loop;
+         if not Is_Overloaded (L) then
+            Try_One_Interp (Etype (L));
+
+         else
+            Get_First_Interp (L, Index, It);
+            while Present (It.Typ) loop
+               Try_One_Interp (It.Typ);
+               Get_Next_Interp (Index, It);
+            end loop;
+         end if;
       end Find_Interpretation;
 
       function Find_Interpretation return Boolean is
@@ -3098,7 +3113,7 @@ package body Sem_Ch4 is
 
       procedure Try_One_Interp (T1 : Entity_Id) is
       begin
-         if Has_Compatible_Type (R, T1) then
+         if Has_Compatible_Type (R, T1, For_Comparison => True) then
             if Found
               and then Base_Type (T1) /= Base_Type (T_F)
             then
@@ -3143,12 +3158,7 @@ package body Sem_Ch4 is
       then
          Analyze (R);
 
-         if not Is_Overloaded (L) then
-            Try_One_Interp (Etype (L));
-
-         else
-            Find_Interpretation;
-         end if;
+         Find_Interpretation;
 
       --  If not a range, it can be a subtype mark, or else it is a degenerate
       --  membership test with a singleton value, i.e. a test for equality,
@@ -3157,16 +3167,11 @@ package body Sem_Ch4 is
       else
          Analyze (R);
 
-         if Is_Entity_Name (R)
-           and then Is_Type (Entity (R))
-         then
+         if Is_Entity_Name (R) and then Is_Type (Entity (R)) then
             Find_Type (R);
             Check_Fully_Declared (Entity (R), R);
 
-         elsif Ada_Version >= Ada_2012 and then
-           ((Is_Overloaded (L) and then Find_Interpretation) or else
-           (not Is_Overloaded (L) and then Has_Compatible_Type (R, Etype (L))))
-         then
+         elsif Ada_Version >= Ada_2012 and then Find_Interpretation then
             if Nkind (N) = N_In then
                Op := Make_Op_Eq (Loc, Left_Opnd => L, Right_Opnd => R);
             else
@@ -3229,7 +3234,7 @@ package body Sem_Ch4 is
         and then Intval (Right_Opnd (Parent (N))) <= Uint_128
       then
          Error_Msg_N
-           ("suspicious MOD value, was '*'* intended'??M?", Parent (N));
+           ("suspicious MOD value, was '*'* intended'??.m?", Parent (N));
       end if;
 
       --  Remaining processing is same as for other arithmetic operators
@@ -4334,7 +4339,7 @@ package body Sem_Ch4 is
                     (if Kind = Conjunct then "conjunct" else "disjunct");
                begin
                   Error_Msg_NE
-                    ("?T?unused variable & in " & Sub, Expr, Loop_Id);
+                    ("?.t?unused variable & in " & Sub, Expr, Loop_Id);
                   Error_Msg_NE
                     ("\consider extracting " & Sub & " from quantified "
                      & "expression", Expr, Loop_Id);
@@ -4354,7 +4359,7 @@ package body Sem_Ch4 is
            and then not (Modify_Tree_For_C and In_Inlined_Body)
          then
             if not Referenced (Loop_Id, Cond) then
-               Error_Msg_N ("?T?unused variable &", Loop_Id);
+               Error_Msg_N ("?.t?unused variable &", Loop_Id);
             else
                Check_Subexpr (Cond, Kind => Full);
             end if;
@@ -4375,7 +4380,7 @@ package body Sem_Ch4 is
         and then Nkind (Cond) = N_If_Expression
         and then No_Else_Or_Trivial_True (Cond)
       then
-         Error_Msg_N ("?T?suspicious expression", N);
+         Error_Msg_N ("?.t?suspicious expression", N);
          Error_Msg_N ("\\did you mean (for all X ='> (if P then Q))", N);
          Error_Msg_N ("\\or (for some X ='> P and then Q) instead'?", N);
       end if;
@@ -5905,14 +5910,16 @@ package body Sem_Ch4 is
       begin
          --  Verify that Op_Id is a visible binary function. Note that since
          --  we know Op_Id is overloaded, potentially use visible means use
-         --  visible for sure (RM 9.4(11)).
+         --  visible for sure (RM 9.4(11)). Be prepared for previous errors.
 
          if Ekind (Op_Id) = E_Function
            and then Present (F2)
            and then (Is_Immediately_Visible (Op_Id)
                       or else Is_Potentially_Use_Visible (Op_Id))
-           and then Has_Compatible_Type (Left_Opnd (N), Etype (F1))
-           and then Has_Compatible_Type (Right_Opnd (N), Etype (F2))
+           and then (Has_Compatible_Type (Left_Opnd (N), Etype (F1))
+                      or else Etype (F1) = Any_Type)
+           and then (Has_Compatible_Type (Right_Opnd (N), Etype (F2))
+                      or else Etype (F2) = Any_Type)
          then
             Add_One_Interp (N, Op_Id, Etype (Op_Id));
 
@@ -6599,7 +6606,9 @@ package body Sem_Ch4 is
             return;
          end if;
 
-         if Valid_Comparison_Arg (T1) and then Has_Compatible_Type (R, T1) then
+         if Valid_Comparison_Arg (T1)
+           and then Has_Compatible_Type (R, T1, For_Comparison => True)
+         then
             if Found and then Base_Type (T1) /= Base_Type (T_F) then
                It := Disambiguate (L, I_F, Index, Any_Type);
 
@@ -6675,6 +6684,12 @@ package body Sem_Ch4 is
       It    : Interp;
 
    begin
+      --  Defend against previous error
+
+      if Nkind (R) = N_Error then
+         return;
+      end if;
+
       if T1 = Universal_Integer or else T1 = Universal_Real
 
         --  If the left operand of an equality operator is null, the visibility
@@ -6697,6 +6712,7 @@ package body Sem_Ch4 is
                Get_Next_Interp (Index, It);
             end loop;
          end if;
+
       elsif Has_Compatible_Type (R, T1) or else Covers (Etype (R), T1) then
          Add_One_Interp (N, Op_Id, Standard_Boolean, Base_Type (T1));
       end if;
@@ -7087,7 +7103,9 @@ package body Sem_Ch4 is
          --  Finally, also check for RM 4.5.2 (9.6/2).
 
          if T1 /= Standard_Void_Type
-           and then (Universal_Access or else Has_Compatible_Type (R, T1))
+           and then (Universal_Access
+                      or else
+                     Has_Compatible_Type (R, T1, For_Comparison => True))
 
            and then
              ((not Is_Limited_Type (T1)
@@ -7148,9 +7166,7 @@ package body Sem_Ch4 is
       --  If left operand is aggregate, the right operand has to
       --  provide a usable type for it.
 
-      if Nkind (L) = N_Aggregate
-        and then Nkind (R) /= N_Aggregate
-      then
+      if Nkind (L) = N_Aggregate and then Nkind (R) /= N_Aggregate then
          Find_Equality_Types (L => R, R => L, Op_Id => Op_Id, N => N);
          return;
       end if;
@@ -7541,6 +7557,9 @@ package body Sem_Ch4 is
             then
                return;
 
+            elsif Has_Possible_Literal_Aspects (N) then
+               return;
+
             --  If we have a logical operator, one of whose operands is
             --  Boolean, then we know that the other operand cannot resolve to
             --  Boolean (since we got no interpretations), but in that case we
@@ -7857,6 +7876,69 @@ package body Sem_Ch4 is
       end if;
    end Operator_Check;
 
+   ----------------------------------
+   -- Has_Possible_Literal_Aspects --
+   ----------------------------------
+
+   function Has_Possible_Literal_Aspects (N : Node_Id) return Boolean is
+      R : constant Node_Id := Right_Opnd (N);
+      L : Node_Id := Empty;
+
+      procedure Check_Literal_Opnd (Opnd : Node_Id);
+      --  If an operand is a literal to which an aspect may apply,
+      --  add the corresponding type to operator node.
+
+      ------------------------
+      -- Check_Literal_Opnd --
+      ------------------------
+
+      procedure Check_Literal_Opnd (Opnd : Node_Id) is
+      begin
+         if Nkind (Opnd) in N_Numeric_Or_String_Literal
+           or else (Is_Entity_Name (Opnd)
+             and then Present (Entity (Opnd))
+             and then Is_Named_Number (Entity (Opnd)))
+         then
+            Add_One_Interp (N, Etype (Opnd), Etype (Opnd));
+         end if;
+      end Check_Literal_Opnd;
+
+   --  Start of processing for Has_Possible_Literal_Aspects
+
+   begin
+      if Ada_Version < Ada_2022 then
+         return False;
+      end if;
+
+      if Nkind (N) in N_Binary_Op then
+         L := Left_Opnd (N);
+      else
+         L := Empty;
+      end if;
+      Check_Literal_Opnd (R);
+
+      --  Check left operand only if right one did not provide a
+      --  possible interpretation. Note that literal types are not
+      --  overloadable, in the sense that there is no overloadable
+      --  entity name whose several interpretations can be used to
+      --  indicate possible resulting types, so there is no way to
+      --  provide more than one interpretation to the operator node.
+      --  The choice of one operand over the other is arbitrary at
+      --  this point, and may lead to spurious resolution when both
+      --  operands are literals of different kinds, but the second
+      --  pass of resolution will examine anew both operands to
+      --  determine whether a user-defined literal may apply to
+      --  either or both.
+
+      if Present (L)
+        and then Etype (N) = Any_Type
+      then
+         Check_Literal_Opnd (L);
+      end if;
+
+      return Etype (N) /= Any_Type;
+   end Has_Possible_Literal_Aspects;
+
    --------------------------------
    -- Remove_Abstract_Operations --
    --------------------------------
diff --git a/gcc/ada/sem_ch5.adb b/gcc/ada/sem_ch5.adb
index 45d2457..e62161a 100644
--- a/gcc/ada/sem_ch5.adb
+++ b/gcc/ada/sem_ch5.adb
@@ -682,6 +682,7 @@ package body Sem_Ch5 is
                Ent := Lhs;
                while Nkind (Ent) in N_Has_Entity
                  and then Present (Entity (Ent))
+                 and then Is_Object (Entity (Ent))
                  and then Present (Renamed_Object (Entity (Ent)))
                loop
                   Ent := Renamed_Object (Entity (Ent));
diff --git a/gcc/ada/sem_ch6.adb b/gcc/ada/sem_ch6.adb
index 1486918..af8756b 100644
--- a/gcc/ada/sem_ch6.adb
+++ b/gcc/ada/sem_ch6.adb
@@ -6254,7 +6254,7 @@ package body Sem_Ch6 is
                           (Old_Id, Old_Type, New_Type)
          then
             Error_Msg_N ("result subtypes conform but come from different "
-                          & "declarations??", New_Id);
+                          & "declarations?_p?", New_Id);
          end if;
 
          --  Ada 2005 (AI-231): In case of anonymous access types check the
@@ -6462,7 +6462,7 @@ package body Sem_Ch6 is
                           (Old_Id, Old_Formal_Base, New_Formal_Base)
          then
             Error_Msg_N ("formal subtypes conform but come from "
-                          & "different declarations??", New_Formal);
+                          & "different declarations?_p?", New_Formal);
          end if;
 
          --  For mode conformance, mode must match
@@ -10918,11 +10918,11 @@ package body Sem_Ch6 is
                         if Pragma_Name (Prag) = Name_Precondition then
                            Error_Msg_N
                              ("info: & inherits `Pre''Class` aspect from "
-                              & "#?L?", E);
+                              & "#?.l?", E);
                         else
                            Error_Msg_N
                              ("info: & inherits `Post''Class` aspect from "
-                              & "#?L?", E);
+                              & "#?.l?", E);
                         end if;
                      end if;
 
@@ -11981,11 +11981,11 @@ package body Sem_Ch6 is
          --  renaming declaration becomes hidden.
 
          if Ekind (E) = E_Package
-           and then Present (Renamed_Object (E))
-           and then Renamed_Object (E) = Current_Scope
-           and then Nkind (Parent (Renamed_Object (E))) =
+           and then Present (Renamed_Entity (E))
+           and then Renamed_Entity (E) = Current_Scope
+           and then Nkind (Parent (Renamed_Entity (E))) =
                                                      N_Package_Specification
-           and then Present (Generic_Parent (Parent (Renamed_Object (E))))
+           and then Present (Generic_Parent (Parent (Renamed_Entity (E))))
          then
             Set_Is_Hidden (E);
             Set_Is_Immediately_Visible (E, False);
diff --git a/gcc/ada/sem_ch7.adb b/gcc/ada/sem_ch7.adb
index 3852a9a..a0bddb1 100644
--- a/gcc/ada/sem_ch7.adb
+++ b/gcc/ada/sem_ch7.adb
@@ -3363,12 +3363,12 @@ package body Sem_Ch7 is
       --  Body required if library package with pragma Elaborate_Body
 
       elsif Has_Pragma_Elaborate_Body (Pack_Id) then
-         Error_Msg_N ("info: & requires body (Elaborate_Body)?Y?", Pack_Id);
+         Error_Msg_N ("info: & requires body (Elaborate_Body)?.y?", Pack_Id);
 
       --  Body required if subprogram
 
       elsif Is_Subprogram_Or_Generic_Subprogram (Pack_Id) then
-         Error_Msg_N ("info: & requires body (subprogram case)?Y?", Pack_Id);
+         Error_Msg_N ("info: & requires body (subprogram case)?.y?", Pack_Id);
 
       --  Body required if generic parent has Elaborate_Body
 
@@ -3381,7 +3381,7 @@ package body Sem_Ch7 is
          begin
             if Has_Pragma_Elaborate_Body (G_P) then
                Error_Msg_N
-                 ("info: & requires body (generic parent Elaborate_Body)?Y?",
+                 ("info: & requires body (generic parent Elaborate_Body)?.y?",
                   Pack_Id);
             end if;
          end;
@@ -3399,7 +3399,7 @@ package body Sem_Ch7 is
                        (Node (First_Elmt (Abstract_States (Pack_Id))))
       then
          Error_Msg_N
-           ("info: & requires body (non-null abstract state aspect)?Y?",
+           ("info: & requires body (non-null abstract state aspect)?.y?",
             Pack_Id);
       end if;
 
@@ -3410,7 +3410,8 @@ package body Sem_Ch7 is
          if Requires_Completion_In_Body (E, Pack_Id) then
             Error_Msg_Node_2 := E;
             Error_Msg_NE
-              ("info: & requires body (& requires completion)?Y?", E, Pack_Id);
+              ("info: & requires body (& requires completion)?.y?", E,
+               Pack_Id);
          end if;
 
          Next_Entity (E);
diff --git a/gcc/ada/sem_ch8.adb b/gcc/ada/sem_ch8.adb
index 7b3dfa6..686d437 100644
--- a/gcc/ada/sem_ch8.adb
+++ b/gcc/ada/sem_ch8.adb
@@ -576,10 +576,10 @@ package body Sem_Ch8 is
         and then Present (Entity (Nam))
         and then Ekind (Entity (Nam)) = E_Exception
       then
-         if Present (Renamed_Object (Entity (Nam))) then
-            Set_Renamed_Object (Id, Renamed_Object (Entity (Nam)));
+         if Present (Renamed_Entity (Entity (Nam))) then
+            Set_Renamed_Entity (Id, Renamed_Entity (Entity (Nam)));
          else
-            Set_Renamed_Object (Id, Entity (Nam));
+            Set_Renamed_Entity (Id, Entity (Nam));
          end if;
 
          --  The exception renaming declaration may become Ghost if it renames
@@ -706,10 +706,10 @@ package body Sem_Ch8 is
          Error_Msg_N ("invalid generic unit name", Name (N));
 
       else
-         if Present (Renamed_Object (Old_P)) then
-            Set_Renamed_Object (New_P, Renamed_Object (Old_P));
+         if Present (Renamed_Entity (Old_P)) then
+            Set_Renamed_Entity (New_P, Renamed_Entity (Old_P));
          else
-            Set_Renamed_Object (New_P, Old_P);
+            Set_Renamed_Entity (New_P, Old_P);
          end if;
 
          --  The generic renaming declaration may become Ghost if it renames a
@@ -1327,13 +1327,13 @@ package body Sem_Ch8 is
                  and then Comes_From_Source (Nam)
                then
                   Error_Msg_N
-                    ("renaming function result object is suspicious?R?", Nam);
+                    ("renaming function result object is suspicious?.r?", Nam);
                   Error_Msg_NE
-                    ("\function & will be called only once?R?", Nam,
+                    ("\function & will be called only once?.r?", Nam,
                      Entity (Name (Nam)));
                   Error_Msg_N -- CODEFIX
                     ("\suggest using an initialized constant object "
-                     & "instead?R?", Nam);
+                     & "instead?.r?", Nam);
                end if;
          end case;
       end if;
@@ -1658,10 +1658,10 @@ package body Sem_Ch8 is
          Mutate_Ekind (New_P, E_Package);
          Set_Etype (New_P, Standard_Void_Type);
 
-         if Present (Renamed_Object (Old_P)) then
-            Set_Renamed_Object (New_P, Renamed_Object (Old_P));
+         if Present (Renamed_Entity (Old_P)) then
+            Set_Renamed_Entity (New_P, Renamed_Entity (Old_P));
          else
-            Set_Renamed_Object (New_P, Old_P);
+            Set_Renamed_Entity (New_P, Old_P);
          end if;
 
          --  The package renaming declaration may become Ghost if it renames a
@@ -2109,42 +2109,6 @@ package body Sem_Ch8 is
       Old_S       : Entity_Id := Empty;
       Rename_Spec : Entity_Id;
 
-      procedure Build_Class_Wide_Wrapper
-        (Ren_Id  : out Entity_Id;
-         Wrap_Id : out Entity_Id);
-      --  Ada 2012 (AI05-0071): A generic/instance scenario involving a formal
-      --  type with unknown discriminants and a generic primitive operation of
-      --  the said type with a box require special processing when the actual
-      --  is a class-wide type:
-      --
-      --    generic
-      --       type Formal_Typ (<>) is private;
-      --       with procedure Prim_Op (Param : Formal_Typ) is <>;
-      --    package Gen is ...
-      --
-      --    package Inst is new Gen (Actual_Typ'Class);
-      --
-      --  In this case the general renaming mechanism used in the prologue of
-      --  an instance no longer applies:
-      --
-      --    procedure Prim_Op (Param : Formal_Typ) renames Prim_Op;
-      --
-      --  The above is replaced the following wrapper/renaming combination:
-      --
-      --    procedure Wrapper (Param : Formal_Typ) is  --  wrapper
-      --    begin
-      --       Prim_Op (Param);                        --  primitive
-      --    end Wrapper;
-      --
-      --    procedure Prim_Op (Param : Formal_Typ) renames Wrapper;
-      --
-      --  This transformation applies only if there is no explicit visible
-      --  class-wide operation at the point of the instantiation. Ren_Id is
-      --  the entity of the renaming declaration. When the transformation
-      --  applies, Wrap_Id is the entity of the generated class-wide wrapper
-      --  (or Any_Id). Otherwise, Wrap_Id is the entity of the class-wide
-      --  operation.
-
       procedure Check_Null_Exclusion
         (Ren : Entity_Id;
          Sub : Entity_Id);
@@ -2170,9 +2134,21 @@ package body Sem_Ch8 is
       --  incomplete untagged formal (RM 13.14(10.2/3)).
 
       function Has_Class_Wide_Actual return Boolean;
-      --  Ada 2012 (AI05-071, AI05-0131): True if N is the renaming for a
-      --  defaulted formal subprogram where the actual for the controlling
-      --  formal type is class-wide.
+      --  Ada 2012 (AI05-071, AI05-0131) and Ada 2022 (AI12-0165): True if N is
+      --  the renaming for a defaulted formal subprogram where the actual for
+      --  the controlling formal type is class-wide.
+
+      procedure Handle_Instance_With_Class_Wide_Type
+        (Inst_Node    : Node_Id;
+         Ren_Id       : Entity_Id;
+         Wrapped_Prim : out Entity_Id;
+         Wrap_Id      : out Entity_Id);
+      --  Ada 2012 (AI05-0071), Ada 2022 (AI12-0165): when the actual type
+      --  of an instantiation is a class-wide type T'Class we may need to
+      --  wrap a primitive operation of T; this routine looks for a suitable
+      --  primitive to be wrapped and (if the wrapper is required) returns the
+      --  Id of the wrapped primitive and the Id of the built wrapper. Ren_Id
+      --  is the defining entity for the renamed subprogram specification.
 
       function Original_Subprogram (Subp : Entity_Id) return Entity_Id;
       --  Find renamed entity when the declaration is a renaming_as_body and
@@ -2181,550 +2157,6 @@ package body Sem_Ch8 is
       --  before the subprogram it completes is frozen, and renaming indirectly
       --  renames the subprogram itself.(Defect Report 8652/0027).
 
-      ------------------------------
-      -- Build_Class_Wide_Wrapper --
-      ------------------------------
-
-      procedure Build_Class_Wide_Wrapper
-        (Ren_Id  : out Entity_Id;
-         Wrap_Id : out Entity_Id)
-      is
-         Loc : constant Source_Ptr := Sloc (N);
-
-         function Build_Call
-           (Subp_Id : Entity_Id;
-            Params  : List_Id) return Node_Id;
-         --  Create a dispatching call to invoke routine Subp_Id with actuals
-         --  built from the parameter specifications of list Params.
-
-         function Build_Expr_Fun_Call
-           (Subp_Id : Entity_Id;
-            Params  : List_Id) return Node_Id;
-         --  Create a dispatching call to invoke function Subp_Id with actuals
-         --  built from the parameter specifications of list Params. Return
-         --  directly the call, so that it can be used inside an expression
-         --  function. This is a specificity of the GNATprove mode.
-
-         function Build_Spec (Subp_Id : Entity_Id) return Node_Id;
-         --  Create a subprogram specification based on the subprogram profile
-         --  of Subp_Id.
-
-         function Find_Primitive (Typ : Entity_Id) return Entity_Id;
-         --  Find a primitive subprogram of type Typ which matches the profile
-         --  of the renaming declaration.
-
-         procedure Interpretation_Error (Subp_Id : Entity_Id);
-         --  Emit a continuation error message suggesting subprogram Subp_Id as
-         --  a possible interpretation.
-
-         function Is_Intrinsic_Equality (Subp_Id : Entity_Id) return Boolean;
-         --  Determine whether subprogram Subp_Id denotes the intrinsic "="
-         --  operator.
-
-         function Is_Suitable_Candidate (Subp_Id : Entity_Id) return Boolean;
-         --  Determine whether subprogram Subp_Id is a suitable candidate for
-         --  the role of a wrapped subprogram.
-
-         ----------------
-         -- Build_Call --
-         ----------------
-
-         function Build_Call
-           (Subp_Id : Entity_Id;
-            Params  : List_Id) return Node_Id
-         is
-            Actuals  : constant List_Id := New_List;
-            Call_Ref : constant Node_Id := New_Occurrence_Of (Subp_Id, Loc);
-            Formal   : Node_Id;
-
-         begin
-            --  Build the actual parameters of the call
-
-            Formal := First (Params);
-            while Present (Formal) loop
-               Append_To (Actuals,
-                 Make_Identifier (Loc, Chars (Defining_Identifier (Formal))));
-               Next (Formal);
-            end loop;
-
-            --  Generate:
-            --    return Subp_Id (Actuals);
-
-            if Ekind (Subp_Id) in E_Function | E_Operator then
-               return
-                 Make_Simple_Return_Statement (Loc,
-                   Expression =>
-                     Make_Function_Call (Loc,
-                       Name                   => Call_Ref,
-                       Parameter_Associations => Actuals));
-
-            --  Generate:
-            --    Subp_Id (Actuals);
-
-            else
-               return
-                 Make_Procedure_Call_Statement (Loc,
-                   Name                   => Call_Ref,
-                   Parameter_Associations => Actuals);
-            end if;
-         end Build_Call;
-
-         -------------------------
-         -- Build_Expr_Fun_Call --
-         -------------------------
-
-         function Build_Expr_Fun_Call
-           (Subp_Id : Entity_Id;
-            Params  : List_Id) return Node_Id
-         is
-            Actuals  : constant List_Id := New_List;
-            Call_Ref : constant Node_Id := New_Occurrence_Of (Subp_Id, Loc);
-            Formal   : Node_Id;
-
-         begin
-            pragma Assert (Ekind (Subp_Id) in E_Function | E_Operator);
-
-            --  Build the actual parameters of the call
-
-            Formal := First (Params);
-            while Present (Formal) loop
-               Append_To (Actuals,
-                 Make_Identifier (Loc, Chars (Defining_Identifier (Formal))));
-               Next (Formal);
-            end loop;
-
-            --  Generate:
-            --    Subp_Id (Actuals);
-
-            return
-              Make_Function_Call (Loc,
-                Name                   => Call_Ref,
-                Parameter_Associations => Actuals);
-         end Build_Expr_Fun_Call;
-
-         ----------------
-         -- Build_Spec --
-         ----------------
-
-         function Build_Spec (Subp_Id : Entity_Id) return Node_Id is
-            Params  : constant List_Id   := Copy_Parameter_List (Subp_Id);
-            Spec_Id : constant Entity_Id :=
-                        Make_Defining_Identifier (Loc,
-                          Chars => New_External_Name (Chars (Subp_Id), 'R'));
-
-         begin
-            if Ekind (Formal_Spec) = E_Procedure then
-               return
-                 Make_Procedure_Specification (Loc,
-                   Defining_Unit_Name       => Spec_Id,
-                   Parameter_Specifications => Params);
-            else
-               return
-                 Make_Function_Specification (Loc,
-                   Defining_Unit_Name       => Spec_Id,
-                   Parameter_Specifications => Params,
-                   Result_Definition =>
-                     New_Copy_Tree (Result_Definition (Spec)));
-            end if;
-         end Build_Spec;
-
-         --------------------
-         -- Find_Primitive --
-         --------------------
-
-         function Find_Primitive (Typ : Entity_Id) return Entity_Id is
-            procedure Replace_Parameter_Types (Spec : Node_Id);
-            --  Given a specification Spec, replace all class-wide parameter
-            --  types with reference to type Typ.
-
-            -----------------------------
-            -- Replace_Parameter_Types --
-            -----------------------------
-
-            procedure Replace_Parameter_Types (Spec : Node_Id) is
-               Formal     : Node_Id;
-               Formal_Id  : Entity_Id;
-               Formal_Typ : Node_Id;
-
-            begin
-               Formal := First (Parameter_Specifications (Spec));
-               while Present (Formal) loop
-                  Formal_Id  := Defining_Identifier (Formal);
-                  Formal_Typ := Parameter_Type (Formal);
-
-                  --  Create a new entity for each class-wide formal to prevent
-                  --  aliasing with the original renaming. Replace the type of
-                  --  such a parameter with the candidate type.
-
-                  if Nkind (Formal_Typ) = N_Identifier
-                    and then Is_Class_Wide_Type (Etype (Formal_Typ))
-                  then
-                     Set_Defining_Identifier (Formal,
-                       Make_Defining_Identifier (Loc, Chars (Formal_Id)));
-
-                     Set_Parameter_Type (Formal, New_Occurrence_Of (Typ, Loc));
-                  end if;
-
-                  Next (Formal);
-               end loop;
-            end Replace_Parameter_Types;
-
-            --  Local variables
-
-            Alt_Ren  : constant Node_Id := New_Copy_Tree (N);
-            Alt_Nam  : constant Node_Id := Name (Alt_Ren);
-            Alt_Spec : constant Node_Id := Specification (Alt_Ren);
-            Subp_Id  : Entity_Id;
-
-         --  Start of processing for Find_Primitive
-
-         begin
-            --  Each attempt to find a suitable primitive of a particular type
-            --  operates on its own copy of the original renaming. As a result
-            --  the original renaming is kept decoration and side-effect free.
-
-            --  Inherit the overloaded status of the renamed subprogram name
-
-            if Is_Overloaded (Nam) then
-               Set_Is_Overloaded (Alt_Nam);
-               Save_Interps (Nam, Alt_Nam);
-            end if;
-
-            --  The copied renaming is hidden from visibility to prevent the
-            --  pollution of the enclosing context.
-
-            Set_Defining_Unit_Name (Alt_Spec, Make_Temporary (Loc, 'R'));
-
-            --  The types of all class-wide parameters must be changed to the
-            --  candidate type.
-
-            Replace_Parameter_Types (Alt_Spec);
-
-            --  Try to find a suitable primitive which matches the altered
-            --  profile of the renaming specification.
-
-            Subp_Id :=
-              Find_Renamed_Entity
-                (N         => Alt_Ren,
-                 Nam       => Name (Alt_Ren),
-                 New_S     => Analyze_Subprogram_Specification (Alt_Spec),
-                 Is_Actual => Is_Actual);
-
-            --  Do not return Any_Id if the resolion of the altered profile
-            --  failed as this complicates further checks on the caller side,
-            --  return Empty instead.
-
-            if Subp_Id = Any_Id then
-               return Empty;
-            else
-               return Subp_Id;
-            end if;
-         end Find_Primitive;
-
-         --------------------------
-         -- Interpretation_Error --
-         --------------------------
-
-         procedure Interpretation_Error (Subp_Id : Entity_Id) is
-         begin
-            Error_Msg_Sloc := Sloc (Subp_Id);
-
-            if Is_Internal (Subp_Id) then
-               Error_Msg_NE
-                 ("\\possible interpretation: predefined & #",
-                  Spec, Formal_Spec);
-            else
-               Error_Msg_NE
-                 ("\\possible interpretation: & defined #", Spec, Formal_Spec);
-            end if;
-         end Interpretation_Error;
-
-         ---------------------------
-         -- Is_Intrinsic_Equality --
-         ---------------------------
-
-         function Is_Intrinsic_Equality (Subp_Id : Entity_Id) return Boolean is
-         begin
-            return
-              Ekind (Subp_Id) = E_Operator
-                and then Chars (Subp_Id) = Name_Op_Eq
-                and then Is_Intrinsic_Subprogram (Subp_Id);
-         end Is_Intrinsic_Equality;
-
-         ---------------------------
-         -- Is_Suitable_Candidate --
-         ---------------------------
-
-         function Is_Suitable_Candidate (Subp_Id : Entity_Id) return Boolean is
-         begin
-            if No (Subp_Id) then
-               return False;
-
-            --  An intrinsic subprogram is never a good candidate. This is an
-            --  indication of a missing primitive, either defined directly or
-            --  inherited from a parent tagged type.
-
-            elsif Is_Intrinsic_Subprogram (Subp_Id) then
-               return False;
-
-            else
-               return True;
-            end if;
-         end Is_Suitable_Candidate;
-
-         --  Local variables
-
-         Actual_Typ : Entity_Id := Empty;
-         --  The actual class-wide type for Formal_Typ
-
-         CW_Prim_OK : Boolean;
-         CW_Prim_Op : Entity_Id;
-         --  The class-wide subprogram (if available) which corresponds to the
-         --  renamed generic formal subprogram.
-
-         Formal_Typ : Entity_Id := Empty;
-         --  The generic formal type with unknown discriminants
-
-         Root_Prim_OK : Boolean;
-         Root_Prim_Op : Entity_Id;
-         --  The root type primitive (if available) which corresponds to the
-         --  renamed generic formal subprogram.
-
-         Root_Typ : Entity_Id := Empty;
-         --  The root type of Actual_Typ
-
-         Body_Decl : Node_Id;
-         Formal    : Node_Id;
-         Prim_Op   : Entity_Id;
-         Spec_Decl : Node_Id;
-         New_Spec  : Node_Id;
-
-      --  Start of processing for Build_Class_Wide_Wrapper
-
-      begin
-         --  Analyze the specification of the renaming in case the generation
-         --  of the class-wide wrapper fails.
-
-         Ren_Id  := Analyze_Subprogram_Specification (Spec);
-         Wrap_Id := Any_Id;
-
-         --  Do not attempt to build a wrapper if the renaming is in error
-
-         if Error_Posted (Nam) then
-            return;
-         end if;
-
-         --  Analyze the renamed name, but do not resolve it. The resolution is
-         --  completed once a suitable subprogram is found.
-
-         Analyze (Nam);
-
-         --  When the renamed name denotes the intrinsic operator equals, the
-         --  name must be treated as overloaded. This allows for a potential
-         --  match against the root type's predefined equality function.
-
-         if Is_Intrinsic_Equality (Entity (Nam)) then
-            Set_Is_Overloaded (Nam);
-            Collect_Interps   (Nam);
-         end if;
-
-         --  Step 1: Find the generic formal type with unknown discriminants
-         --  and its corresponding class-wide actual type from the renamed
-         --  generic formal subprogram.
-
-         Formal := First_Formal (Formal_Spec);
-         while Present (Formal) loop
-            if Has_Unknown_Discriminants (Etype (Formal))
-              and then not Is_Class_Wide_Type (Etype (Formal))
-              and then Is_Class_Wide_Type (Get_Instance_Of (Etype (Formal)))
-            then
-               Formal_Typ := Etype (Formal);
-               Actual_Typ := Base_Type (Get_Instance_Of (Formal_Typ));
-               Root_Typ   := Root_Type (Actual_Typ);
-               exit;
-            end if;
-
-            Next_Formal (Formal);
-         end loop;
-
-         --  The specification of the generic formal subprogram should always
-         --  contain a formal type with unknown discriminants whose actual is
-         --  a class-wide type, otherwise this indicates a failure in routine
-         --  Has_Class_Wide_Actual.
-
-         pragma Assert (Present (Formal_Typ));
-
-         --  Step 2: Find the proper class-wide subprogram or primitive which
-         --  corresponds to the renamed generic formal subprogram.
-
-         CW_Prim_Op   := Find_Primitive (Actual_Typ);
-         CW_Prim_OK   := Is_Suitable_Candidate (CW_Prim_Op);
-         Root_Prim_Op := Find_Primitive (Root_Typ);
-         Root_Prim_OK := Is_Suitable_Candidate (Root_Prim_Op);
-
-         --  The class-wide actual type has two subprograms which correspond to
-         --  the renamed generic formal subprogram:
-
-         --    with procedure Prim_Op (Param : Formal_Typ);
-
-         --    procedure Prim_Op (Param : Actual_Typ);  --  may be inherited
-         --    procedure Prim_Op (Param : Actual_Typ'Class);
-
-         --  Even though the declaration of the two subprograms is legal, a
-         --  call to either one is ambiguous and therefore illegal.
-
-         if CW_Prim_OK and Root_Prim_OK then
-
-            --  A user-defined primitive has precedence over a predefined one
-
-            if Is_Internal (CW_Prim_Op)
-              and then not Is_Internal (Root_Prim_Op)
-            then
-               Prim_Op := Root_Prim_Op;
-
-            elsif Is_Internal (Root_Prim_Op)
-              and then not Is_Internal (CW_Prim_Op)
-            then
-               Prim_Op := CW_Prim_Op;
-
-            elsif CW_Prim_Op = Root_Prim_Op then
-               Prim_Op := Root_Prim_Op;
-
-            --  The two subprograms are legal but the class-wide subprogram is
-            --  a class-wide wrapper built for a previous instantiation; the
-            --  wrapper has precedence.
-
-            elsif Present (Alias (CW_Prim_Op))
-              and then Is_Class_Wide_Wrapper (Ultimate_Alias (CW_Prim_Op))
-            then
-               Prim_Op := CW_Prim_Op;
-
-            --  Otherwise both candidate subprograms are user-defined and
-            --  ambiguous.
-
-            else
-               Error_Msg_NE
-                 ("ambiguous actual for generic subprogram &",
-                  Spec, Formal_Spec);
-               Interpretation_Error (Root_Prim_Op);
-               Interpretation_Error (CW_Prim_Op);
-               return;
-            end if;
-
-         elsif CW_Prim_OK and not Root_Prim_OK then
-            Prim_Op := CW_Prim_Op;
-
-         elsif not CW_Prim_OK and Root_Prim_OK then
-            Prim_Op := Root_Prim_Op;
-
-         --  An intrinsic equality may act as a suitable candidate in the case
-         --  of a null type extension where the parent's equality is hidden. A
-         --  call to an intrinsic equality is expanded as dispatching.
-
-         elsif Present (Root_Prim_Op)
-           and then Is_Intrinsic_Equality (Root_Prim_Op)
-         then
-            Prim_Op := Root_Prim_Op;
-
-         --  Otherwise there are no candidate subprograms. Let the caller
-         --  diagnose the error.
-
-         else
-            return;
-         end if;
-
-         --  At this point resolution has taken place and the name is no longer
-         --  overloaded. Mark the primitive as referenced.
-
-         Set_Is_Overloaded (Name (N), False);
-         Set_Referenced    (Prim_Op);
-
-         --  Do not generate a wrapper when the only candidate is a class-wide
-         --  subprogram. Instead modify the renaming to directly map the actual
-         --  to the generic formal.
-
-         if CW_Prim_OK and then Prim_Op = CW_Prim_Op then
-            Wrap_Id := Prim_Op;
-            Rewrite (Nam, New_Occurrence_Of (Prim_Op, Loc));
-            return;
-         end if;
-
-         --  Step 3: Create the declaration and the body of the wrapper, insert
-         --  all the pieces into the tree.
-
-         --  In GNATprove mode, create a function wrapper in the form of an
-         --  expression function, so that an implicit postcondition relating
-         --  the result of calling the wrapper function and the result of the
-         --  dispatching call to the wrapped function is known during proof.
-
-         if GNATprove_Mode
-           and then Ekind (Ren_Id) in E_Function | E_Operator
-         then
-            New_Spec := Build_Spec (Ren_Id);
-            Body_Decl :=
-              Make_Expression_Function (Loc,
-                Specification => New_Spec,
-                Expression    =>
-                  Build_Expr_Fun_Call
-                    (Subp_Id => Prim_Op,
-                     Params  => Parameter_Specifications (New_Spec)));
-
-            Wrap_Id := Defining_Entity (Body_Decl);
-
-         --  Otherwise, create separate spec and body for the subprogram
-
-         else
-            Spec_Decl :=
-              Make_Subprogram_Declaration (Loc,
-                Specification => Build_Spec (Ren_Id));
-            Insert_Before_And_Analyze (N, Spec_Decl);
-
-            Wrap_Id := Defining_Entity (Spec_Decl);
-
-            Body_Decl :=
-              Make_Subprogram_Body (Loc,
-                Specification              => Build_Spec (Ren_Id),
-                Declarations               => New_List,
-                Handled_Statement_Sequence =>
-                  Make_Handled_Sequence_Of_Statements (Loc,
-                    Statements => New_List (
-                      Build_Call
-                        (Subp_Id => Prim_Op,
-                         Params  =>
-                           Parameter_Specifications
-                             (Specification (Spec_Decl))))));
-
-            Set_Corresponding_Body (Spec_Decl, Defining_Entity (Body_Decl));
-         end if;
-
-         Set_Is_Class_Wide_Wrapper (Wrap_Id);
-
-         --  If the operator carries an Eliminated pragma, indicate that the
-         --  wrapper is also to be eliminated, to prevent spurious error when
-         --  using gnatelim on programs that include box-initialization of
-         --  equality operators.
-
-         Set_Is_Eliminated (Wrap_Id, Is_Eliminated (Prim_Op));
-
-         --  In GNATprove mode, insert the body in the tree for analysis
-
-         if GNATprove_Mode then
-            Insert_Before_And_Analyze (N, Body_Decl);
-         end if;
-
-         --  The generated body does not freeze and must be analyzed when the
-         --  class-wide wrapper is frozen. The body is only needed if expansion
-         --  is enabled.
-
-         if Expander_Active then
-            Append_Freeze_Action (Wrap_Id, Body_Decl);
-         end if;
-
-         --  Step 4: The subprogram renaming aliases the wrapper
-
-         Rewrite (Nam, New_Occurrence_Of (Wrap_Id, Loc));
-      end Build_Class_Wide_Wrapper;
-
       --------------------------
       -- Check_Null_Exclusion --
       --------------------------
@@ -2919,6 +2351,703 @@ package body Sem_Ch8 is
          return False;
       end Has_Class_Wide_Actual;
 
+      ------------------------------------------
+      -- Handle_Instance_With_Class_Wide_Type --
+      ------------------------------------------
+
+      procedure Handle_Instance_With_Class_Wide_Type
+        (Inst_Node    : Node_Id;
+         Ren_Id       : Entity_Id;
+         Wrapped_Prim : out Entity_Id;
+         Wrap_Id      : out Entity_Id)
+      is
+         procedure Build_Class_Wide_Wrapper
+           (Ren_Id  : Entity_Id;
+            Prim_Op : Entity_Id;
+            Wrap_Id : out Entity_Id);
+         --  Build a wrapper for the renaming Ren_Id of subprogram Prim_Op.
+
+         procedure Find_Suitable_Candidate
+           (Prim_Op    : out Entity_Id;
+            Is_CW_Prim : out Boolean);
+         --  Look for a suitable primitive to be wrapped (Prim_Op); Is_CW_Prim
+         --  indicates that the found candidate is a class-wide primitive (to
+         --  help the caller decide if the wrapper is required).
+
+         ------------------------------
+         -- Build_Class_Wide_Wrapper --
+         ------------------------------
+
+         procedure Build_Class_Wide_Wrapper
+           (Ren_Id  : Entity_Id;
+            Prim_Op : Entity_Id;
+            Wrap_Id : out Entity_Id)
+         is
+            Loc : constant Source_Ptr := Sloc (N);
+
+            function Build_Call
+              (Subp_Id : Entity_Id;
+               Params  : List_Id) return Node_Id;
+            --  Create a dispatching call to invoke routine Subp_Id with
+            --  actuals built from the parameter specifications of list Params.
+
+            function Build_Expr_Fun_Call
+              (Subp_Id : Entity_Id;
+               Params  : List_Id) return Node_Id;
+            --  Create a dispatching call to invoke function Subp_Id with
+            --  actuals built from the parameter specifications of list Params.
+            --  Directly return the call, so that it can be used inside an
+            --  expression function. This is a requirement of GNATprove mode.
+
+            function Build_Spec (Subp_Id : Entity_Id) return Node_Id;
+            --  Create a subprogram specification based on the subprogram
+            --  profile of Subp_Id.
+
+            ----------------
+            -- Build_Call --
+            ----------------
+
+            function Build_Call
+              (Subp_Id : Entity_Id;
+               Params  : List_Id) return Node_Id
+            is
+               Actuals  : constant List_Id := New_List;
+               Call_Ref : constant Node_Id := New_Occurrence_Of (Subp_Id, Loc);
+               Formal   : Node_Id;
+
+            begin
+               --  Build the actual parameters of the call
+
+               Formal := First (Params);
+               while Present (Formal) loop
+                  Append_To (Actuals,
+                    Make_Identifier (Loc,
+                      Chars (Defining_Identifier (Formal))));
+                  Next (Formal);
+               end loop;
+
+               --  Generate:
+               --    return Subp_Id (Actuals);
+
+               if Ekind (Subp_Id) in E_Function | E_Operator then
+                  return
+                    Make_Simple_Return_Statement (Loc,
+                      Expression =>
+                        Make_Function_Call (Loc,
+                          Name                   => Call_Ref,
+                          Parameter_Associations => Actuals));
+
+               --  Generate:
+               --    Subp_Id (Actuals);
+
+               else
+                  return
+                    Make_Procedure_Call_Statement (Loc,
+                      Name                   => Call_Ref,
+                      Parameter_Associations => Actuals);
+               end if;
+            end Build_Call;
+
+            -------------------------
+            -- Build_Expr_Fun_Call --
+            -------------------------
+
+            function Build_Expr_Fun_Call
+              (Subp_Id : Entity_Id;
+               Params  : List_Id) return Node_Id
+            is
+               Actuals  : constant List_Id := New_List;
+               Call_Ref : constant Node_Id := New_Occurrence_Of (Subp_Id, Loc);
+               Formal   : Node_Id;
+
+            begin
+               pragma Assert (Ekind (Subp_Id) in E_Function | E_Operator);
+
+               --  Build the actual parameters of the call
+
+               Formal := First (Params);
+               while Present (Formal) loop
+                  Append_To (Actuals,
+                    Make_Identifier (Loc,
+                      Chars (Defining_Identifier (Formal))));
+                  Next (Formal);
+               end loop;
+
+               --  Generate:
+               --    Subp_Id (Actuals);
+
+               return
+                 Make_Function_Call (Loc,
+                   Name                   => Call_Ref,
+                   Parameter_Associations => Actuals);
+            end Build_Expr_Fun_Call;
+
+            ----------------
+            -- Build_Spec --
+            ----------------
+
+            function Build_Spec (Subp_Id : Entity_Id) return Node_Id is
+               Params  : constant List_Id   := Copy_Parameter_List (Subp_Id);
+               Spec_Id : constant Entity_Id :=
+                           Make_Defining_Identifier (Loc,
+                             New_External_Name (Chars (Subp_Id), 'R'));
+
+            begin
+               if Ekind (Formal_Spec) = E_Procedure then
+                  return
+                    Make_Procedure_Specification (Loc,
+                      Defining_Unit_Name       => Spec_Id,
+                      Parameter_Specifications => Params);
+               else
+                  return
+                    Make_Function_Specification (Loc,
+                      Defining_Unit_Name       => Spec_Id,
+                      Parameter_Specifications => Params,
+                      Result_Definition =>
+                        New_Copy_Tree (Result_Definition (Spec)));
+               end if;
+            end Build_Spec;
+
+            --  Local variables
+
+            Body_Decl : Node_Id;
+            Spec_Decl : Node_Id;
+            New_Spec  : Node_Id;
+
+         --  Start of processing for Build_Class_Wide_Wrapper
+
+         begin
+            pragma Assert (not Error_Posted (Nam));
+
+            --  Step 1: Create the declaration and the body of the wrapper,
+            --  insert all the pieces into the tree.
+
+            --  In GNATprove mode, create a function wrapper in the form of an
+            --  expression function, so that an implicit postcondition relating
+            --  the result of calling the wrapper function and the result of
+            --  the dispatching call to the wrapped function is known during
+            --  proof.
+
+            if GNATprove_Mode
+              and then Ekind (Ren_Id) in E_Function | E_Operator
+            then
+               New_Spec := Build_Spec (Ren_Id);
+               Body_Decl :=
+                 Make_Expression_Function (Loc,
+                   Specification => New_Spec,
+                   Expression    =>
+                     Build_Expr_Fun_Call
+                       (Subp_Id => Prim_Op,
+                        Params  => Parameter_Specifications (New_Spec)));
+
+               Wrap_Id := Defining_Entity (Body_Decl);
+
+            --  Otherwise, create separate spec and body for the subprogram
+
+            else
+               Spec_Decl :=
+                 Make_Subprogram_Declaration (Loc,
+                   Specification => Build_Spec (Ren_Id));
+               Insert_Before_And_Analyze (N, Spec_Decl);
+
+               Wrap_Id := Defining_Entity (Spec_Decl);
+
+               Body_Decl :=
+                 Make_Subprogram_Body (Loc,
+                   Specification              => Build_Spec (Ren_Id),
+                   Declarations               => New_List,
+                   Handled_Statement_Sequence =>
+                     Make_Handled_Sequence_Of_Statements (Loc,
+                       Statements => New_List (
+                         Build_Call
+                           (Subp_Id => Prim_Op,
+                            Params  =>
+                              Parameter_Specifications
+                                (Specification (Spec_Decl))))));
+
+               Set_Corresponding_Body (Spec_Decl, Defining_Entity (Body_Decl));
+            end if;
+
+            Set_Is_Class_Wide_Wrapper (Wrap_Id);
+
+            --  If the operator carries an Eliminated pragma, indicate that
+            --  the wrapper is also to be eliminated, to prevent spurious
+            --  errors when using gnatelim on programs that include box-
+            --  defaulted initialization of equality operators.
+
+            Set_Is_Eliminated (Wrap_Id, Is_Eliminated (Prim_Op));
+
+            --  In GNATprove mode, insert the body in the tree for analysis
+
+            if GNATprove_Mode then
+               Insert_Before_And_Analyze (N, Body_Decl);
+            end if;
+
+            --  The generated body does not freeze and must be analyzed when
+            --  the class-wide wrapper is frozen. The body is only needed if
+            --  expansion is enabled.
+
+            if Expander_Active then
+               Append_Freeze_Action (Wrap_Id, Body_Decl);
+            end if;
+
+            --  Step 2: The subprogram renaming aliases the wrapper
+
+            Rewrite (Name (N), New_Occurrence_Of (Wrap_Id, Loc));
+         end Build_Class_Wide_Wrapper;
+
+         -----------------------------
+         -- Find_Suitable_Candidate --
+         -----------------------------
+
+         procedure Find_Suitable_Candidate
+           (Prim_Op    : out Entity_Id;
+            Is_CW_Prim : out Boolean)
+         is
+            Loc : constant Source_Ptr := Sloc (N);
+
+            function Find_Primitive (Typ : Entity_Id) return Entity_Id;
+            --  Find a primitive subprogram of type Typ which matches the
+            --  profile of the renaming declaration.
+
+            procedure Interpretation_Error (Subp_Id : Entity_Id);
+            --  Emit a continuation error message suggesting subprogram Subp_Id
+            --  as a possible interpretation.
+
+            function Is_Intrinsic_Equality
+              (Subp_Id : Entity_Id) return Boolean;
+            --  Determine whether subprogram Subp_Id denotes the intrinsic "="
+            --  operator.
+
+            function Is_Suitable_Candidate
+              (Subp_Id : Entity_Id) return Boolean;
+            --  Determine whether subprogram Subp_Id is a suitable candidate
+            --  for the role of a wrapped subprogram.
+
+            --------------------
+            -- Find_Primitive --
+            --------------------
+
+            function Find_Primitive (Typ : Entity_Id) return Entity_Id is
+               procedure Replace_Parameter_Types (Spec : Node_Id);
+               --  Given a specification Spec, replace all class-wide parameter
+               --  types with reference to type Typ.
+
+               -----------------------------
+               -- Replace_Parameter_Types --
+               -----------------------------
+
+               procedure Replace_Parameter_Types (Spec : Node_Id) is
+                  Formal     : Node_Id;
+                  Formal_Id  : Entity_Id;
+                  Formal_Typ : Node_Id;
+
+               begin
+                  Formal := First (Parameter_Specifications (Spec));
+                  while Present (Formal) loop
+                     Formal_Id  := Defining_Identifier (Formal);
+                     Formal_Typ := Parameter_Type (Formal);
+
+                     --  Create a new entity for each class-wide formal to
+                     --  prevent aliasing with the original renaming. Replace
+                     --  the type of such a parameter with the candidate type.
+
+                     if Nkind (Formal_Typ) = N_Identifier
+                       and then Is_Class_Wide_Type (Etype (Formal_Typ))
+                     then
+                        Set_Defining_Identifier (Formal,
+                          Make_Defining_Identifier (Loc, Chars (Formal_Id)));
+
+                        Set_Parameter_Type (Formal,
+                          New_Occurrence_Of (Typ, Loc));
+                     end if;
+
+                     Next (Formal);
+                  end loop;
+               end Replace_Parameter_Types;
+
+               --  Local variables
+
+               Alt_Ren  : constant Node_Id := New_Copy_Tree (N);
+               Alt_Nam  : constant Node_Id := Name (Alt_Ren);
+               Alt_Spec : constant Node_Id := Specification (Alt_Ren);
+               Subp_Id  : Entity_Id;
+
+            --  Start of processing for Find_Primitive
+
+            begin
+               --  Each attempt to find a suitable primitive of a particular
+               --  type operates on its own copy of the original renaming.
+               --  As a result the original renaming is kept decoration and
+               --  side-effect free.
+
+               --  Inherit the overloaded status of the renamed subprogram name
+
+               if Is_Overloaded (Nam) then
+                  Set_Is_Overloaded (Alt_Nam);
+                  Save_Interps (Nam, Alt_Nam);
+               end if;
+
+               --  The copied renaming is hidden from visibility to prevent the
+               --  pollution of the enclosing context.
+
+               Set_Defining_Unit_Name (Alt_Spec, Make_Temporary (Loc, 'R'));
+
+               --  The types of all class-wide parameters must be changed to
+               --  the candidate type.
+
+               Replace_Parameter_Types (Alt_Spec);
+
+               --  Try to find a suitable primitive that matches the altered
+               --  profile of the renaming specification.
+
+               Subp_Id :=
+                 Find_Renamed_Entity
+                   (N         => Alt_Ren,
+                    Nam       => Name (Alt_Ren),
+                    New_S     => Analyze_Subprogram_Specification (Alt_Spec),
+                    Is_Actual => Is_Actual);
+
+               --  Do not return Any_Id if the resolution of the altered
+               --  profile failed as this complicates further checks on
+               --  the caller side; return Empty instead.
+
+               if Subp_Id = Any_Id then
+                  return Empty;
+               else
+                  return Subp_Id;
+               end if;
+            end Find_Primitive;
+
+            --------------------------
+            -- Interpretation_Error --
+            --------------------------
+
+            procedure Interpretation_Error (Subp_Id : Entity_Id) is
+            begin
+               Error_Msg_Sloc := Sloc (Subp_Id);
+
+               if Is_Internal (Subp_Id) then
+                  Error_Msg_NE
+                    ("\\possible interpretation: predefined & #",
+                     Spec, Formal_Spec);
+               else
+                  Error_Msg_NE
+                    ("\\possible interpretation: & defined #",
+                     Spec, Formal_Spec);
+               end if;
+            end Interpretation_Error;
+
+            ---------------------------
+            -- Is_Intrinsic_Equality --
+            ---------------------------
+
+            function Is_Intrinsic_Equality (Subp_Id : Entity_Id) return Boolean
+            is
+            begin
+               return
+                 Ekind (Subp_Id) = E_Operator
+                   and then Chars (Subp_Id) = Name_Op_Eq
+                   and then Is_Intrinsic_Subprogram (Subp_Id);
+            end Is_Intrinsic_Equality;
+
+            ---------------------------
+            -- Is_Suitable_Candidate --
+            ---------------------------
+
+            function Is_Suitable_Candidate (Subp_Id : Entity_Id) return Boolean
+            is
+            begin
+               if No (Subp_Id) then
+                  return False;
+
+               --  An intrinsic subprogram is never a good candidate. This
+               --  is an indication of a missing primitive, either defined
+               --  directly or inherited from a parent tagged type.
+
+               elsif Is_Intrinsic_Subprogram (Subp_Id) then
+                  return False;
+
+               else
+                  return True;
+               end if;
+            end Is_Suitable_Candidate;
+
+            --  Local variables
+
+            Actual_Typ : Entity_Id := Empty;
+            --  The actual class-wide type for Formal_Typ
+
+            CW_Prim_OK : Boolean;
+            CW_Prim_Op : Entity_Id;
+            --  The class-wide subprogram (if available) that corresponds to
+            --  the renamed generic formal subprogram.
+
+            Formal_Typ : Entity_Id := Empty;
+            --  The generic formal type with unknown discriminants
+
+            Root_Prim_OK : Boolean;
+            Root_Prim_Op : Entity_Id;
+            --  The root type primitive (if available) that corresponds to the
+            --  renamed generic formal subprogram.
+
+            Root_Typ : Entity_Id := Empty;
+            --  The root type of Actual_Typ
+
+            Formal   : Node_Id;
+
+         --  Start of processing for Find_Suitable_Candidate
+
+         begin
+            pragma Assert (not Error_Posted (Nam));
+
+            Prim_Op    := Empty;
+            Is_CW_Prim := False;
+
+            --  Analyze the renamed name, but do not resolve it. The resolution
+            --  is completed once a suitable subprogram is found.
+
+            Analyze (Nam);
+
+            --  When the renamed name denotes the intrinsic operator equals,
+            --  the name must be treated as overloaded. This allows for a
+            --  potential match against the root type's predefined equality
+            --  function.
+
+            if Is_Intrinsic_Equality (Entity (Nam)) then
+               Set_Is_Overloaded (Nam);
+               Collect_Interps   (Nam);
+            end if;
+
+            --  Step 1: Find the generic formal type and its corresponding
+            --  class-wide actual type from the renamed generic formal
+            --  subprogram.
+
+            Formal := First_Formal (Formal_Spec);
+            while Present (Formal) loop
+               if Has_Unknown_Discriminants (Etype (Formal))
+                 and then not Is_Class_Wide_Type (Etype (Formal))
+                 and then Is_Class_Wide_Type (Get_Instance_Of (Etype (Formal)))
+               then
+                  Formal_Typ := Etype (Formal);
+                  Actual_Typ := Base_Type (Get_Instance_Of (Formal_Typ));
+                  Root_Typ   := Root_Type (Actual_Typ);
+                  exit;
+               end if;
+
+               Next_Formal (Formal);
+            end loop;
+
+            --  The specification of the generic formal subprogram should
+            --  always contain a formal type with unknown discriminants whose
+            --  actual is a class-wide type; otherwise this indicates a failure
+            --  in function Has_Class_Wide_Actual.
+
+            pragma Assert (Present (Formal_Typ));
+
+            --  Step 2: Find the proper class-wide subprogram or primitive
+            --  that corresponds to the renamed generic formal subprogram.
+
+            CW_Prim_Op   := Find_Primitive (Actual_Typ);
+            CW_Prim_OK   := Is_Suitable_Candidate (CW_Prim_Op);
+            Root_Prim_Op := Find_Primitive (Root_Typ);
+            Root_Prim_OK := Is_Suitable_Candidate (Root_Prim_Op);
+
+            --  The class-wide actual type has two subprograms that correspond
+            --  to the renamed generic formal subprogram:
+
+            --    with procedure Prim_Op (Param : Formal_Typ);
+
+            --    procedure Prim_Op (Param : Actual_Typ);  --  may be inherited
+            --    procedure Prim_Op (Param : Actual_Typ'Class);
+
+            --  Even though the declaration of the two subprograms is legal, a
+            --  call to either one is ambiguous and therefore illegal.
+
+            if CW_Prim_OK and Root_Prim_OK then
+
+               --  A user-defined primitive has precedence over a predefined
+               --  one.
+
+               if Is_Internal (CW_Prim_Op)
+                 and then not Is_Internal (Root_Prim_Op)
+               then
+                  Prim_Op := Root_Prim_Op;
+
+               elsif Is_Internal (Root_Prim_Op)
+                 and then not Is_Internal (CW_Prim_Op)
+               then
+                  Prim_Op := CW_Prim_Op;
+                  Is_CW_Prim := True;
+
+               elsif CW_Prim_Op = Root_Prim_Op then
+                  Prim_Op := Root_Prim_Op;
+
+               --  The two subprograms are legal but the class-wide subprogram
+               --  is a class-wide wrapper built for a previous instantiation;
+               --  the wrapper has precedence.
+
+               elsif Present (Alias (CW_Prim_Op))
+                 and then Is_Class_Wide_Wrapper (Ultimate_Alias (CW_Prim_Op))
+               then
+                  Prim_Op := CW_Prim_Op;
+                  Is_CW_Prim := True;
+
+               --  Otherwise both candidate subprograms are user-defined and
+               --  ambiguous.
+
+               else
+                  Error_Msg_NE
+                    ("ambiguous actual for generic subprogram &",
+                     Spec, Formal_Spec);
+                  Interpretation_Error (Root_Prim_Op);
+                  Interpretation_Error (CW_Prim_Op);
+                  return;
+               end if;
+
+            elsif CW_Prim_OK and not Root_Prim_OK then
+               Prim_Op := CW_Prim_Op;
+               Is_CW_Prim := True;
+
+            elsif not CW_Prim_OK and Root_Prim_OK then
+               Prim_Op := Root_Prim_Op;
+
+            --  An intrinsic equality may act as a suitable candidate in the
+            --  case of a null type extension where the parent's equality
+            --  is hidden. A call to an intrinsic equality is expanded as
+            --  dispatching.
+
+            elsif Present (Root_Prim_Op)
+              and then Is_Intrinsic_Equality (Root_Prim_Op)
+            then
+               Prim_Op := Root_Prim_Op;
+
+            --  Otherwise there are no candidate subprograms. Let the caller
+            --  diagnose the error.
+
+            else
+               return;
+            end if;
+
+            --  At this point resolution has taken place and the name is no
+            --  longer overloaded. Mark the primitive as referenced.
+
+            Set_Is_Overloaded (Name (N), False);
+            Set_Referenced    (Prim_Op);
+         end Find_Suitable_Candidate;
+
+         --  Local variables
+
+         Is_CW_Prim : Boolean;
+
+      --  Start of processing for Handle_Instance_With_Class_Wide_Type
+
+      begin
+         Wrapped_Prim := Empty;
+         Wrap_Id := Empty;
+
+         --  Ada 2012 (AI05-0071): A generic/instance scenario involving a
+         --  formal type with unknown discriminants and a generic primitive
+         --  operation of the said type with a box require special processing
+         --  when the actual is a class-wide type:
+         --
+         --    generic
+         --       type Formal_Typ (<>) is private;
+         --       with procedure Prim_Op (Param : Formal_Typ) is <>;
+         --    package Gen is ...
+         --
+         --    package Inst is new Gen (Actual_Typ'Class);
+         --
+         --  In this case the general renaming mechanism used in the prologue
+         --  of an instance no longer applies:
+         --
+         --    procedure Prim_Op (Param : Formal_Typ) renames Prim_Op;
+         --
+         --  The above is replaced the following wrapper/renaming combination:
+         --
+         --    procedure Wrapper (Param : Formal_Typ) is  --  wrapper
+         --    begin
+         --       Prim_Op (Param);                        --  primitive
+         --    end Wrapper;
+         --
+         --    procedure Prim_Op (Param : Formal_Typ) renames Wrapper;
+         --
+         --  This transformation applies only if there is no explicit visible
+         --  class-wide operation at the point of the instantiation. Ren_Id is
+         --  the entity of the renaming declaration. When the transformation
+         --  applies, Wrapped_Prim is the entity of the wrapped primitive.
+
+         if Box_Present (Inst_Node) then
+            Find_Suitable_Candidate
+              (Prim_Op    => Wrapped_Prim,
+               Is_CW_Prim => Is_CW_Prim);
+
+            if Present (Wrapped_Prim) then
+               if not Is_CW_Prim then
+                  Build_Class_Wide_Wrapper (Ren_Id, Wrapped_Prim, Wrap_Id);
+
+               --  Small optimization: When the candidate is a class-wide
+               --  subprogram we don't build the wrapper; we modify the
+               --  renaming declaration to directly map the actual to the
+               --  generic formal and discard the candidate.
+
+               else
+                  Rewrite (Nam, New_Occurrence_Of (Wrapped_Prim, Sloc (N)));
+                  Wrapped_Prim := Empty;
+               end if;
+            end if;
+
+         --  Ada 2022 (AI12-0165, RM 12.6(8.5/3)): The actual subprogram for a
+         --  formal_abstract_subprogram_declaration shall be:
+         --   a) a dispatching operation of the controlling type; or
+         --   b) if the controlling type is a formal type, and the actual
+         --      type corresponding to that formal type is a specific type T,
+         --      a dispatching operation of type T; or
+         --   c) if the controlling type is a formal type, and the actual
+         --      type is a class-wide type T'Class, an implicitly declared
+         --      subprogram corresponding to a primitive operation of type T.
+
+         elsif Nkind (Inst_Node) = N_Formal_Abstract_Subprogram_Declaration
+           and then Is_Entity_Name (Nam)
+         then
+            Find_Suitable_Candidate
+              (Prim_Op    => Wrapped_Prim,
+               Is_CW_Prim => Is_CW_Prim);
+
+            if Present (Wrapped_Prim) then
+
+               --  Cases (a) and (b); see previous description.
+
+               if not Is_CW_Prim then
+                  Build_Class_Wide_Wrapper (Ren_Id, Wrapped_Prim, Wrap_Id);
+
+               --  Case (c); see previous description.
+
+               --  Implicit operations of T'Class for subtype declarations
+               --  are built by Derive_Subprogram, and their Alias attribute
+               --  references the primitive operation of T.
+
+               elsif not Comes_From_Source (Wrapped_Prim)
+                 and then Nkind (Parent (Wrapped_Prim)) = N_Subtype_Declaration
+                 and then Present (Alias (Wrapped_Prim))
+               then
+                  --  We don't need to build the wrapper; we modify the
+                  --  renaming declaration to directly map the actual to
+                  --  the generic formal and discard the candidate.
+
+                  Rewrite (Nam,
+                    New_Occurrence_Of (Alias (Wrapped_Prim), Sloc (N)));
+                  Wrapped_Prim := Empty;
+
+               --  Legality rules do not apply; discard the candidate.
+
+               else
+                  Wrapped_Prim := Empty;
+               end if;
+            end if;
+         end if;
+      end Handle_Instance_With_Class_Wide_Type;
+
       -------------------------
       -- Original_Subprogram --
       -------------------------
@@ -2965,12 +3094,13 @@ package body Sem_Ch8 is
       --  Local variables
 
       CW_Actual : constant Boolean := Has_Class_Wide_Actual;
-      --  Ada 2012 (AI05-071, AI05-0131): True if the renaming is for a
-      --  defaulted formal subprogram when the actual for a related formal
-      --  type is class-wide.
+      --  Ada 2012 (AI05-071, AI05-0131) and Ada 2022 (AI12-0165): True if the
+      --  renaming is for a defaulted formal subprogram when the actual for a
+      --  related formal type is class-wide.
 
-      Inst_Node : Node_Id := Empty;
-      New_S     : Entity_Id;
+      Inst_Node    : Node_Id   := Empty;
+      New_S        : Entity_Id := Empty;
+      Wrapped_Prim : Entity_Id := Empty;
 
    --  Start of processing for Analyze_Subprogram_Renaming
 
@@ -3101,11 +3231,64 @@ package body Sem_Ch8 is
       if Is_Actual then
          Inst_Node := Unit_Declaration_Node (Formal_Spec);
 
-         --  Check whether the renaming is for a defaulted actual subprogram
-         --  with a class-wide actual.
+         --  Ada 2012 (AI05-0071) and Ada 2022 (AI12-0165): when the actual
+         --  type is a class-wide type T'Class we may need to wrap a primitive
+         --  operation of T. Search for the wrapped primitive and (if required)
+         --  build a wrapper whose body consists of a dispatching call to the
+         --  wrapped primitive of T, with its formal parameters as the actual
+         --  parameters.
 
-         if CW_Actual and then Box_Present (Inst_Node) then
-            Build_Class_Wide_Wrapper (New_S, Old_S);
+         if CW_Actual and then
+
+         --  Ada 2012 (AI05-0071): Check whether the renaming is for a
+         --  defaulted actual subprogram with a class-wide actual.
+
+            (Box_Present (Inst_Node)
+
+                or else
+
+         --  Ada 2022 (AI12-0165): Check whether the renaming is for a formal
+         --  abstract subprogram declaration with a class-wide actual.
+
+             (Nkind (Inst_Node) = N_Formal_Abstract_Subprogram_Declaration
+               and then Is_Entity_Name (Nam)))
+         then
+            New_S := Analyze_Subprogram_Specification (Spec);
+
+            --  Do not attempt to build the wrapper if the renaming is in error
+
+            if not Error_Posted (Nam) then
+               Handle_Instance_With_Class_Wide_Type
+                 (Inst_Node    => Inst_Node,
+                  Ren_Id       => New_S,
+                  Wrapped_Prim => Wrapped_Prim,
+                  Wrap_Id      => Old_S);
+
+               --  If several candidates were found, then we reported the
+               --  ambiguity; stop processing the renaming declaration to
+               --  avoid reporting further (spurious) errors.
+
+               if Error_Posted (Spec) then
+                  return;
+               end if;
+
+            end if;
+         end if;
+
+         if Present (Wrapped_Prim) then
+
+            --  When the wrapper is built, the subprogram renaming aliases
+            --  the wrapper.
+
+            Analyze (Nam);
+
+            pragma Assert (Old_S = Entity (Nam)
+              and then Is_Class_Wide_Wrapper (Old_S));
+
+            --  The subprogram renaming declaration may become Ghost if it
+            --  renames a wrapper of a Ghost entity.
+
+            Mark_Ghost_Renaming (N, Wrapped_Prim);
 
          elsif Is_Entity_Name (Nam)
            and then Present (Entity (Nam))
@@ -3685,7 +3868,15 @@ package body Sem_Ch8 is
             --  indicate that the renaming is an abstract dispatching operation
             --  with a controlling type.
 
-            if Is_Actual and then Is_Abstract_Subprogram (Formal_Spec) then
+            --  Skip this decoration when the renaming corresponds to an
+            --  association with class-wide wrapper (see above) because such
+            --  wrapper is neither abstract nor a dispatching operation (its
+            --  body has the dispatching call to the wrapped primitive).
+
+            if Is_Actual
+              and then Is_Abstract_Subprogram (Formal_Spec)
+              and then No (Wrapped_Prim)
+            then
 
                --  Mark the renaming as abstract here, so Find_Dispatching_Type
                --  see it as corresponding to a generic association for a
@@ -5008,9 +5199,9 @@ package body Sem_Ch8 is
                Next_Entity (Id);
             end loop;
 
-            if Present (Renamed_Object (Pack)) then
-               Set_In_Use (Renamed_Object (Pack), False);
-               Set_Current_Use_Clause (Renamed_Object (Pack), Empty);
+            if Present (Renamed_Entity (Pack)) then
+               Set_In_Use (Renamed_Entity (Pack), False);
+               Set_Current_Use_Clause (Renamed_Entity (Pack), Empty);
             end if;
 
             if Chars (Pack) = Name_System
@@ -5224,8 +5415,8 @@ package body Sem_Ch8 is
          ------------------------
 
          function Declared_In_Actual (Pack : Entity_Id) return Boolean is
+            pragma Assert (Ekind (Pack) = E_Package);
             Act : Entity_Id;
-
          begin
             if No (Associated_Formal_Package (Pack)) then
                return False;
@@ -5233,13 +5424,13 @@ package body Sem_Ch8 is
             else
                Act := First_Entity (Pack);
                while Present (Act) loop
-                  if Renamed_Object (Pack) = Scop then
+                  if Renamed_Entity (Pack) = Scop then
                      return True;
 
                   --  Check for end of list of actuals
 
                   elsif Ekind (Act) = E_Package
-                    and then Renamed_Object (Act) = Pack
+                    and then Renamed_Entity (Act) = Pack
                   then
                      return False;
 
@@ -6056,9 +6247,9 @@ package body Sem_Ch8 is
 
                if Scope (E) = Scope (E2)
                  and then Ekind (E) = E_Package
-                 and then Present (Renamed_Object (E))
-                 and then Is_Generic_Instance (Renamed_Object (E))
-                 and then In_Open_Scopes (Renamed_Object (E))
+                 and then Present (Renamed_Entity (E))
+                 and then Is_Generic_Instance (Renamed_Entity (E))
+                 and then In_Open_Scopes (Renamed_Entity (E))
                  and then Comes_From_Source (N)
                then
                   Set_Is_Immediately_Visible (E, False);
@@ -6389,9 +6580,9 @@ package body Sem_Ch8 is
       --  original package.
 
       if Ekind (P_Name) = E_Package
-        and then Present (Renamed_Object (P_Name))
+        and then Present (Renamed_Entity (P_Name))
       then
-         P_Name := Renamed_Object (P_Name);
+         P_Name := Renamed_Entity (P_Name);
 
          if From_Limited_With (P_Name)
            and then not Unit_Is_Visible (Cunit (Get_Source_Unit (P_Name)))
@@ -7196,9 +7387,9 @@ package body Sem_Ch8 is
                Scop := Entity (Prefix (Nam));
 
                if Ekind (Scop) = E_Package
-                 and then Present (Renamed_Object (Scop))
+                 and then Present (Renamed_Entity (Scop))
                then
-                  Scop := Renamed_Object (Scop);
+                  Scop := Renamed_Entity (Scop);
                end if;
 
                --  Operator is visible if prefix of expanded name denotes
@@ -9045,7 +9236,7 @@ package body Sem_Ch8 is
       if Ekind (S) = E_Void then
          null;
 
-      --  Set scope depth if not a non-concurrent type, and we have not yet set
+      --  Set scope depth if not a nonconcurrent type, and we have not yet set
       --  the scope depth. This means that we have the first occurrence of the
       --  scope, and this is where the depth is set.
 
@@ -9882,8 +10073,8 @@ package body Sem_Ch8 is
          --  When a renaming exists we must check it for redundancy. The
          --  original package would have already been seen at this point.
 
-         if Present (Renamed_Object (Entity (Pack_Name))) then
-            P := Renamed_Object (Entity (Pack_Name));
+         if Present (Renamed_Entity (Entity (Pack_Name))) then
+            P := Renamed_Entity (Entity (Pack_Name));
          else
             P := Entity (Pack_Name);
          end if;
@@ -9949,10 +10140,10 @@ package body Sem_Ch8 is
       --  also in use (the flags on both entities must remain consistent, and a
       --  subsequent use of either of them should be recognized as redundant).
 
-      if Present (Renamed_Object (P)) then
-         Set_In_Use (Renamed_Object (P));
-         Set_Current_Use_Clause (Renamed_Object (P), N);
-         Real_P := Renamed_Object (P);
+      if Present (Renamed_Entity (P)) then
+         Set_In_Use (Renamed_Entity (P));
+         Set_Current_Use_Clause (Renamed_Entity (P), N);
+         Real_P := Renamed_Entity (P);
       else
          Real_P := P;
       end if;
diff --git a/gcc/ada/sem_elab.adb b/gcc/ada/sem_elab.adb
index 1fbe037..c906708 100644
--- a/gcc/ada/sem_elab.adb
+++ b/gcc/ada/sem_elab.adb
@@ -3815,14 +3815,14 @@ package body Sem_Elab is
       -----------------------
 
       function Ultimate_Variable (Var_Id : Entity_Id) return Entity_Id is
+         pragma Assert (Ekind (Var_Id) = E_Variable);
          Ren_Id : Entity_Id;
-
       begin
          Ren_Id := Var_Id;
-         while Present (Renamed_Entity (Ren_Id))
-           and then Nkind (Renamed_Entity (Ren_Id)) in N_Entity
+         while Present (Renamed_Object (Ren_Id))
+           and then Nkind (Renamed_Object (Ren_Id)) in N_Entity
          loop
-            Ren_Id := Renamed_Entity (Ren_Id);
+            Ren_Id := Renamed_Object (Ren_Id);
          end loop;
 
          return Ren_Id;
diff --git a/gcc/ada/sem_prag.adb b/gcc/ada/sem_prag.adb
index 1e6397f..c3ea16d 100644
--- a/gcc/ada/sem_prag.adb
+++ b/gcc/ada/sem_prag.adb
@@ -84,6 +84,7 @@ with Sinfo.Utils;    use Sinfo.Utils;
 with Sinfo.CN;       use Sinfo.CN;
 with Sinput;         use Sinput;
 with Stringt;        use Stringt;
+with Strub;          use Strub;
 with Stylesw;        use Stylesw;
 with Table;
 with Targparm;       use Targparm;
@@ -1684,7 +1685,15 @@ package body Sem_Prag is
                --  States and global objects are not used properly only when
                --  the subprogram is subject to pragma Global.
 
-               elsif Global_Seen then
+               elsif Global_Seen
+                 and then Ekind (Item_Id) in E_Abstract_State
+                                           | E_Constant
+                                           | E_Loop_Parameter
+                                           | E_Protected_Type
+                                           | E_Task_Type
+                                           | E_Variable
+                                           | Formal_Kind
+               then
                   Usage_Error (Item_Id);
                end if;
             end if;
@@ -10435,6 +10444,49 @@ package body Sem_Prag is
          Expr  : Node_Id;
          Val   : Uint;
 
+         procedure Process_No_Specification_of_Aspect;
+         --  Process the No_Specification_of_Aspect restriction
+
+         procedure Process_No_Use_Of_Attribute;
+         --  Process the No_Use_Of_Attribute restriction
+
+         ----------------------------------------
+         -- Process_No_Specification_of_Aspect --
+         ----------------------------------------
+
+         procedure Process_No_Specification_of_Aspect is
+            Name : constant Name_Id := Chars (Expr);
+         begin
+            if Nkind (Expr) = N_Identifier
+               and then Is_Aspect_Id (Name)
+            then
+               Set_Restriction_No_Specification_Of_Aspect (Expr, Warn);
+            else
+               Bad_Aspect (Expr, Name, Warn => True);
+
+               raise Pragma_Exit;
+            end if;
+         end Process_No_Specification_of_Aspect;
+
+         ---------------------------------
+         -- Process_No_Use_Of_Attribute --
+         ---------------------------------
+
+         procedure Process_No_Use_Of_Attribute is
+            Name : constant Name_Id := Chars (Expr);
+         begin
+            if Nkind (Expr) = N_Identifier
+               and then Is_Attribute_Name (Name)
+            then
+               Set_Restriction_No_Use_Of_Attribute (Expr, Warn);
+            else
+               Bad_Attribute (Expr, Name, Warn => True);
+            end if;
+
+         end Process_No_Use_Of_Attribute;
+
+      --  Start of processing for Process_Restrictions_Or_Restriction_Warnings
+
       begin
          --  Ignore all Restrictions pragmas in CodePeer mode
 
@@ -10659,34 +10711,12 @@ package body Sem_Prag is
             --  Case of No_Specification_Of_Aspect => aspect-identifier
 
             elsif Id = Name_No_Specification_Of_Aspect then
-               declare
-                  A_Id : Aspect_Id;
-
-               begin
-                  if Nkind (Expr) /= N_Identifier then
-                     A_Id := No_Aspect;
-                  else
-                     A_Id := Get_Aspect_Id (Chars (Expr));
-                  end if;
-
-                  if A_Id = No_Aspect then
-                     Error_Pragma_Arg ("invalid restriction name", Arg);
-                  else
-                     Set_Restriction_No_Specification_Of_Aspect (Expr, Warn);
-                  end if;
-               end;
+               Process_No_Specification_of_Aspect;
 
             --  Case of No_Use_Of_Attribute => attribute-identifier
 
             elsif Id = Name_No_Use_Of_Attribute then
-               if Nkind (Expr) /= N_Identifier
-                 or else not Is_Attribute_Name (Chars (Expr))
-               then
-                  Error_Msg_N ("unknown attribute name??", Expr);
-
-               else
-                  Set_Restriction_No_Use_Of_Attribute (Expr, Warn);
-               end if;
+               Process_No_Use_Of_Attribute;
 
             --  Case of No_Use_Of_Entity => fully-qualified-name
 
@@ -11479,7 +11509,7 @@ package body Sem_Prag is
 
       Check_Restriction_No_Use_Of_Pragma (N);
 
-      if Get_Aspect_Id (Chars (Pragma_Identifier (N))) /= No_Aspect then
+      if Is_Aspect_Id (Chars (Pragma_Identifier (N))) then
          --  6.1/3 No_Specification_of_Aspect: Identifies an aspect for which
          --    no aspect_specification, attribute_definition_clause, or pragma
          --    is given.
@@ -19438,7 +19468,29 @@ package body Sem_Prag is
             Check_Arg_Is_OK_Static_Expression (Arg2, Standard_String);
             Def_Id := Entity (Get_Pragma_Arg (Arg1));
 
-            if Is_Access_Type (Def_Id) then
+            --  Apply the pragma to the designated type, rather than to the
+            --  access type, unless it's a strub annotation.  We wish to enable
+            --  objects of access type, as well as access types themselves, to
+            --  be annotated, so that reading the access objects (as oposed to
+            --  the designated data) automatically enables stack
+            --  scrubbing. That said, as in the attribute handler that
+            --  processes the pragma turned into a compiler attribute, a strub
+            --  annotation that must be associated with a subprogram type (for
+            --  holding an explicit strub mode), when applied to an
+            --  access-to-subprogram, gets promoted to the subprogram type. We
+            --  might be tempted to leave it alone here, since the C attribute
+            --  handler will adjust it, but then GNAT would convert the
+            --  annotated subprogram types to naked ones before using them,
+            --  cancelling out their intended effects.
+
+            if Is_Access_Type (Def_Id)
+              and then (not Strub_Pragma_P (N)
+                          or else
+                          (Present (Arg3)
+                             and then
+                             Ekind (Designated_Type
+                                      (Def_Id)) = E_Subprogram_Type))
+            then
                Def_Id := Designated_Type (Def_Id);
             end if;
 
@@ -19456,7 +19508,7 @@ package body Sem_Prag is
             if Rep_Item_Too_Late (Def_Id, N) then
                return;
             else
-               Set_Has_Gigi_Rep_Item (Entity (Get_Pragma_Arg (Arg1)));
+               Set_Has_Gigi_Rep_Item (Def_Id);
             end if;
          end Machine_Attribute;
 
@@ -31976,7 +32028,7 @@ package body Sem_Prag is
          Error_Msg_N ("condition is not known at compile time", Arg1x);
 
       elsif Warn_On_Unknown_Compile_Time_Warning then
-         Error_Msg_N ("??condition is not known at compile time", Arg1x);
+         Error_Msg_N ("?_c?condition is not known at compile time", Arg1x);
       end if;
    end Validate_Compile_Time_Warning_Or_Error;
 
diff --git a/gcc/ada/sem_res.adb b/gcc/ada/sem_res.adb
index 0bdc463..ac262fa 100644
--- a/gcc/ada/sem_res.adb
+++ b/gcc/ada/sem_res.adb
@@ -82,6 +82,7 @@ with Sinfo.CN;       use Sinfo.CN;
 with Snames;         use Snames;
 with Stand;          use Stand;
 with Stringt;        use Stringt;
+with Strub;          use Strub;
 with Style;          use Style;
 with Targparm;       use Targparm;
 with Tbuild;         use Tbuild;
@@ -103,6 +104,14 @@ package body Sem_Res is
 
    --  Note that Resolve_Attribute is separated off in Sem_Attr
 
+   function Has_Applicable_User_Defined_Literal
+     (N   : Node_Id;
+      Typ : Entity_Id) return Boolean;
+   --  If N is a literal or a named number, check whether Typ
+   --  has a user-defined literal aspect that can apply to N.
+   --  If present, replace N with a call to the corresponding
+   --  function and return True.
+
    procedure Check_Discriminant_Use (N : Node_Id);
    --  Enforce the restrictions on the use of discriminants when constraining
    --  a component of a discriminated type (record or concurrent type).
@@ -286,6 +295,15 @@ package body Sem_Res is
    --  is only one requires a search over all visible entities, and happens
    --  only in very pathological cases (see 6115-006).
 
+   function Try_User_Defined_Literal
+     (N   : Node_Id;
+      Typ : Entity_Id) return Boolean;
+   --  If an operator node has a literal operand, check whether the type
+   --  of the context, or the type of the other operand has a user-defined
+   --  literal aspect that can be applied to the literal to resolve the node.
+   --  If such aspect exists, replace literal with a call to the
+   --  corresponing function and return True, return false otherwise.
+
    -------------------------
    -- Ambiguous_Character --
    -------------------------
@@ -409,6 +427,165 @@ package body Sem_Res is
       end if;
    end Analyze_And_Resolve;
 
+   -------------------------------------
+   -- Has_Applicable_User_Defined_Literal --
+   -------------------------------------
+
+   function Has_Applicable_User_Defined_Literal
+     (N   : Node_Id;
+      Typ : Entity_Id) return Boolean
+   is
+      Loc  : constant Source_Ptr := Sloc (N);
+      Literal_Aspect_Map :
+        constant array (N_Numeric_Or_String_Literal) of Aspect_Id :=
+          (N_Integer_Literal => Aspect_Integer_Literal,
+           N_Real_Literal    => Aspect_Real_Literal,
+           N_String_Literal  => Aspect_String_Literal);
+
+      Named_Number_Aspect_Map : constant array (Named_Kind) of Aspect_Id :=
+        (E_Named_Integer => Aspect_Integer_Literal,
+         E_Named_Real    => Aspect_Real_Literal);
+
+      Lit_Aspect : Aspect_Id;
+
+      Callee : Entity_Id;
+      Name   : Node_Id;
+      Param1 : Node_Id;
+      Param2 : Node_Id;
+      Params : List_Id;
+      Call   : Node_Id;
+      Expr   : Node_Id;
+
+   begin
+      if (Nkind (N) in N_Numeric_Or_String_Literal
+           and then Present
+            (Find_Aspect (Typ, Literal_Aspect_Map (Nkind (N)))))
+        or else
+          (Nkind (N) = N_Identifier
+            and then Is_Named_Number (Entity (N))
+            and then
+              Present
+                (Find_Aspect
+                  (Typ, Named_Number_Aspect_Map (Ekind (Entity (N))))))
+      then
+         Lit_Aspect :=
+           (if Nkind (N) = N_Identifier
+            then Named_Number_Aspect_Map (Ekind (Entity (N)))
+            else Literal_Aspect_Map (Nkind (N)));
+         Callee :=
+           Entity (Expression (Find_Aspect (Typ, Lit_Aspect)));
+         Name := Make_Identifier (Loc, Chars (Callee));
+
+         if Is_Derived_Type (Typ)
+           and then Is_Tagged_Type (Typ)
+           and then Base_Type (Etype (Callee)) /= Base_Type (Typ)
+         then
+            Callee :=
+              Corresponding_Primitive_Op
+                (Ancestor_Op     => Callee,
+                 Descendant_Type => Base_Type (Typ));
+         end if;
+
+         --  Handle an identifier that denotes a named number.
+
+         if Nkind (N) = N_Identifier then
+            Expr := Expression (Declaration_Node (Entity (N)));
+
+            if Ekind (Entity (N)) = E_Named_Integer then
+               UI_Image (Expr_Value (Expr), Decimal);
+               Start_String;
+               Store_String_Chars
+                 (UI_Image_Buffer (1 .. UI_Image_Length));
+               Param1 := Make_String_Literal (Loc, End_String);
+               Params := New_List (Param1);
+
+            else
+               UI_Image (Norm_Num (Expr_Value_R (Expr)), Decimal);
+               Start_String;
+
+               if UR_Is_Negative (Expr_Value_R (Expr)) then
+                  Store_String_Chars ("-");
+               end if;
+
+               Store_String_Chars
+                 (UI_Image_Buffer (1 .. UI_Image_Length));
+               Param1 := Make_String_Literal (Loc, End_String);
+
+               --  Note: Set_Etype is called below on Param1
+
+               UI_Image (Norm_Den (Expr_Value_R (Expr)), Decimal);
+               Start_String;
+               Store_String_Chars
+                 (UI_Image_Buffer (1 .. UI_Image_Length));
+               Param2 := Make_String_Literal (Loc, End_String);
+               Set_Etype (Param2, Standard_String);
+
+               Params := New_List (Param1, Param2);
+
+               if Present (Related_Expression (Callee)) then
+                  Callee := Related_Expression (Callee);
+               else
+                  Error_Msg_NE
+                    ("cannot resolve & for a named real", N, Callee);
+                  return False;
+               end if;
+            end if;
+
+         elsif Nkind (N) = N_String_Literal then
+            Param1 := Make_String_Literal (Loc, Strval (N));
+            Params := New_List (Param1);
+
+         else
+            Param1 :=
+              Make_String_Literal
+                (Loc, String_From_Numeric_Literal (N));
+            Params := New_List (Param1);
+         end if;
+
+         Call :=
+           Make_Function_Call
+             (Sloc                   => Loc,
+              Name                   => Name,
+              Parameter_Associations => Params);
+
+         Set_Entity (Name, Callee);
+         Set_Is_Overloaded (Name, False);
+
+         if Lit_Aspect = Aspect_String_Literal then
+            Set_Etype (Param1, Standard_Wide_Wide_String);
+         else
+            Set_Etype (Param1, Standard_String);
+         end if;
+
+         Set_Etype (Call, Etype (Callee));
+
+         if Base_Type (Etype (Call)) /= Base_Type (Typ) then
+            --  Conversion may be needed in case of an inherited
+            --  aspect of a derived type. For a null extension, we
+            --  use a null extension aggregate instead because the
+            --  downward type conversion would be illegal.
+
+            if Is_Null_Extension_Of
+                 (Descendant => Typ,
+                  Ancestor   => Etype (Call))
+            then
+               Call := Make_Extension_Aggregate (Loc,
+                         Ancestor_Part       => Call,
+                         Null_Record_Present => True);
+            else
+               Call := Convert_To (Typ, Call);
+            end if;
+         end if;
+
+         Rewrite (N, Call);
+
+         Analyze_And_Resolve (N, Typ);
+         return True;
+      else
+         return False;
+      end if;
+   end Has_Applicable_User_Defined_Literal;
+
    ----------------------------
    -- Check_Discriminant_Use --
    ----------------------------
@@ -2156,16 +2333,6 @@ package body Sem_Res is
          return;
       end Resolution_Failed;
 
-      Literal_Aspect_Map :
-        constant array (N_Numeric_Or_String_Literal) of Aspect_Id :=
-          (N_Integer_Literal => Aspect_Integer_Literal,
-           N_Real_Literal    => Aspect_Real_Literal,
-           N_String_Literal  => Aspect_String_Literal);
-
-      Named_Number_Aspect_Map : constant array (Named_Kind) of Aspect_Id :=
-        (E_Named_Integer => Aspect_Integer_Literal,
-         E_Named_Real    => Aspect_Real_Literal);
-
    --  Start of processing for Resolve
 
    begin
@@ -2884,143 +3051,14 @@ package body Sem_Res is
                end;
             end if;
 
-            --  Rewrite Literal as a call if the corresponding literal aspect
-            --  is set.
+            --  If node is a literal and context type has a user-defined
+            --  literal aspect, rewrite node as a call to the corresponding
+            --  function, which plays the role of an implicit conversion.
 
-            if (Nkind (N) in N_Numeric_Or_String_Literal
-                 and then
-                   Present
-                     (Find_Aspect (Typ, Literal_Aspect_Map (Nkind (N)))))
-              or else
-                (Nkind (N) = N_Identifier
-                  and then Is_Named_Number (Entity (N))
-                  and then
-                    Present
-                      (Find_Aspect
-                        (Typ, Named_Number_Aspect_Map (Ekind (Entity (N))))))
+            if Nkind (N) in
+                N_Numeric_Or_String_Literal | N_Identifier
+              and then Has_Applicable_User_Defined_Literal (N, Typ)
             then
-               declare
-                  Lit_Aspect : constant Aspect_Id :=
-                    (if Nkind (N) = N_Identifier
-                     then Named_Number_Aspect_Map (Ekind (Entity (N)))
-                     else Literal_Aspect_Map (Nkind (N)));
-
-                  Loc  : constant Source_Ptr := Sloc (N);
-
-                  Callee : Entity_Id :=
-                    Entity (Expression (Find_Aspect (Typ, Lit_Aspect)));
-
-                  Name : constant Node_Id :=
-                    Make_Identifier (Loc, Chars (Callee));
-
-                  Param1 : Node_Id;
-                  Param2 : Node_Id;
-                  Params : List_Id;
-                  Call   : Node_Id;
-                  Expr   : Node_Id;
-
-               begin
-                  if Is_Derived_Type (Typ)
-                    and then Is_Tagged_Type (Typ)
-                    and then Base_Type (Etype (Callee)) /= Base_Type (Typ)
-                  then
-                     Callee :=
-                       Corresponding_Primitive_Op
-                         (Ancestor_Op     => Callee,
-                          Descendant_Type => Base_Type (Typ));
-                  end if;
-
-                  if Nkind (N) = N_Identifier then
-                     Expr := Expression (Declaration_Node (Entity (N)));
-
-                     if Ekind (Entity (N)) = E_Named_Integer then
-                        UI_Image (Expr_Value (Expr), Decimal);
-                        Start_String;
-                        Store_String_Chars
-                          (UI_Image_Buffer (1 .. UI_Image_Length));
-                        Param1 := Make_String_Literal (Loc, End_String);
-                        Params := New_List (Param1);
-
-                     else
-                        UI_Image (Norm_Num (Expr_Value_R (Expr)), Decimal);
-                        Start_String;
-
-                        if UR_Is_Negative (Expr_Value_R (Expr)) then
-                           Store_String_Chars ("-");
-                        end if;
-
-                        Store_String_Chars
-                          (UI_Image_Buffer (1 .. UI_Image_Length));
-                        Param1 := Make_String_Literal (Loc, End_String);
-
-                        --  Note: Set_Etype is called below on Param1
-
-                        UI_Image (Norm_Den (Expr_Value_R (Expr)), Decimal);
-                        Start_String;
-                        Store_String_Chars
-                          (UI_Image_Buffer (1 .. UI_Image_Length));
-                        Param2 := Make_String_Literal (Loc, End_String);
-                        Set_Etype (Param2, Standard_String);
-
-                        Params := New_List (Param1, Param2);
-
-                        if Present (Related_Expression (Callee)) then
-                           Callee := Related_Expression (Callee);
-                        else
-                           Error_Msg_NE
-                             ("cannot resolve & for a named real", N, Callee);
-                           return;
-                        end if;
-                     end if;
-
-                  elsif Nkind (N) = N_String_Literal then
-                     Param1 := Make_String_Literal (Loc, Strval (N));
-                     Params := New_List (Param1);
-                  else
-                     Param1 :=
-                       Make_String_Literal
-                         (Loc, String_From_Numeric_Literal (N));
-                     Params := New_List (Param1);
-                  end if;
-
-                  Call :=
-                    Make_Function_Call
-                      (Sloc                   => Loc,
-                       Name                   => Name,
-                       Parameter_Associations => Params);
-
-                  Set_Entity (Name, Callee);
-                  Set_Is_Overloaded (Name, False);
-
-                  if Lit_Aspect = Aspect_String_Literal then
-                     Set_Etype (Param1, Standard_Wide_Wide_String);
-                  else
-                     Set_Etype (Param1, Standard_String);
-                  end if;
-
-                  Set_Etype (Call, Etype (Callee));
-
-                  if Base_Type (Etype (Call)) /= Base_Type (Typ) then
-                     --  Conversion may be needed in case of an inherited
-                     --  aspect of a derived type. For a null extension, we
-                     --  use a null extension aggregate instead because the
-                     --  downward type conversion would be illegal.
-
-                     if Is_Null_Extension_Of
-                          (Descendant => Typ,
-                           Ancestor   => Etype (Call))
-                     then
-                        Call := Make_Extension_Aggregate (Loc,
-                                  Ancestor_Part       => Call,
-                                  Null_Record_Present => True);
-                     else
-                        Call := Convert_To (Typ, Call);
-                     end if;
-                  end if;
-
-                  Rewrite (N, Call);
-               end;
-
                Analyze_And_Resolve (N, Typ);
                return;
             end if;
@@ -3116,6 +3154,14 @@ package body Sem_Res is
                     ("missing ALL or SOME in quantified expression",
                      Defining_Identifier (First (Component_Associations (N))));
 
+               --  For an operator with no interpretation, check whether
+               --  one of its operands may be a user-defined literal.
+
+               elsif Nkind (N) in N_Op
+                 and then Try_User_Defined_Literal (N, Typ)
+               then
+                  return;
+
                else
                   Wrong_Type (N, Typ);
                end if;
@@ -3134,6 +3180,27 @@ package body Sem_Res is
       --  Only one interpretation
 
       else
+         --  Prevent implicit conversions between access-to-subprogram types
+         --  with different strub modes. Explicit conversions are acceptable in
+         --  some circumstances. We don't have to be concerned about data or
+         --  access-to-data types. Conversions between data types can safely
+         --  drop or add strub attributes from types, because strub effects are
+         --  associated with the locations rather than values. E.g., converting
+         --  a hypothetical Strub_Integer variable to Integer would load the
+         --  value from the variable, enabling stack scrabbing for the
+         --  enclosing subprogram, and then convert the value to Integer. As
+         --  for conversions between access-to-data types, that's no different
+         --  from any other case of type punning.
+
+         if Is_Access_Type (Typ)
+           and then Ekind (Designated_Type (Typ)) = E_Subprogram_Type
+           and then Is_Access_Type (Expr_Type)
+           and then Ekind (Designated_Type (Expr_Type)) = E_Subprogram_Type
+         then
+            Check_Same_Strub_Mode
+              (Designated_Type (Typ), Designated_Type (Expr_Type));
+         end if;
+
          --  In Ada 2005, if we have something like "X : T := 2 + 2;", where
          --  the "+" on T is abstract, and the operands are of universal type,
          --  the above code will have (incorrectly) resolved the "+" to the
@@ -3712,7 +3779,7 @@ package body Sem_Res is
 
                if Wrong_Order then
                   Error_Msg_N
-                    ("?P?actuals for this call may be in wrong order", N);
+                    ("?.p?actuals for this call may be in wrong order", N);
                end if;
             end;
          end;
@@ -7488,7 +7555,7 @@ package body Sem_Res is
       if Bad_Unordered_Enumeration_Reference (N, Etype (L)) then
          Error_Msg_Sloc := Sloc (Etype (L));
          Error_Msg_NE
-           ("comparison on unordered enumeration type& declared#?U?",
+           ("comparison on unordered enumeration type& declared#?.u?",
             N, Etype (L));
       end if;
 
@@ -10647,7 +10714,7 @@ package body Sem_Res is
       then
          Error_Msg_Sloc := Sloc (Typ);
          Error_Msg_NE
-           ("subrange of unordered enumeration type& declared#?U?", N, Typ);
+           ("subrange of unordered enumeration type& declared#?.u?", N, Typ);
       end if;
 
       Check_Unset_Reference (L);
@@ -11163,7 +11230,7 @@ package body Sem_Res is
                      --  of the First_Node call here.
 
                      Error_Msg_F
-                       ("?A?assertion would fail at run time!",
+                       ("?.a?assertion would fail at run time!",
                         Expression
                           (First (Pragma_Argument_Associations (Orig))));
                   end if;
@@ -11194,7 +11261,7 @@ package body Sem_Res is
                      --  comment above for an explanation of why we do this.
 
                      Error_Msg_F
-                       ("?A?check would fail at run time!",
+                       ("?.a?check would fail at run time!",
                         Expression
                           (Last (Pragma_Argument_Associations (Orig))));
                   end if;
@@ -12185,7 +12252,7 @@ package body Sem_Res is
         and then Expr_Value (R) > Uint_1
       then
          Error_Msg_N
-           ("?M?negative literal of modular type is in fact positive", N);
+           ("?.m?negative literal of modular type is in fact positive", N);
          Error_Msg_Uint_1 := (-Expr_Value (R)) mod Modulus (B_Typ);
          Error_Msg_Uint_2 := Expr_Value (R);
          Error_Msg_N ("\do you really mean^ when writing -^ '?", N);
@@ -12847,6 +12914,76 @@ package body Sem_Res is
       end if;
    end Simplify_Type_Conversion;
 
+   ------------------------------
+   -- Try_User_Defined_Literal --
+   ------------------------------
+
+   function Try_User_Defined_Literal
+     (N   : Node_Id;
+      Typ : Entity_Id) return Boolean
+   is
+   begin
+      if Nkind (N) in N_Op_Add | N_Op_Divide | N_Op_Mod | N_Op_Multiply
+        | N_Op_Rem | N_Op_Subtract
+      then
+
+         --  Both operands must have the same type as the context.
+         --  (ignoring for now fixed-point and exponentiation ops).
+
+         if Has_Applicable_User_Defined_Literal (Right_Opnd (N), Typ) then
+            Resolve (Left_Opnd (N), Typ);
+            Analyze_And_Resolve (N, Typ);
+            return True;
+         end if;
+
+         if
+           Has_Applicable_User_Defined_Literal (Left_Opnd (N), Typ)
+         then
+            Resolve (Right_Opnd (N), Typ);
+            Analyze_And_Resolve (N, Typ);
+            return True;
+
+         else
+            return False;
+         end if;
+
+      elsif Nkind (N) in N_Binary_Op then
+         --  For other operators the context does not impose a type on
+         --  the operands, but their types must match.
+
+         if (Nkind (Left_Opnd (N))
+           not in N_Integer_Literal | N_String_Literal | N_Real_Literal)
+         and then
+           Has_Applicable_User_Defined_Literal
+             (Right_Opnd (N), Etype (Left_Opnd (N)))
+         then
+            Analyze_And_Resolve (N, Typ);
+            return True;
+
+         elsif (Nkind (Right_Opnd (N))
+           not in N_Integer_Literal | N_String_Literal | N_Real_Literal)
+         and then
+           Has_Applicable_User_Defined_Literal
+             (Left_Opnd (N), Etype (Right_Opnd (N)))
+         then
+            Analyze_And_Resolve (N, Typ);
+            return True;
+         else
+            return False;
+         end if;
+
+      elsif Nkind (N) in N_Unary_Op
+        and then
+          Has_Applicable_User_Defined_Literal (Right_Opnd (N), Typ)
+      then
+         Analyze_And_Resolve (N, Typ);
+         return True;
+
+      else   --  Other operators
+         return False;
+      end if;
+   end Try_User_Defined_Literal;
+
    -----------------------------
    -- Unique_Fixed_Point_Type --
    -----------------------------
@@ -14039,7 +14176,15 @@ package body Sem_Res is
             end;
          end if;
 
-         return True;
+         --  Check that the strub modes are compatible.
+         --  We wish to reject explicit conversions only for
+         --  incompatible modes.
+
+         return Conversion_Check
+                  (Compatible_Strub_Modes
+                     (Designated_Type (Target_Type),
+                      Designated_Type (Opnd_Type)),
+                   "incompatible `strub` modes");
 
       --  Remote access to subprogram types
 
@@ -14065,7 +14210,16 @@ package body Sem_Res is
               Designated_Type (Corresponding_Remote_Type (Opnd_Type)),
             Err_Loc =>
               N);
-         return True;
+
+         --  Check that the strub modes are compatible.
+         --  We wish to reject explicit conversions only for
+         --  incompatible modes.
+
+         return Conversion_Check
+                  (Compatible_Strub_Modes
+                     (Designated_Type (Target_Type),
+                      Designated_Type (Opnd_Type)),
+                   "incompatible `strub` modes");
 
       --  If it was legal in the generic, it's legal in the instance
 
diff --git a/gcc/ada/sem_type.adb b/gcc/ada/sem_type.adb
index 8e5b067..4419fb3 100644
--- a/gcc/ada/sem_type.adb
+++ b/gcc/ada/sem_type.adb
@@ -2438,8 +2438,9 @@ package body Sem_Type is
    -------------------------
 
    function Has_Compatible_Type
-     (N   : Node_Id;
-      Typ : Entity_Id) return Boolean
+     (N              : Node_Id;
+      Typ            : Entity_Id;
+      For_Comparison : Boolean := False) return Boolean
    is
       I  : Interp_Index;
       It : Interp;
@@ -2449,11 +2450,8 @@ package body Sem_Type is
          return False;
       end if;
 
-      if Nkind (N) = N_Subtype_Indication
-        or else not Is_Overloaded (N)
-      then
-         return
-           Covers (Typ, Etype (N))
+      if Nkind (N) = N_Subtype_Indication or else not Is_Overloaded (N) then
+         if Covers (Typ, Etype (N))
 
             --  Ada 2005 (AI-345): The context may be a synchronized interface.
             --  If the type is already frozen use the corresponding_record
@@ -2472,11 +2470,6 @@ package body Sem_Type is
                and then Covers (Corresponding_Record_Type (Typ), Etype (N)))
 
            or else
-             (not Is_Tagged_Type (Typ)
-               and then Ekind (Typ) /= E_Anonymous_Access_Type
-               and then Covers (Etype (N), Typ))
-
-           or else
              (Nkind (N) = N_Integer_Literal
                and then Present (Find_Aspect (Typ, Aspect_Integer_Literal)))
 
@@ -2486,7 +2479,16 @@ package body Sem_Type is
 
            or else
              (Nkind (N) = N_String_Literal
-               and then Present (Find_Aspect (Typ, Aspect_String_Literal)));
+               and then Present (Find_Aspect (Typ, Aspect_String_Literal)))
+
+           or else
+             (For_Comparison
+               and then not Is_Tagged_Type (Typ)
+               and then Ekind (Typ) /= E_Anonymous_Access_Type
+               and then Covers (Etype (N), Typ))
+         then
+            return True;
+         end if;
 
       --  Overloaded case
 
@@ -2501,24 +2503,27 @@ package body Sem_Type is
                --  Ada 2005 (AI-345)
 
               or else
-                (Is_Concurrent_Type (It.Typ)
+                (Is_Record_Type (Typ)
+                  and then Is_Concurrent_Type (It.Typ)
                   and then Present (Corresponding_Record_Type
                                                              (Etype (It.Typ)))
                   and then Covers (Typ, Corresponding_Record_Type
                                                              (Etype (It.Typ))))
 
-              or else (not Is_Tagged_Type (Typ)
-                         and then Ekind (Typ) /= E_Anonymous_Access_Type
-                         and then Covers (It.Typ, Typ))
+             or else
+               (For_Comparison
+                 and then not Is_Tagged_Type (Typ)
+                 and then Ekind (Typ) /= E_Anonymous_Access_Type
+                 and then Covers (It.Typ, Typ))
             then
                return True;
             end if;
 
             Get_Next_Interp (I, It);
          end loop;
-
-         return False;
       end if;
+
+      return False;
    end Has_Compatible_Type;
 
    ---------------------
diff --git a/gcc/ada/sem_type.ads b/gcc/ada/sem_type.ads
index 018c283..dfe4c7c 100644
--- a/gcc/ada/sem_type.ads
+++ b/gcc/ada/sem_type.ads
@@ -186,11 +186,17 @@ package Sem_Type is
    --  right operand, which has one interpretation compatible with that of L.
    --  Return the type intersection of the two.
 
-   function Has_Compatible_Type (N : Node_Id; Typ : Entity_Id) return Boolean;
+   function Has_Compatible_Type
+     (N              : Node_Id;
+      Typ            : Entity_Id;
+      For_Comparison : Boolean := False) return Boolean;
    --  Verify that some interpretation of the node N has a type compatible with
    --  Typ. If N is not overloaded, then its unique type must be compatible
    --  with Typ. Otherwise iterate through the interpretations of N looking for
-   --  a compatible one.
+   --  a compatible one. If For_Comparison is true, the function is invoked for
+   --  a comparison (or equality) operator and also needs to verify the reverse
+   --  compatibility, because the implementation of type resolution for these
+   --  operators is not fully symmetrical.
 
    function Hides_Op (F : Entity_Id; Op : Entity_Id) return Boolean;
    --  A user-defined function hides a predefined operator if it matches the
diff --git a/gcc/ada/sem_util.adb b/gcc/ada/sem_util.adb
index db4d55a..5feb83d 100644
--- a/gcc/ada/sem_util.adb
+++ b/gcc/ada/sem_util.adb
@@ -686,9 +686,9 @@ package body Sem_Util is
                return New_Occurrence_Of
                         (Get_Dynamic_Accessibility (E), Loc);
 
-            --  Initialization procedures have a special extra accessitility
+            --  Initialization procedures have a special extra accessibility
             --  parameter associated with the level at which the object
-            --  begin initialized exists
+            --  being initialized exists
 
             elsif Ekind (E) = E_Record_Type
               and then Is_Limited_Record (E)
@@ -723,17 +723,11 @@ package body Sem_Util is
             --  Note: We check if the original node of the renaming comes
             --  from source because the node may have been rewritten.
 
-            elsif Present (Renamed_Object (E))
-              and then Comes_From_Source (Original_Node (Renamed_Object (E)))
+            elsif Present (Renamed_Entity_Or_Object (E))
+              and then Comes_From_Source
+                (Original_Node (Renamed_Entity_Or_Object (E)))
             then
-               return Accessibility_Level (Renamed_Object (E));
-
-            --  Move up renamed entities
-
-            elsif Present (Renamed_Entity (E))
-              and then Comes_From_Source (Original_Node (Renamed_Entity (E)))
-            then
-               return Accessibility_Level (Renamed_Entity (E));
+               return Accessibility_Level (Renamed_Entity_Or_Object (E));
 
             --  Named access types get their level from their associated type
 
@@ -750,6 +744,13 @@ package body Sem_Util is
             then
                return Accessibility_Level (Related_Expression (E));
 
+            elsif Level = Dynamic_Level
+               and then Ekind (E) in E_In_Parameter | E_In_Out_Parameter
+               and then Present (Init_Proc_Level_Formal (Scope (E)))
+            then
+               return New_Occurrence_Of
+                        (Init_Proc_Level_Formal (Scope (E)), Loc);
+
             --  Normal object - get the level of the enclosing scope
 
             else
@@ -1605,6 +1606,27 @@ package body Sem_Util is
         and then Scope_Depth (ST) >= Scope_Depth (SCT);
    end Available_Full_View_Of_Component;
 
+   ----------------
+   -- Bad_Aspect --
+   ----------------
+
+   procedure Bad_Aspect
+     (N    : Node_Id;
+      Nam  : Name_Id;
+      Warn : Boolean := False)
+   is
+   begin
+      Error_Msg_Warn := Warn;
+      Error_Msg_N ("<<& is not a valid aspect identifier", N);
+
+      --  Check bad spelling
+      Error_Msg_Name_1 := Aspect_Spell_Check (Nam);
+      if Error_Msg_Name_1 /= No_Name then
+         Error_Msg_N -- CODEFIX
+            ("\<<possible misspelling of %", N);
+      end if;
+   end Bad_Aspect;
+
    -------------------
    -- Bad_Attribute --
    -------------------
@@ -1616,20 +1638,15 @@ package body Sem_Util is
    is
    begin
       Error_Msg_Warn := Warn;
-      Error_Msg_N ("unrecognized attribute&<<", N);
+      Error_Msg_N ("<<unrecognized attribute&", N);
 
       --  Check for possible misspelling
 
-      Error_Msg_Name_1 := First_Attribute_Name;
-      while Error_Msg_Name_1 <= Last_Attribute_Name loop
-         if Is_Bad_Spelling_Of (Nam, Error_Msg_Name_1) then
-            Error_Msg_N -- CODEFIX
-              ("\possible misspelling of %<<", N);
-            exit;
-         end if;
-
-         Error_Msg_Name_1 := Error_Msg_Name_1 + 1;
-      end loop;
+      Error_Msg_Name_1 := Attribute_Spell_Check (Nam);
+      if Error_Msg_Name_1 /= No_Name then
+         Error_Msg_N -- CODEFIX
+            ("\<<possible misspelling of %", N);
+      end if;
    end Bad_Attribute;
 
    --------------------------------
@@ -4860,17 +4877,17 @@ package body Sem_Util is
                if Pragma_Name (Prag) = Name_Contract_Cases then
                   Error_Msg_NE (Adjust_Message
                     ("contract case does not check the outcome of calling "
-                     & "&?T?"), Expr, Subp_Id);
+                     & "&?.t?"), Expr, Subp_Id);
 
                elsif Pragma_Name (Prag) = Name_Refined_Post then
                   Error_Msg_NE (Adjust_Message
                     ("refined postcondition does not check the outcome of "
-                     & "calling &?T?"), Err_Node, Subp_Id);
+                     & "calling &?.t?"), Err_Node, Subp_Id);
 
                else
                   Error_Msg_NE (Adjust_Message
                     ("postcondition does not check the outcome of calling "
-                     & "&?T?"), Err_Node, Subp_Id);
+                     & "&?.t?"), Err_Node, Subp_Id);
                end if;
             end if;
          end Check_Conjunct;
@@ -5138,20 +5155,20 @@ package body Sem_Util is
       then
          Error_Msg_N
            ("neither postcondition nor contract cases mention function "
-            & "result?T?", Post_Prag);
+            & "result?.t?", Post_Prag);
 
       --  The function has contract cases only and they do not mention
       --  attribute 'Result.
 
       elsif Present (Case_Prag) and then not Seen_In_Case then
-         Error_Msg_N ("contract cases do not mention result?T?", Case_Prag);
+         Error_Msg_N ("contract cases do not mention result?.t?", Case_Prag);
 
       --  The function has postconditions only and they do not mention
       --  attribute 'Result.
 
       elsif Present (Post_Prag) and then not Seen_In_Post then
          Error_Msg_N
-           ("postcondition does not mention function result?T?", Post_Prag);
+           ("postcondition does not mention function result?.t?", Post_Prag);
       end if;
    end Check_Result_And_Post_State;
 
@@ -6803,14 +6820,11 @@ package body Sem_Util is
    function Copy_Parameter_List (Subp_Id : Entity_Id) return List_Id is
       Loc    : constant Source_Ptr := Sloc (Subp_Id);
       Plist  : List_Id;
-      Formal : Entity_Id;
+      Formal : Entity_Id := First_Formal (Subp_Id);
 
    begin
-      if No (First_Formal (Subp_Id)) then
-         return No_List;
-      else
-         Plist  := New_List;
-         Formal := First_Formal (Subp_Id);
+      if Present (Formal) then
+         Plist := New_List;
          while Present (Formal) loop
             Append_To (Plist,
               Make_Parameter_Specification (Loc,
@@ -6825,6 +6839,8 @@ package body Sem_Util is
 
             Next_Formal (Formal);
          end loop;
+      else
+         Plist := No_List;
       end if;
 
       return Plist;
@@ -7369,7 +7385,7 @@ package body Sem_Util is
       function Is_Valid_Renaming (N : Node_Id) return Boolean is
       begin
          if Is_Object_Renaming (N)
-           and then not Is_Valid_Renaming (Renamed_Entity (Entity (N)))
+           and then not Is_Valid_Renaming (Renamed_Object (Entity (N)))
          then
             return False;
          end if;
@@ -7561,12 +7577,12 @@ package body Sem_Util is
       elsif Is_Object_Renaming (A1)
         and then Is_Valid_Renaming (A1)
       then
-         return Denotes_Same_Object (Renamed_Entity (Entity (A1)), A2);
+         return Denotes_Same_Object (Renamed_Object (Entity (A1)), A2);
 
       elsif Is_Object_Renaming (A2)
         and then Is_Valid_Renaming (A2)
       then
-         return Denotes_Same_Object (A1, Renamed_Entity (Entity (A2)));
+         return Denotes_Same_Object (A1, Renamed_Object (Entity (A2)));
 
       else
          return False;
@@ -10888,8 +10904,8 @@ package body Sem_Util is
    function Get_Generic_Entity (N : Node_Id) return Entity_Id is
       Ent : constant Entity_Id := Entity (Name (N));
    begin
-      if Present (Renamed_Object (Ent)) then
-         return Renamed_Object (Ent);
+      if Present (Renamed_Entity (Ent)) then
+         return Renamed_Entity (Ent);
       else
          return Ent;
       end if;
@@ -11406,10 +11422,8 @@ package body Sem_Util is
    ------------------------
 
    function Get_Renamed_Entity (E : Entity_Id) return Entity_Id is
-      R : Entity_Id;
-
+      R : Entity_Id := E;
    begin
-      R := E;
       while Present (Renamed_Entity (R)) loop
          R := Renamed_Entity (R);
       end loop;
@@ -16644,7 +16658,8 @@ package body Sem_Util is
             --  Predicate_Failure aspect, for which we do not construct a
             --  wrapper procedure. The subtype will be replaced by the
             --  expression being tested when the corresponding predicate
-            --  check is expanded.
+            --  check is expanded. It may also appear in the pragma Predicate
+            --  expression during legality checking.
 
             elsif Nkind (P) = N_Aspect_Specification
               and then Nkind (Parent (P)) = N_Subtype_Declaration
@@ -16652,7 +16667,8 @@ package body Sem_Util is
                return True;
 
             elsif Nkind (P) = N_Pragma
-              and then Get_Pragma_Id (P) = Pragma_Predicate_Failure
+              and then Get_Pragma_Id (P) in Pragma_Predicate
+                                          | Pragma_Predicate_Failure
             then
                return True;
             end if;
@@ -24095,7 +24111,8 @@ package body Sem_Util is
             --  declaration.
 
             elsif Nkind (N) = N_Object_Renaming_Declaration then
-               Set_Renamed_Object (Defining_Entity (Result), Name (Result));
+               Set_Renamed_Object_Of_Possibly_Void
+                 (Defining_Entity (Result), Name (Result));
 
             --  Update the First_Real_Statement attribute of a replicated
             --  handled sequence of statements.
diff --git a/gcc/ada/sem_util.ads b/gcc/ada/sem_util.ads
index 85010b5..abc18ec 100644
--- a/gcc/ada/sem_util.ads
+++ b/gcc/ada/sem_util.ads
@@ -209,6 +209,14 @@ package Sem_Util is
    --  are open, and the scope of the array is not outside the scope of the
    --  component.
 
+   procedure Bad_Aspect
+     (N    : Node_Id;
+      Nam  : Name_Id;
+      Warn : Boolean := False);
+   --  Called when node N is expected to contain a valid aspect name, and
+   --  Nam is found instead. If Warn is set True this is a warning, else this
+   --  is an error.
+
    procedure Bad_Attribute
      (N    : Node_Id;
       Nam  : Name_Id;
diff --git a/gcc/ada/sem_warn.adb b/gcc/ada/sem_warn.adb
index 9e337f9..d9d5d95 100644
--- a/gcc/ada/sem_warn.adb
+++ b/gcc/ada/sem_warn.adb
@@ -2408,12 +2408,12 @@ package body Sem_Warn is
 
             E1 := First_Entity (P);
             while Present (E1) loop
-               if Ekind (E1) = E_Package and then Renamed_Object (E1) = L then
+               if Ekind (E1) = E_Package and then Renamed_Entity (E1) = L then
                   Is_Visible_Renaming := not Is_Hidden (E1);
                   return E1;
 
                elsif Ekind (E1) = E_Package
-                 and then No (Renamed_Object (E1))
+                 and then No (Renamed_Entity (E1))
                  and then not Is_Generic_Instance (E1)
                then
                   R := Find_Package_Renaming (E1, L);
@@ -3293,21 +3293,21 @@ package body Sem_Warn is
 
             elsif Warnings_Off_Used_Unmodified (E) then
                Error_Msg_NE
-                 ("?W?could use Unmodified instead of "
+                 ("?.w?could use Unmodified instead of "
                   & "Warnings Off for &", Pragma_Identifier (N), E);
 
             --  Used only in context where Unreferenced would have worked
 
             elsif Warnings_Off_Used_Unreferenced (E) then
                Error_Msg_NE
-                 ("?W?could use Unreferenced instead of "
+                 ("?.w?could use Unreferenced instead of "
                   & "Warnings Off for &", Pragma_Identifier (N), E);
 
             --  Not used at all
 
             else
                Error_Msg_NE
-                 ("?W?pragma Warnings Off for & unused, "
+                 ("?.w?pragma Warnings Off for & unused, "
                   & "could be omitted", N, E);
             end if;
          end;
@@ -3863,7 +3863,7 @@ package body Sem_Warn is
                      --  This is one of the messages
 
                      Error_Msg_FE
-                       ("<I<writable actual for & overlaps with actual for &",
+                       ("<.i<writable actual for & overlaps with actual for &",
                         Act1, Form1);
                   end if;
                end if;
@@ -4220,11 +4220,11 @@ package body Sem_Warn is
                if Nkind (Par) = N_Op_Eq then
                   Error_Msg_N
                     ("suspicious equality test with modified version of "
-                     & "same object?T?", Par);
+                     & "same object?.t?", Par);
                else
                   Error_Msg_N
                     ("suspicious inequality test with modified version of "
-                     & "same object?T?", Par);
+                     & "same object?.t?", Par);
                end if;
             end if;
          end if;
diff --git a/gcc/ada/sigtramp-ios.c b/gcc/ada/sigtramp-ios.c
index 6e2913d..8403a18 100644
--- a/gcc/ada/sigtramp-ios.c
+++ b/gcc/ada/sigtramp-ios.c
@@ -71,6 +71,10 @@
 
 /* sigtramp stub providing unwind info for common registers.  */
 
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
 extern void __gnat_sigtramp_common
   (int signo, void *siginfo, void *sigcontext,
    __sigtramphandler_t * handler);
@@ -87,6 +91,10 @@ void __gnat_sigtramp (int signo, void *si, void *ucontext,
   __gnat_sigtramp_common (signo, si, mcontext, handler);
 }
 
+#if defined(__cplusplus)
+}
+#endif
+
 /* asm string construction helpers.  */
 
 #define STR(TEXT) #TEXT
diff --git a/gcc/ada/sinfo-utils.adb b/gcc/ada/sinfo-utils.adb
index 79269a5..ad8606f 100644
--- a/gcc/ada/sinfo-utils.adb
+++ b/gcc/ada/sinfo-utils.adb
@@ -189,13 +189,9 @@ package body Sinfo.Utils is
    ------------------
 
    function End_Location (N : Node_Id) return Source_Ptr is
-      L : constant Uint := End_Span (N);
+      L : constant Valid_Uint := End_Span (N);
    begin
-      if No (L) then
-         return No_Location;
-      else
-         return Source_Ptr (Int (Sloc (N)) + UI_To_Int (L));
-      end if;
+      return Source_Ptr (Int (Sloc (N)) + UI_To_Int (L));
    end End_Location;
 
    --------------------
@@ -279,6 +275,8 @@ package body Sinfo.Utils is
             declare
                Desc : Field_Descriptor renames
                  Field_Descriptors (Fields (J));
+               pragma Assert (Desc.Type_Only = No_Type_Only);
+               --  Type_Only is for entities
             begin
                if Is_In_Union_Id (Desc.Kind) then
                   Action (Get_Node_Field_Union (N, Desc.Offset));
@@ -304,6 +302,8 @@ package body Sinfo.Utils is
             declare
                Desc : Field_Descriptor renames
                  Field_Descriptors (Fields (J));
+               pragma Assert (Desc.Type_Only = No_Type_Only);
+               --  Type_Only is for entities
             begin
                if Is_In_Union_Id (Desc.Kind) then
                   Set_Node_Field_Union
diff --git a/gcc/ada/sinput.ads b/gcc/ada/sinput.ads
index 2926b85..881e775 100644
--- a/gcc/ada/sinput.ads
+++ b/gcc/ada/sinput.ads
@@ -412,10 +412,10 @@ package Sinput is
    --  Initialized so that some tools (such as gprbuild) can be built with
    --  -gnatVa and pragma Initialize_Scalars without problems.
 
-   Current_Source_Unit : Unit_Number_Type;
-   --  Unit number of source file currently being scanned. The special value
-   --  of No_Unit indicates that the configuration pragma file is currently
-   --  being scanned (this has no entry in the unit table).
+   Current_Source_Unit : Unit_Number_Type := No_Unit;
+   --  Unit number of source file currently being scanned. Initialized to
+   --  No_Unit for pre-processing and the configuration pragma file scanning,
+   --  since both stages have no corresponding entry in the unit table.
 
    Source_gnat_adc : Source_File_Index := No_Source_File;
    --  This is set if a gnat.adc file is present to reference this file
diff --git a/gcc/ada/sprint.adb b/gcc/ada/sprint.adb
index 69996cb..611563a 100644
--- a/gcc/ada/sprint.adb
+++ b/gcc/ada/sprint.adb
@@ -1116,6 +1116,12 @@ package body Sprint is
                Write_Char (']');
             end if;
 
+            if Present (Procedure_To_Call (Node)) then
+               Write_Str_With_Col_Check ("[procedure_to_call = ");
+               Sprint_Node (Procedure_To_Call (Node));
+               Write_Char (']');
+            end if;
+
          when N_And_Then =>
             Sprint_Left_Opnd (Node);
             Write_Str_Sloc (" and then ");
@@ -1785,11 +1791,23 @@ package body Sprint is
             if Present (Handled_Statement_Sequence (Node)) then
                Write_Str_With_Col_Check (" do");
                Sprint_Node (Handled_Statement_Sequence (Node));
-               Write_Indent_Str ("end return;");
-            else
-               Write_Indent_Str (";");
+               Write_Indent_Str ("end return");
+            end if;
+
+            if Present (Storage_Pool (Node)) then
+               Write_Str_With_Col_Check ("[storage_pool = ");
+               Sprint_Node (Storage_Pool (Node));
+               Write_Char (']');
             end if;
 
+            if Present (Procedure_To_Call (Node)) then
+               Write_Str_With_Col_Check ("[procedure_to_call = ");
+               Sprint_Node (Procedure_To_Call (Node));
+               Write_Char (']');
+            end if;
+
+            Write_Char (';');
+
          when N_Delta_Aggregate =>
             Write_Str_With_Col_Check_Sloc ("(");
             Sprint_Node (Expression (Node));
@@ -1973,6 +1991,19 @@ package body Sprint is
          when N_Free_Statement =>
             Write_Indent_Str_Sloc ("free ");
             Sprint_Node (Expression (Node));
+
+            if Present (Storage_Pool (Node)) then
+               Write_Str_With_Col_Check ("[storage_pool = ");
+               Sprint_Node (Storage_Pool (Node));
+               Write_Char (']');
+            end if;
+
+            if Present (Procedure_To_Call (Node)) then
+               Write_Str_With_Col_Check ("[procedure_to_call = ");
+               Sprint_Node (Procedure_To_Call (Node));
+               Write_Char (']');
+            end if;
+
             Write_Char (';');
 
          when N_Freeze_Entity =>
@@ -3170,11 +3201,24 @@ package body Sprint is
             if Present (Expression (Node)) then
                Write_Indent_Str_Sloc ("return ");
                Sprint_Node (Expression (Node));
-               Write_Char (';');
             else
-               Write_Indent_Str_Sloc ("return;");
+               Write_Indent_Str_Sloc ("return");
+            end if;
+
+            if Present (Storage_Pool (Node)) then
+               Write_Str_With_Col_Check ("[storage_pool = ");
+               Sprint_Node (Storage_Pool (Node));
+               Write_Char (']');
             end if;
 
+            if Present (Procedure_To_Call (Node)) then
+               Write_Str_With_Col_Check ("[procedure_to_call = ");
+               Sprint_Node (Procedure_To_Call (Node));
+               Write_Char (']');
+            end if;
+
+            Write_Char (';');
+
          when N_Selective_Accept =>
             Write_Indent_Str_Sloc ("select");
 
diff --git a/gcc/ada/strub.adb b/gcc/ada/strub.adb
new file mode 100644
index 0000000..8414658
--- /dev/null
+++ b/gcc/ada/strub.adb
@@ -0,0 +1,301 @@
+------------------------------------------------------------------------------
+--                                                                          --
+--                         GNAT COMPILER COMPONENTS                         --
+--                                                                          --
+--                                S T R U B                                 --
+--                                                                          --
+--                                 B o d y                                  --
+--                                                                          --
+--            Copyright (C) 2021, Free Software Foundation, Inc.            --
+--                                                                          --
+-- GNAT is free software;  you can  redistribute it  and/or modify it under --
+-- terms of the  GNU General Public License as published  by the Free Soft- --
+-- ware  Foundation;  either version 3,  or (at your option) any later ver- --
+-- sion.  GNAT is distributed in the hope that it will be useful, but WITH- --
+-- OUT ANY WARRANTY;  without even the  implied warranty of MERCHANTABILITY --
+-- or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License --
+-- for  more details.  You should have  received  a copy of the GNU General --
+-- Public License  distributed with GNAT; see file COPYING3.  If not, go to --
+-- http://www.gnu.org/licenses for a complete copy of the license.          --
+--                                                                          --
+-- GNAT was originally developed  by the GNAT team at  New York University. --
+-- Extensive contributions were provided by Ada Core Technologies Inc.      --
+--                                                                          --
+------------------------------------------------------------------------------
+
+--  Package containing utility procedures related to Stack Scrubbing
+
+with Atree;          use Atree;
+with Einfo;          use Einfo;
+with Einfo.Entities; use Einfo.Entities;
+with Errout;         use Errout;
+with Namet;          use Namet;
+with Nlists;         use Nlists;
+with Sem_Eval;       use Sem_Eval;
+with Sinfo;          use Sinfo;
+with Sinfo.Nodes;    use Sinfo.Nodes;
+with Sinfo.Utils;    use Sinfo.Utils;
+with Snames;         use Snames;
+with Stringt;        use Stringt;
+
+package body Strub is
+   -----------------------
+   -- Local Subprograms --
+   -----------------------
+
+   function Find_Explicit_Strub_Pragma (Id : Entity_Id) return Node_Id;
+   --  Return a pragma Machine_Attribute (Id, "strub"[, "mode"]) node
+   --  if Id has one.
+
+   function Strub_Pragma_Arg (Item : Node_Id) return Node_Id is
+      (Get_Pragma_Arg
+         (Next (Next (First (Pragma_Argument_Associations (Item))))));
+   --  Return the pragma argument holding the strub mode associated
+   --  with Item, a subprogram, variable, constant, or type. Bear in
+   --  mind that strub pragmas with an explicit strub mode argument,
+   --  naming access-to-subprogram types, are applied to the
+   --  designated subprogram type.
+
+   function Strub_Pragma_Arg_To_String (Item : Node_Id) return String is
+      (To_String (Strval (Expr_Value_S (Item))));
+   --  Extract and return as a String the strub mode held in a node
+   --  returned by Strub_Pragma_Arg.
+
+   function Strub_Pragma_Mode
+     (Id   : Entity_Id;
+      Item : Node_Id) return Strub_Mode;
+   --  Return the strub mode associated with Item expressed in Id.
+   --  Strub_Pragma_P (Id) must hold.
+
+   ---------------------------
+   -- Check_Same_Strub_Mode --
+   ---------------------------
+
+   procedure Check_Same_Strub_Mode
+     (Dest, Src : Entity_Id;
+      Report    : Boolean := True)
+   is
+      Src_Strub_Mode  : constant Strub_Mode := Explicit_Strub_Mode (Src);
+      Dest_Strub_Mode : constant Strub_Mode := Explicit_Strub_Mode (Dest);
+
+   begin
+      if Dest_Strub_Mode = Src_Strub_Mode then
+         return;
+      end if;
+
+      --  Internal is not part of the interface, it's an *internal*
+      --  implementation detail, so consider it equivalent to unspecified here.
+      --  ??? -fstrub=relaxed|strict makes them interface-equivalent to
+      --  Callable or Disabled, respectively, but we don't look at that flag in
+      --  the front-end, and it seems undesirable for that flag to affect
+      --  whether specifications are conformant. Maybe there should be some
+      --  means to specify Callable or Disabled along with Internal?
+
+      if Dest_Strub_Mode in Unspecified | Internal
+        and then Src_Strub_Mode in Unspecified | Internal
+      then
+         return;
+      end if;
+
+      if not Report then
+         return;
+      end if;
+
+      if Src_Strub_Mode /= Unspecified then
+         Error_Msg_Sloc := Sloc (Find_Explicit_Strub_Pragma (Src));
+      else
+         Error_Msg_Sloc := Sloc (Src);
+      end if;
+      Error_Msg_Node_2 := Src;
+      Error_Msg_NE ("& requires the same `strub` mode as &#",
+                    (if Dest_Strub_Mode /= Unspecified
+                       then Find_Explicit_Strub_Pragma (Dest)
+                       else Dest),
+                    Dest);
+   end Check_Same_Strub_Mode;
+
+   ----------------------------
+   -- Compatible_Strub_Modes --
+   ----------------------------
+
+   function Compatible_Strub_Modes
+     (Dest, Src : Entity_Id) return Boolean
+   is
+      Src_Strub_Mode : constant Strub_Mode := Explicit_Strub_Mode (Src);
+      Dest_Strub_Mode : constant Strub_Mode := Explicit_Strub_Mode (Dest);
+
+   begin
+      return Src_Strub_Mode = Dest_Strub_Mode
+        or else At_Calls not in Src_Strub_Mode | Dest_Strub_Mode;
+   end Compatible_Strub_Modes;
+
+   ---------------------
+   -- Copy_Strub_Mode --
+   ---------------------
+
+   procedure Copy_Strub_Mode (Dest, Src : Entity_Id) is
+      Strub : Node_Id := Find_Explicit_Strub_Pragma (Src);
+      Src_Strub_Mode : constant Strub_Mode := Strub_Pragma_Mode (Src, Strub);
+
+   begin
+      pragma Assert (Explicit_Strub_Mode (Dest) = Unspecified);
+
+      --  Refrain from copying Internal to subprogram types.
+      --  It affects code generation for the subprogram,
+      --  but it has no effect on its type or interface.
+
+      if Src_Strub_Mode = Unspecified
+        or else (Ekind (Dest) = E_Subprogram_Type
+                   and then Src_Strub_Mode = Internal)
+      then
+         return;
+      end if;
+
+      Strub := New_Copy (Strub);
+      Set_Next_Rep_Item (Strub, First_Rep_Item (Dest));
+      Set_First_Rep_Item (Dest, Strub);
+      Set_Has_Gigi_Rep_Item (Dest);
+   end Copy_Strub_Mode;
+
+   -------------------------
+   -- Explicit_Strub_Mode --
+   -------------------------
+
+   function Explicit_Strub_Mode (Id : Entity_Id) return Strub_Mode is
+      Item : constant Node_Id := Find_Explicit_Strub_Pragma (Id);
+
+   begin
+      return Strub_Pragma_Mode (Id, Item);
+   end Explicit_Strub_Mode;
+
+   --------------------------------
+   -- Find_Explicit_Strub_Pragma --
+   --------------------------------
+
+   function Find_Explicit_Strub_Pragma (Id : Entity_Id) return Node_Id is
+      Item : Node_Id;
+
+   begin
+      if not Has_Gigi_Rep_Item (Id) then
+         return Empty;
+      end if;
+
+      Item := First_Rep_Item (Id);
+      while Present (Item) loop
+         if Strub_Pragma_P (Item) then
+            return Item;
+         end if;
+         Item := Next_Rep_Item (Item);
+      end loop;
+
+      return Empty;
+   end Find_Explicit_Strub_Pragma;
+
+   -----------------------
+   -- Strub_Pragma_Mode --
+   -----------------------
+
+   function Strub_Pragma_Mode
+     (Id   : Entity_Id;
+      Item : Node_Id) return Strub_Mode
+   is
+      Arg : Node_Id := Empty;
+
+   begin
+      --  ??? Enumeration literals, despite being conceptually functions, have
+      --  neither bodies nor stack frames, and it's not clear whether it would
+      --  make more sense to treat them as subprograms or as constants, but
+      --  they can be renamed as functions.  Should we require all literals of
+      --  a type to have the same strub mode?  Rule out their annotation?
+
+      if Ekind (Id) in E_Subprogram_Type
+                     | Overloadable_Kind
+                     | Generic_Subprogram_Kind
+      then
+         if Item = Empty then
+            return Unspecified;
+         end if;
+
+         Arg := Strub_Pragma_Arg (Item);
+         if Arg = Empty then
+            return At_Calls;
+         end if;
+
+         declare
+            Str : constant String := Strub_Pragma_Arg_To_String (Arg);
+         begin
+            if Str'Length /= 8 then
+               return Unspecified;
+            end if;
+
+            case Str (Str'First) is
+               when 'a' =>
+                  if Str = "at-calls" then
+                     return At_Calls;
+                  end if;
+
+               when 'i' =>
+                  if Str = "internal" then
+                     return Internal;
+                  end if;
+
+               when 'c' =>
+                  if Str = "callable" then
+                     return Callable;
+                  end if;
+
+               when 'd' =>
+                  if Str = "disabled" then
+                     return Disabled;
+                  end if;
+
+               when others =>
+                  null;
+            end case;
+            return Unspecified;
+         end;
+
+      --  Access-to-subprogram types and variables can be treated just like
+      --  other access types, because the pragma logic has already promoted to
+      --  subprogram types any annotations applicable to them.
+
+      elsif Ekind (Id) in Type_Kind -- except E_Subprogram_Type, covered above
+                        | Scalar_Kind
+                        | Object_Kind
+                        | Named_Kind
+      then
+         if Item = Empty then
+            return Unspecified;
+         end if;
+
+         Arg := Strub_Pragma_Arg (Item);
+         if Arg /= Empty then
+            --  A strub parameter is not applicable to variables,
+            --  and will be ignored.
+
+            return Unspecified;
+         end if;
+
+         return Enabled;
+
+      else
+         pragma Assert (Item = Empty);
+         return Not_Applicable;
+      end if;
+   end Strub_Pragma_Mode;
+
+   --------------------
+   -- Strub_Pragma_P --
+   --------------------
+
+   function Strub_Pragma_P
+     (Item : Node_Id) return Boolean is
+      (Nkind (Item) = N_Pragma
+         and then Pragma_Name (Item) = Name_Machine_Attribute
+         and then
+           Strub_Pragma_Arg_To_String
+             (Get_Pragma_Arg
+                (Next (First (Pragma_Argument_Associations (Item)))))
+             = "strub");
+
+end Strub;
diff --git a/gcc/ada/strub.ads b/gcc/ada/strub.ads
new file mode 100644
index 0000000..2753dd0
--- /dev/null
+++ b/gcc/ada/strub.ads
@@ -0,0 +1,115 @@
+------------------------------------------------------------------------------
+--                                                                          --
+--                         GNAT COMPILER COMPONENTS                         --
+--                                                                          --
+--                                S T R U B                                 --
+--                                                                          --
+--                                 S p e c                                  --
+--                                                                          --
+--            Copyright (C) 2021, Free Software Foundation, Inc.            --
+--                                                                          --
+-- GNAT is free software;  you can  redistribute it  and/or modify it under --
+-- terms of the  GNU General Public License as published  by the Free Soft- --
+-- ware  Foundation;  either version 3,  or (at your option) any later ver- --
+-- sion.  GNAT is distributed in the hope that it will be useful, but WITH- --
+-- OUT ANY WARRANTY;  without even the  implied warranty of MERCHANTABILITY --
+-- or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License --
+-- for  more details.  You should have  received  a copy of the GNU General --
+-- Public License  distributed with GNAT; see file COPYING3.  If not, go to --
+-- http://www.gnu.org/licenses for a complete copy of the license.          --
+--                                                                          --
+-- GNAT was originally developed  by the GNAT team at  New York University. --
+-- Extensive contributions were provided by Ada Core Technologies Inc.      --
+--                                                                          --
+------------------------------------------------------------------------------
+
+--  Package containing utility procedures related to Stack Scrubbing
+
+with Types; use Types;
+
+package Strub is
+   type Strub_Mode is
+     (Disabled,        --  Subprogram cannot be called from strub contexts
+      At_Calls,        --  Subprogram strubbed by caller
+      Internal,        --  Subprogram strubbed by wrapper
+      Callable,        --  Subprogram safe to call despite no strub
+      Unspecified,     --  Subprogram or data without strub annotation
+      Enabled,         --  Data (variable or constant) that enables strub
+      Not_Applicable); --  Entities that are not strub-capable
+   --  This is the type that expresses decoded strub annotations
+
+   --  We compare strub modes in the following circumstances:
+
+   --  * subprogram definition vs specification
+   --  * overriding vs overridden dispatch subprograms
+   --  * implementation vs interface dispatch subprogram
+   --  * renaming vs renamed subprogram
+   --  * type resolution
+   --  * explicit conversions
+
+   --  Explicit conversions can convert between strub modes other than
+   --  at-calls (see Compatible_Strub_Modes), but for the other cases
+   --  above, we insist on identity of the strub modes (see
+   --  Check_Same_Strub_Mode). Anything else would be
+   --  troublesome.
+
+   --  E.g., overriding a callable subprogram with a strub-disabled
+   --  implementation would enable a subprogram that's unsafe to call
+   --  in strub contexts to be called through a dispatching
+   --  interface. An explicitly strub-disabled subprogram shall not be
+   --  called from strub contexts, and a callable overriding
+   --  subprogram would still seem not-callable, so accepting
+   --  different modes would be surprising.
+
+   --  We could relax the requirement for overriders from equality to
+   --  compatibility, with the understanding that the dispatching ABI
+   --  is what prevails. For renaming, however, if we don't require
+   --  equality, it would have to encompass an implicit conversion.
+
+   procedure Check_Same_Strub_Mode
+     (Dest, Src : Entity_Id;
+      Report    : Boolean := True);
+   --  Check whether Dest and Src are subprograms or subprogram types
+   --  annotated (or not) with the same strub mode. If Report is
+   --  requested, and the strub modes are not equivalent, an error
+   --  message is issued. Unspecified and Internal are considered
+   --  equivalent, because Internal is an internal implementation
+   --  detail. Unspecified decays to Disabled or Callable depending on
+   --  -fstrub=(strict|relaxed), but this procedure does not take this
+   --  decay into account, which avoids turning strub-equivalent
+   --  declarations into incompatible ones at command-line changes.
+
+   function Compatible_Strub_Modes
+     (Dest, Src : Entity_Id) return Boolean;
+   --  Return True if Dest and Src are subprograms or subprogram types
+   --  annotated (or not) with ABI-compatible strub modes. At-calls is
+   --  incompatible to other strub modes, because the back end
+   --  internally modifies the signature of such subprograms, adding
+   --  hidden parameters. Calling a subprogram through an
+   --  access-to-subprogram object converted between strub-at-calls
+   --  and other strub modes should be deemed equivalent to
+   --  dereferencing an uninitialized access-to-data object, though
+   --  one-way conversions might seem to work in some circumstances.
+   --
+   --  Unspecified, Disabled, Internal and Callable
+   --  (access-to-)subprograms, on the other hand, can be safely but
+   --  explicitly converted to each other, because these strub modes
+   --  do not require signature changes; so it is possible to alter
+   --  the caller-side stack scrubbing semantics of the call (e.g. to
+   --  call a subprogram that isn't strub-callable from within a strub
+   --  context, or to prevent it from being called through an access
+   --  object) without any incompatibilities.
+
+   procedure Copy_Strub_Mode (Dest, Src : Entity_Id);
+   --  Copy the strub mode from Src to Dest, subprograms or subprogram
+   --  types. Dest is required to not have a strub mode already set.
+
+   function Explicit_Strub_Mode (Id : Entity_Id) return Strub_Mode;
+   --  Return the strub mode associated with Id, that should refer to
+   --  a subprogram, a data object, or a type.
+
+   function Strub_Pragma_P (Item : Node_Id) return Boolean;
+   --  Return True iff Item is a strub annotation, specifically, one
+   --  introduced by pragma Machine_Attribute (Entity, "strub"[, "mode"]).
+
+end Strub;
diff --git a/gcc/ada/tbuild.adb b/gcc/ada/tbuild.adb
index 4d9c1c4..bcd77c7 100644
--- a/gcc/ada/tbuild.adb
+++ b/gcc/ada/tbuild.adb
@@ -699,11 +699,10 @@ package body Tbuild is
       Loc    : Source_Ptr) return Node_Id
    is
       pragma Assert (Present (Def_Id) and then Nkind (Def_Id) in N_Entity);
-      Occurrence : Node_Id;
+      Occurrence : constant Node_Id :=
+        Make_Identifier (Loc, Chars (Def_Id));
 
    begin
-      Occurrence := New_Node (N_Identifier, Loc);
-      Set_Chars (Occurrence, Chars (Def_Id));
       Set_Entity (Occurrence, Def_Id);
 
       if Is_Type (Def_Id) then
diff --git a/gcc/ada/tbuild.ads b/gcc/ada/tbuild.ads
index eb17865..5199e6d 100644
--- a/gcc/ada/tbuild.ads
+++ b/gcc/ada/tbuild.ads
@@ -312,11 +312,10 @@ package Tbuild is
      (Def_Id : Entity_Id;
       Loc    : Source_Ptr) return Node_Id;
    --  New_Occurrence_Of creates an N_Identifier node that is an occurrence of
-   --  the defining identifier Def_Id. The Entity and Etype of the result are
-   --  set from the given defining identifier as follows: Entity is a copy of
-   --  Def_Id. Etype is a copy of Def_Id for types, and a copy of the Etype of
-   --  Def_Id for other entities. Note that Is_Static_Expression is set if this
-   --  call creates an occurrence of an enumeration literal.
+   --  the defining identifier Def_Id. The Entity of the result is Def_Id. The
+   --  Etype of the result is Def_Id for types, and Etype (Def_Id) otherwise.
+   --  Is_Static_Expression is set if this call creates an occurrence of an
+   --  enumeration literal.
 
    function New_Suffixed_Name
      (Related_Id : Name_Id;
diff --git a/gcc/ada/treepr.adb b/gcc/ada/treepr.adb
index d36042c..aa06506 100644
--- a/gcc/ada/treepr.adb
+++ b/gcc/ada/treepr.adb
@@ -1024,6 +1024,8 @@ package body Treepr is
       FD     : Field_Descriptor;
       Format : UI_Format := Auto)
    is
+      pragma Assert (FD.Type_Only = No_Type_Only);
+      --  Type_Only is for entities
    begin
       if not Field_Is_Initial_Zero (N, Field) then
          Print_Field (Prefix, Image (Field), N, FD, Format);
@@ -1041,9 +1043,10 @@ package body Treepr is
       FD     : Field_Descriptor;
       Format : UI_Format := Auto)
    is
+      NN : constant Node_Id := Node_To_Fetch_From (N, Field);
    begin
       if not Field_Is_Initial_Zero (N, Field) then
-         Print_Field (Prefix, Image (Field), N, FD, Format);
+         Print_Field (Prefix, Image (Field), NN, FD, Format);
       end if;
    end Print_Entity_Field;
 
diff --git a/gcc/ada/uintp.adb b/gcc/ada/uintp.adb
index 5d1dec1..06f6680 100644
--- a/gcc/ada/uintp.adb
+++ b/gcc/ada/uintp.adb
@@ -380,8 +380,8 @@ package body Uintp is
 
       procedure Image_String (S : String) is
       begin
-         for X in S'Range loop
-            Image_Char (S (X));
+         for X of S loop
+            Image_Char (X);
          end loop;
       end Image_String;
 
diff --git a/gcc/ada/uintp.ads b/gcc/ada/uintp.ads
index d9f1f8f..08f6738 100644
--- a/gcc/ada/uintp.ads
+++ b/gcc/ada/uintp.ads
@@ -104,6 +104,7 @@ package Uintp is
    subtype Unat is Valid_Uint with Predicate => Unat >= Uint_0; -- natural
    subtype Upos is Valid_Uint with Predicate => Upos >= Uint_1; -- positive
    subtype Nonzero_Uint is Valid_Uint with Predicate => Nonzero_Uint /= Uint_0;
+   subtype Unegative is Valid_Uint with Predicate => Unegative < Uint_0;
    subtype Ubool is Valid_Uint with Predicate => Ubool in Uint_0 | Uint_1;
    subtype Opt_Ubool is Uint with
      Predicate => No (Opt_Ubool) or else Opt_Ubool in Ubool;
diff --git a/gcc/ada/vxworks7-cert-rtp-base-link.spec b/gcc/ada/vxworks7-cert-rtp-base-link.spec
new file mode 100644
index 0000000..1d6ee49
--- /dev/null
+++ b/gcc/ada/vxworks7-cert-rtp-base-link.spec
@@ -0,0 +1,2 @@
+*base_link:
+--defsym=__wrs_rtp_base=0x80000000
diff --git a/gcc/ada/vxworks7-cert-rtp-base-link__ppc64.spec b/gcc/ada/vxworks7-cert-rtp-base-link__ppc64.spec
new file mode 100644
index 0000000..97332b8
--- /dev/null
+++ b/gcc/ada/vxworks7-cert-rtp-base-link__ppc64.spec
@@ -0,0 +1,2 @@
+*base_link:
+--defsym=__wrs_rtp_base=0x40000000
diff --git a/gcc/ada/vxworks7-cert-rtp-base-link__x86.spec b/gcc/ada/vxworks7-cert-rtp-base-link__x86.spec
new file mode 100644
index 0000000..eafb582
--- /dev/null
+++ b/gcc/ada/vxworks7-cert-rtp-base-link__x86.spec
@@ -0,0 +1,2 @@
+*base_link:
+--defsym=__wrs_rtp_base=0x400000
diff --git a/gcc/ada/vxworks7-cert-rtp-base-link__x86_64.spec b/gcc/ada/vxworks7-cert-rtp-base-link__x86_64.spec
new file mode 100644
index 0000000..dd28869
--- /dev/null
+++ b/gcc/ada/vxworks7-cert-rtp-base-link__x86_64.spec
@@ -0,0 +1,2 @@
+*base_link:
+--defsym=__wrs_rtp_base=0x200000
diff --git a/gcc/ada/vxworks7-cert-rtp-link.spec b/gcc/ada/vxworks7-cert-rtp-link.spec
index 4bbf376..9923c58 100644
--- a/gcc/ada/vxworks7-cert-rtp-link.spec
+++ b/gcc/ada/vxworks7-cert-rtp-link.spec
@@ -3,8 +3,8 @@
 
 *link:
 + %{!nostdlib:%{mrtp:%{!shared: \
+     %(base_link) \
      -l:certRtp.o \
      -L%:getenv(VSB_DIR /usr/lib/common/objcert) \
-     --defsym=__wrs_rtp_base=0x80000000 \
      -T%:getenv(VSB_DIR /usr/ldscripts/rtp.ld) \
    }}}
diff --git a/gcc/ada/warnsw.ads b/gcc/ada/warnsw.ads
index 6113538..f58be1e 100644
--- a/gcc/ada/warnsw.ads
+++ b/gcc/ada/warnsw.ads
@@ -50,7 +50,8 @@ package Warnsw is
 
    Warn_On_Unknown_Compile_Time_Warning : Boolean := True;
    --  Warn on a pragma Compile_Time_Warning whose condition has a value that
-   --  is not known at compile time.
+   --  is not known at compile time. On by default, modified by use
+   --  of -gnatw_c/_C and set as part of -gnatwa.
 
    Warn_On_Overridden_Size : Boolean := False;
    --  Warn when explicit record component clause or array component_size
diff --git a/gcc/analyzer/ChangeLog b/gcc/analyzer/ChangeLog
index 5328f85..40678a1 100644
--- a/gcc/analyzer/ChangeLog
+++ b/gcc/analyzer/ChangeLog
@@ -1,3 +1,90 @@
+2021-11-13  David Malcolm  <dmalcolm@redhat.com>
+
+	* analyzer.opt (Wanalyzer-tainted-allocation-size): New.
+	(Wanalyzer-tainted-divisor): New.
+	(Wanalyzer-tainted-offset): New.
+	(Wanalyzer-tainted-size): New.
+	* engine.cc (impl_region_model_context::get_taint_map): New.
+	* exploded-graph.h (impl_region_model_context::get_taint_map):
+	New decl.
+	* program-state.cc (sm_state_map::get_state): Call
+	alt_get_inherited_state.
+	(sm_state_map::impl_set_state): Modify states within
+	compound svalues.
+	(program_state::impl_call_analyzer_dump_state): Undo casts.
+	(selftest::test_program_state_1): Update for new context param of
+	create_region_for_heap_alloc.
+	(selftest::test_program_state_merging): Likewise.
+	* region-model-impl-calls.cc (region_model::impl_call_alloca):
+	Likewise.
+	(region_model::impl_call_calloc): Likewise.
+	(region_model::impl_call_malloc): Likewise.
+	(region_model::impl_call_operator_new): Likewise.
+	(region_model::impl_call_realloc): Likewise.
+	* region-model.cc (region_model::check_region_access): Call
+	check_region_for_taint.
+	(region_model::get_representative_path_var_1): Handle binops.
+	(region_model::create_region_for_heap_alloc): Add "ctxt" param and
+	pass it to set_dynamic_extents.
+	(region_model::create_region_for_alloca): Likewise.
+	(region_model::set_dynamic_extents): Add "ctxt" param and use it
+	to call check_dynamic_size_for_taint.
+	(selftest::test_state_merging): Update for new context param of
+	create_region_for_heap_alloc.
+	(selftest::test_malloc_constraints): Likewise.
+	(selftest::test_malloc): Likewise.
+	(selftest::test_alloca): Likewise for create_region_for_alloca.
+	* region-model.h (region_model::create_region_for_heap_alloc): Add
+	"ctxt" param.
+	(region_model::create_region_for_alloca): Likewise.
+	(region_model::set_dynamic_extents): Likewise.
+	(region_model::check_dynamic_size_for_taint): New decl.
+	(region_model::check_region_for_taint): New decl.
+	(region_model_context::get_taint_map): New vfunc.
+	(noop_region_model_context::get_taint_map): New.
+	* sm-taint.cc: Remove include of "diagnostic-event-id.h"; add
+	includes of "gimple-iterator.h", "tristate.h", "selftest.h",
+	"ordered-hash-map.h", "cgraph.h", "cfg.h", "digraph.h",
+	"analyzer/supergraph.h", "analyzer/call-string.h",
+	"analyzer/program-point.h", "analyzer/store.h",
+	"analyzer/region-model.h", and "analyzer/program-state.h".
+	(enum bounds): Move to top of file.
+	(class taint_diagnostic): New.
+	(class tainted_array_index): Convert to subclass of taint_diagnostic.
+	(tainted_array_index::emit): Add CWE-129.  Reword warning to use
+	"attacker-controlled" rather than "tainted".
+	(tainted_array_index::describe_state_change): Move to
+	taint_diagnostic::describe_state_change.
+	(tainted_array_index::describe_final_event): Reword to use
+	"attacker-controlled" rather than "tainted".
+	(class tainted_offset): New.
+	(class tainted_size): New.
+	(class tainted_divisor): New.
+	(class tainted_allocation_size): New.
+	(taint_state_machine::alt_get_inherited_state): New.
+	(taint_state_machine::on_stmt): In assignment handling, remove
+	ARRAY_REF handling in favor of check_region_for_taint.  Add
+	detection of tainted divisors.
+	(taint_state_machine::get_taint): New.
+	(taint_state_machine::combine_states): New.
+	(region_model::check_region_for_taint): New.
+	(region_model::check_dynamic_size_for_taint): New.
+	* sm.h (state_machine::alt_get_inherited_state): New.
+
+2021-11-12  David Malcolm  <dmalcolm@redhat.com>
+
+	* engine.cc (exploded_node::on_stmt_pre): Return when handling
+	"__analyzer_dump_state".
+
+2021-11-11  Richard Biener  <rguenther@suse.de>
+
+	* supergraph.cc: Include bitmap.h.
+
+2021-11-04  David Malcolm  <dmalcolm@redhat.com>
+
+	* program-state.cc (sm_state_map::dump): Use default_tree_printer
+	as format decoder.
+
 2021-09-16  Maxim Blinov  <maxim.blinov@embecosm.com>
 
 	PR bootstrap/102242
diff --git a/gcc/analyzer/analyzer.opt b/gcc/analyzer/analyzer.opt
index 6ddb6e3..c85e30f 100644
--- a/gcc/analyzer/analyzer.opt
+++ b/gcc/analyzer/analyzer.opt
@@ -114,10 +114,26 @@ Wanalyzer-stale-setjmp-buffer
 Common Var(warn_analyzer_stale_setjmp_buffer) Init(1) Warning
 Warn about code paths in which a longjmp rewinds to a jmp_buf saved in a stack frame that has returned.
 
+Wanalyzer-tainted-allocation-size
+Common Var(warn_analyzer_tainted_allocation_size) Init(1) Warning
+Warn about code paths in which an unsanitized value is used as an allocation size.
+
 Wanalyzer-tainted-array-index
 Common Var(warn_analyzer_tainted_array_index) Init(1) Warning
 Warn about code paths in which an unsanitized value is used as an array index.
 
+Wanalyzer-tainted-divisor
+Common Var(warn_analyzer_tainted_divisor) Init(1) Warning
+Warn about code paths in which an unsanitized value is used as a divisor.
+
+Wanalyzer-tainted-offset
+Common Var(warn_analyzer_tainted_offset) Init(1) Warning
+Warn about code paths in which an unsanitized value is used as a pointer offset.
+
+Wanalyzer-tainted-size
+Common Var(warn_analyzer_tainted_size) Init(1) Warning
+Warn about code paths in which an unsanitized value is used as a size.
+
 Wanalyzer-use-after-free
 Common Var(warn_analyzer_use_after_free) Init(1) Warning
 Warn about code paths in which a freed value is used.
diff --git a/gcc/analyzer/engine.cc b/gcc/analyzer/engine.cc
index f21f8e5..096e219 100644
--- a/gcc/analyzer/engine.cc
+++ b/gcc/analyzer/engine.cc
@@ -220,6 +220,24 @@ impl_region_model_context::get_malloc_map (sm_state_map **out_smap,
   return true;
 }
 
+bool
+impl_region_model_context::get_taint_map (sm_state_map **out_smap,
+					  const state_machine **out_sm,
+					  unsigned *out_sm_idx)
+{
+  if (!m_new_state)
+    return false;
+
+  unsigned taint_sm_idx;
+  if (!m_ext_state.get_sm_idx_by_name ("taint", &taint_sm_idx))
+    return false;
+
+  *out_smap = m_new_state->m_checker_states[taint_sm_idx];
+  *out_sm = &m_ext_state.get_sm (taint_sm_idx);
+  *out_sm_idx = taint_sm_idx;
+  return true;
+}
+
 /* struct setjmp_record.  */
 
 int
@@ -1325,8 +1343,11 @@ exploded_node::on_stmt_pre (exploded_graph &eg,
 	  return;
 	}
       else if (is_special_named_call_p (call, "__analyzer_dump_state", 2))
-	state->impl_call_analyzer_dump_state (call, eg.get_ext_state (),
-					      ctxt);
+	{
+	  state->impl_call_analyzer_dump_state (call, eg.get_ext_state (),
+						ctxt);
+	  return;
+	}
       else if (is_setjmp_call_p (call))
 	{
 	  state->m_region_model->on_setjmp (call, this, ctxt);
diff --git a/gcc/analyzer/exploded-graph.h b/gcc/analyzer/exploded-graph.h
index b9c1767..9b18b48 100644
--- a/gcc/analyzer/exploded-graph.h
+++ b/gcc/analyzer/exploded-graph.h
@@ -86,6 +86,9 @@ class impl_region_model_context : public region_model_context
   bool get_malloc_map (sm_state_map **out_smap,
 		       const state_machine **out_sm,
 		       unsigned *out_sm_idx) FINAL OVERRIDE;
+  bool get_taint_map (sm_state_map **out_smap,
+		       const state_machine **out_sm,
+		       unsigned *out_sm_idx) FINAL OVERRIDE;
 
   exploded_graph *m_eg;
   log_user m_logger;
diff --git a/gcc/analyzer/program-state.cc b/gcc/analyzer/program-state.cc
index c1ff0d8..1c87af0 100644
--- a/gcc/analyzer/program-state.cc
+++ b/gcc/analyzer/program-state.cc
@@ -273,6 +273,7 @@ DEBUG_FUNCTION void
 sm_state_map::dump (bool simple) const
 {
   pretty_printer pp;
+  pp_format_decoder (&pp) = default_tree_printer;
   pp_show_color (&pp) = pp_show_color (global_dc->printer);
   pp.buffer->stream = stderr;
   print (NULL, simple, true, &pp);
@@ -419,6 +420,10 @@ sm_state_map::get_state (const svalue *sval,
 	  }
       }
 
+  if (state_machine::state_t state
+      = m_sm.alt_get_inherited_state (*this, sval, ext_state))
+    return state;
+
   return m_sm.get_default_state (sval);
 }
 
@@ -494,6 +499,18 @@ sm_state_map::impl_set_state (const svalue *sval,
 
   gcc_assert (sval->can_have_associated_state_p ());
 
+  if (m_sm.inherited_state_p ())
+    {
+      if (const compound_svalue *compound_sval
+	    = sval->dyn_cast_compound_svalue ())
+	for (auto iter : *compound_sval)
+	  {
+	    const svalue *inner_sval = iter.second;
+	    if (inner_sval->can_have_associated_state_p ())
+	      impl_set_state (inner_sval, state, origin, ext_state);
+	  }
+    }
+
   /* Special-case state 0 as the default value.  */
   if (state == 0)
     {
@@ -1383,6 +1400,10 @@ program_state::impl_call_analyzer_dump_state (const gcall *call,
 
   const svalue *sval = cd.get_arg_svalue (1);
 
+  /* Strip off cast to int (due to variadic args).  */
+  if (const svalue *cast = sval->maybe_undo_cast ())
+    sval = cast;
+
   state_machine::state_t state = smap->get_state (sval, ext_state);
   warning_at (call->location, 0, "state: %qs", state->get_name ());
 }
@@ -1542,7 +1563,8 @@ test_program_state_1 ()
   region_model *model = s.m_region_model;
   const svalue *size_in_bytes
     = mgr->get_or_create_unknown_svalue (size_type_node);
-  const region *new_reg = model->create_region_for_heap_alloc (size_in_bytes);
+  const region *new_reg
+    = model->create_region_for_heap_alloc (size_in_bytes, NULL);
   const svalue *ptr_sval = mgr->get_ptr_svalue (ptr_type_node, new_reg);
   model->set_value (model->get_lvalue (p, NULL),
 		    ptr_sval, NULL);
@@ -1598,7 +1620,8 @@ test_program_state_merging ()
   region_model *model0 = s0.m_region_model;
   const svalue *size_in_bytes
     = mgr->get_or_create_unknown_svalue (size_type_node);
-  const region *new_reg = model0->create_region_for_heap_alloc (size_in_bytes);
+  const region *new_reg
+    = model0->create_region_for_heap_alloc (size_in_bytes, NULL);
   const svalue *ptr_sval = mgr->get_ptr_svalue (ptr_type_node, new_reg);
   model0->set_value (model0->get_lvalue (p, &ctxt),
 		     ptr_sval, &ctxt);
diff --git a/gcc/analyzer/region-model-impl-calls.cc b/gcc/analyzer/region-model-impl-calls.cc
index ff2ae9c..90d4cf9 100644
--- a/gcc/analyzer/region-model-impl-calls.cc
+++ b/gcc/analyzer/region-model-impl-calls.cc
@@ -221,7 +221,7 @@ void
 region_model::impl_call_alloca (const call_details &cd)
 {
   const svalue *size_sval = cd.get_arg_svalue (0);
-  const region *new_reg = create_region_for_alloca (size_sval);
+  const region *new_reg = create_region_for_alloca (size_sval, cd.get_ctxt ());
   const svalue *ptr_sval
     = m_mgr->get_ptr_svalue (cd.get_lhs_type (), new_reg);
   cd.maybe_set_lhs (ptr_sval);
@@ -302,7 +302,8 @@ region_model::impl_call_calloc (const call_details &cd)
   const svalue *prod_sval
     = m_mgr->get_or_create_binop (size_type_node, MULT_EXPR,
 				  nmemb_sval, size_sval);
-  const region *new_reg = create_region_for_heap_alloc (prod_sval);
+  const region *new_reg
+    = create_region_for_heap_alloc (prod_sval, cd.get_ctxt ());
   zero_fill_region (new_reg);
   if (cd.get_lhs_type ())
     {
@@ -408,7 +409,8 @@ void
 region_model::impl_call_malloc (const call_details &cd)
 {
   const svalue *size_sval = cd.get_arg_svalue (0);
-  const region *new_reg = create_region_for_heap_alloc (size_sval);
+  const region *new_reg
+    = create_region_for_heap_alloc (size_sval, cd.get_ctxt ());
   if (cd.get_lhs_type ())
     {
       const svalue *ptr_sval
@@ -471,7 +473,8 @@ void
 region_model::impl_call_operator_new (const call_details &cd)
 {
   const svalue *size_sval = cd.get_arg_svalue (0);
-  const region *new_reg = create_region_for_heap_alloc (size_sval);
+  const region *new_reg
+    = create_region_for_heap_alloc (size_sval, cd.get_ctxt ());
   if (cd.get_lhs_type ())
     {
       const svalue *ptr_sval
@@ -579,7 +582,7 @@ region_model::impl_call_realloc (const call_details &cd)
       const svalue *size_sval = cd.get_arg_svalue (1);
       if (const region *buffer_reg = ptr_sval->maybe_get_region ())
 	if (compat_types_p (size_sval->get_type (), size_type_node))
-	  model->set_dynamic_extents (buffer_reg, size_sval);
+	  model->set_dynamic_extents (buffer_reg, size_sval, ctxt);
       if (cd.get_lhs_region ())
 	{
 	  model->set_value (cd.get_lhs_region (), ptr_sval, cd.get_ctxt ());
@@ -619,7 +622,7 @@ region_model::impl_call_realloc (const call_details &cd)
 
       /* Create the new region.  */
       const region *new_reg
-	= model->create_region_for_heap_alloc (new_size_sval);
+	= model->create_region_for_heap_alloc (new_size_sval, ctxt);
       const svalue *new_ptr_sval
 	= model->m_mgr->get_ptr_svalue (cd.get_lhs_type (), new_reg);
       if (cd.get_lhs_type ())
diff --git a/gcc/analyzer/region-model.cc b/gcc/analyzer/region-model.cc
index a14d107..416a5ac 100644
--- a/gcc/analyzer/region-model.cc
+++ b/gcc/analyzer/region-model.cc
@@ -2301,6 +2301,8 @@ region_model::check_region_access (const region *reg,
   if (!ctxt)
     return;
 
+  check_region_for_taint (reg, dir, ctxt);
+
   switch (dir)
     {
     default:
@@ -2862,6 +2864,17 @@ region_model::get_representative_path_var_1 (const svalue *sval,
 			   parent_pv.m_stack_depth);
     }
 
+  /* Handle binops.  */
+  if (const binop_svalue *binop_sval = sval->dyn_cast_binop_svalue ())
+    if (path_var lhs_pv
+	= get_representative_path_var (binop_sval->get_arg0 (), visited))
+      if (path_var rhs_pv
+	  = get_representative_path_var (binop_sval->get_arg1 (), visited))
+	return path_var (build2 (binop_sval->get_op (),
+				 sval->get_type (),
+				 lhs_pv.m_tree, rhs_pv.m_tree),
+			 lhs_pv.m_stack_depth);
+
   if (pvs.length () < 1)
     return path_var (NULL_TREE, 0);
 
@@ -3720,36 +3733,45 @@ region_model::append_ssa_names_cb (const region *base_reg,
     }
 }
 
-/* Return a new region describing a heap-allocated block of memory.  */
+/* Return a new region describing a heap-allocated block of memory.
+   Use CTXT to complain about tainted sizes.  */
 
 const region *
-region_model::create_region_for_heap_alloc (const svalue *size_in_bytes)
+region_model::create_region_for_heap_alloc (const svalue *size_in_bytes,
+					    region_model_context *ctxt)
 {
   const region *reg = m_mgr->create_region_for_heap_alloc ();
   if (compat_types_p (size_in_bytes->get_type (), size_type_node))
-    set_dynamic_extents (reg, size_in_bytes);
+    set_dynamic_extents (reg, size_in_bytes, ctxt);
   return reg;
 }
 
 /* Return a new region describing a block of memory allocated within the
-   current frame.  */
+   current frame.
+   Use CTXT to complain about tainted sizes.  */
 
 const region *
-region_model::create_region_for_alloca (const svalue *size_in_bytes)
+region_model::create_region_for_alloca (const svalue *size_in_bytes,
+					region_model_context *ctxt)
 {
   const region *reg = m_mgr->create_region_for_alloca (m_current_frame);
   if (compat_types_p (size_in_bytes->get_type (), size_type_node))
-    set_dynamic_extents (reg, size_in_bytes);
+    set_dynamic_extents (reg, size_in_bytes, ctxt);
   return reg;
 }
 
-/* Record that the size of REG is SIZE_IN_BYTES.  */
+/* Record that the size of REG is SIZE_IN_BYTES.
+   Use CTXT to complain about tainted sizes.  */
 
 void
 region_model::set_dynamic_extents (const region *reg,
-				   const svalue *size_in_bytes)
+				   const svalue *size_in_bytes,
+				   region_model_context *ctxt)
 {
   assert_compat_types (size_in_bytes->get_type (), size_type_node);
+  if (ctxt)
+    check_dynamic_size_for_taint (reg->get_memory_space (), size_in_bytes,
+				  ctxt);
   m_dynamic_extents.put (reg, size_in_bytes);
 }
 
@@ -5096,7 +5118,8 @@ test_state_merging ()
     region_model model0 (&mgr);
     tree size = build_int_cst (size_type_node, 1024);
     const svalue *size_sval = mgr.get_or_create_constant_svalue (size);
-    const region *new_reg = model0.create_region_for_heap_alloc (size_sval);
+    const region *new_reg
+      = model0.create_region_for_heap_alloc (size_sval, &ctxt);
     const svalue *ptr_sval = mgr.get_ptr_svalue (ptr_type_node, new_reg);
     model0.set_value (model0.get_lvalue (p, &ctxt),
 		      ptr_sval, &ctxt);
@@ -5484,7 +5507,7 @@ test_malloc_constraints ()
 
   const svalue *size_in_bytes
     = mgr.get_or_create_unknown_svalue (size_type_node);
-  const region *reg = model.create_region_for_heap_alloc (size_in_bytes);
+  const region *reg = model.create_region_for_heap_alloc (size_in_bytes, NULL);
   const svalue *sval = mgr.get_ptr_svalue (ptr_type_node, reg);
   model.set_value (model.get_lvalue (p, NULL), sval, NULL);
   model.set_value (q, p, NULL);
@@ -5705,7 +5728,7 @@ test_malloc ()
 
   /* "p = malloc (n * 4);".  */
   const svalue *size_sval = model.get_rvalue (n_times_4, &ctxt);
-  const region *reg = model.create_region_for_heap_alloc (size_sval);
+  const region *reg = model.create_region_for_heap_alloc (size_sval, &ctxt);
   const svalue *ptr = mgr.get_ptr_svalue (int_star, reg);
   model.set_value (model.get_lvalue (p, &ctxt), ptr, &ctxt);
   ASSERT_EQ (model.get_capacity (reg), size_sval);
@@ -5739,7 +5762,7 @@ test_alloca ()
 			NULL, &ctxt);
   /* "p = alloca (n * 4);".  */
   const svalue *size_sval = model.get_rvalue (n_times_4, &ctxt);
-  const region *reg = model.create_region_for_alloca (size_sval);
+  const region *reg = model.create_region_for_alloca (size_sval, &ctxt);
   ASSERT_EQ (reg->get_parent_region (), frame_reg);
   const svalue *ptr = mgr.get_ptr_svalue (int_star, reg);
   model.set_value (model.get_lvalue (p, &ctxt), ptr, &ctxt);
diff --git a/gcc/analyzer/region-model.h b/gcc/analyzer/region-model.h
index 5fabf78..13e8109 100644
--- a/gcc/analyzer/region-model.h
+++ b/gcc/analyzer/region-model.h
@@ -676,8 +676,10 @@ class region_model
 		       region_model_context *ctxt,
 		       rejected_constraint **out);
 
-  const region *create_region_for_heap_alloc (const svalue *size_in_bytes);
-  const region *create_region_for_alloca (const svalue *size_in_bytes);
+  const region *create_region_for_heap_alloc (const svalue *size_in_bytes,
+					      region_model_context *ctxt);
+  const region *create_region_for_alloca (const svalue *size_in_bytes,
+					  region_model_context *ctxt);
 
   tree get_representative_tree (const svalue *sval) const;
   path_var
@@ -703,7 +705,8 @@ class region_model
   }
   const svalue *get_dynamic_extents (const region *reg) const;
   void set_dynamic_extents (const region *reg,
-			    const svalue *size_in_bytes);
+			    const svalue *size_in_bytes,
+			    region_model_context *ctxt);
   void unset_dynamic_extents (const region *reg);
 
   region_model_manager *get_manager () const { return m_mgr; }
@@ -792,6 +795,14 @@ class region_model
 				  tree expr,
 				  region_model_context *ctxt) const;
 
+  void check_dynamic_size_for_taint (enum memory_space mem_space,
+				     const svalue *size_in_bytes,
+				     region_model_context *ctxt) const;
+
+  void check_region_for_taint (const region *reg,
+			       enum access_direction dir,
+			       region_model_context *ctxt) const;
+
   void check_for_writable_region (const region* dest_reg,
 				  region_model_context *ctxt) const;
   void check_region_access (const region *reg,
@@ -891,6 +902,10 @@ class region_model_context
   virtual bool get_malloc_map (sm_state_map **out_smap,
 			       const state_machine **out_sm,
 			       unsigned *out_sm_idx) = 0;
+  /* Likewise for the "taint" state machine.  */
+  virtual bool get_taint_map (sm_state_map **out_smap,
+			      const state_machine **out_sm,
+			      unsigned *out_sm_idx) = 0;
 };
 
 /* A "do nothing" subclass of region_model_context.  */
@@ -935,6 +950,12 @@ public:
   {
     return false;
   }
+  bool get_taint_map (sm_state_map **,
+		      const state_machine **,
+		      unsigned *) OVERRIDE
+  {
+    return false;
+  }
 };
 
 /* A subclass of region_model_context for determining if operations fail
diff --git a/gcc/analyzer/sm-taint.cc b/gcc/analyzer/sm-taint.cc
index 721d3ea..0a51a1f 100644
--- a/gcc/analyzer/sm-taint.cc
+++ b/gcc/analyzer/sm-taint.cc
@@ -33,9 +33,21 @@ along with GCC; see the file COPYING3.  If not see
 #include "function.h"
 #include "json.h"
 #include "analyzer/analyzer.h"
-#include "diagnostic-event-id.h"
 #include "analyzer/analyzer-logging.h"
+#include "gimple-iterator.h"
+#include "tristate.h"
+#include "selftest.h"
+#include "ordered-hash-map.h"
+#include "cgraph.h"
+#include "cfg.h"
+#include "digraph.h"
+#include "analyzer/supergraph.h"
+#include "analyzer/call-string.h"
+#include "analyzer/program-point.h"
+#include "analyzer/store.h"
+#include "analyzer/region-model.h"
 #include "analyzer/sm.h"
+#include "analyzer/program-state.h"
 #include "analyzer/pending-diagnostic.h"
 
 #if ENABLE_ANALYZER
@@ -44,6 +56,20 @@ namespace ana {
 
 namespace {
 
+/* An enum for describing tainted values.  */
+
+enum bounds
+{
+  /* This tainted value has no upper or lower bound.  */
+  BOUNDS_NONE,
+
+  /* This tainted value has an upper bound but not lower bound.  */
+  BOUNDS_UPPER,
+
+  /* This tainted value has a lower bound but no upper bound.  */
+  BOUNDS_LOWER
+};
+
 /* An experimental state machine, for tracking "taint": unsanitized uses
    of data potentially under an attacker's control.  */
 
@@ -54,6 +80,11 @@ public:
 
   bool inherited_state_p () const FINAL OVERRIDE { return true; }
 
+  state_t alt_get_inherited_state (const sm_state_map &map,
+				   const svalue *sval,
+				   const extrinsic_state &ext_state)
+    const FINAL OVERRIDE;
+
   bool on_stmt (sm_context *sm_ctxt,
 		const supernode *node,
 		const gimple *stmt) const FINAL OVERRIDE;
@@ -67,6 +98,10 @@ public:
 
   bool can_purge_p (state_t s) const FINAL OVERRIDE;
 
+  bool get_taint (state_t s, tree type, enum bounds *out) const;
+
+  state_t combine_states (state_t s0, state_t s1) const;
+
   /* State for a "tainted" value: unsanitized data potentially under an
      attacker's control.  */
   state_t m_tainted;
@@ -81,31 +116,65 @@ public:
   state_t m_stop;
 };
 
-enum bounds
+/* Class for diagnostics relating to taint_state_machine.  */
+
+class taint_diagnostic : public pending_diagnostic
 {
-  BOUNDS_NONE,
-  BOUNDS_UPPER,
-  BOUNDS_LOWER
+public:
+  taint_diagnostic (const taint_state_machine &sm, tree arg,
+		    enum bounds has_bounds)
+  : m_sm (sm), m_arg (arg), m_has_bounds (has_bounds)
+  {}
+
+  bool subclass_equal_p (const pending_diagnostic &base_other) const OVERRIDE
+  {
+    return same_tree_p (m_arg, ((const taint_diagnostic &)base_other).m_arg);
+  }
+
+  label_text describe_state_change (const evdesc::state_change &change)
+    FINAL OVERRIDE
+  {
+    if (change.m_new_state == m_sm.m_tainted)
+      {
+	if (change.m_origin)
+	  return change.formatted_print ("%qE has an unchecked value here"
+					 " (from %qE)",
+					 change.m_expr, change.m_origin);
+	else
+	  return change.formatted_print ("%qE gets an unchecked value here",
+					 change.m_expr);
+      }
+    else if (change.m_new_state == m_sm.m_has_lb)
+      return change.formatted_print ("%qE has its lower bound checked here",
+				     change.m_expr);
+    else if (change.m_new_state == m_sm.m_has_ub)
+      return change.formatted_print ("%qE has its upper bound checked here",
+				     change.m_expr);
+    return label_text ();
+  }
+protected:
+  const taint_state_machine &m_sm;
+  tree m_arg;
+  enum bounds m_has_bounds;
 };
 
-class tainted_array_index
-  : public pending_diagnostic_subclass<tainted_array_index>
+/* Concrete taint_diagnostic subclass for reporting attacker-controlled
+   array index.  */
+
+class tainted_array_index : public taint_diagnostic
 {
 public:
   tainted_array_index (const taint_state_machine &sm, tree arg,
 		       enum bounds has_bounds)
-  : m_sm (sm), m_arg (arg), m_has_bounds (has_bounds) {}
+  : taint_diagnostic (sm, arg, has_bounds)
+  {}
 
   const char *get_kind () const FINAL OVERRIDE { return "tainted_array_index"; }
 
-  bool operator== (const tainted_array_index &other) const
-  {
-    return same_tree_p (m_arg, other.m_arg);
-  }
-
   bool emit (rich_location *rich_loc) FINAL OVERRIDE
   {
     diagnostic_metadata m;
+    /* CWE-129: "Improper Validation of Array Index".  */
     m.add_cwe (129);
     switch (m_has_bounds)
       {
@@ -113,45 +182,197 @@ public:
 	gcc_unreachable ();
       case BOUNDS_NONE:
 	return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_array_index,
-			     "use of tainted value %qE in array lookup"
-			     " without bounds checking",
+			     "use of attacker-controlled value %qE"
+			     " in array lookup without bounds checking",
 			     m_arg);
 	break;
       case BOUNDS_UPPER:
 	return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_array_index,
-			     "use of tainted value %qE in array lookup"
-			     " without lower-bounds checking",
+			     "use of attacker-controlled value %qE"
+			     " in array lookup without checking for negative",
 			     m_arg);
 	break;
       case BOUNDS_LOWER:
 	return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_array_index,
-			     "use of tainted value %qE in array lookup"
-			     " without upper-bounds checking",
+			     "use of attacker-controlled value %qE"
+			     " in array lookup without upper-bounds checking",
 			     m_arg);
 	break;
       }
   }
 
-  label_text describe_state_change (const evdesc::state_change &change)
-    FINAL OVERRIDE
+  label_text describe_final_event (const evdesc::final_event &ev) FINAL OVERRIDE
   {
-    if (change.m_new_state == m_sm.m_tainted)
+    switch (m_has_bounds)
       {
-	if (change.m_origin)
-	  return change.formatted_print ("%qE has an unchecked value here"
-					 " (from %qE)",
-					 change.m_expr, change.m_origin);
-	else
-	  return change.formatted_print ("%qE gets an unchecked value here",
-					 change.m_expr);
+      default:
+	gcc_unreachable ();
+      case BOUNDS_NONE:
+	return ev.formatted_print
+	  ("use of attacker-controlled value %qE in array lookup"
+	   " without bounds checking",
+	   m_arg);
+      case BOUNDS_UPPER:
+	return ev.formatted_print
+	  ("use of attacker-controlled value %qE"
+	   " in array lookup without checking for negative",
+	   m_arg);
+      case BOUNDS_LOWER:
+	return ev.formatted_print
+	  ("use of attacker-controlled value %qE"
+	   " in array lookup without upper-bounds checking",
+	   m_arg);
+      }
+  }
+};
+
+/* Concrete taint_diagnostic subclass for reporting attacker-controlled
+   pointer offset.  */
+
+class tainted_offset : public taint_diagnostic
+{
+public:
+  tainted_offset (const taint_state_machine &sm, tree arg,
+		       enum bounds has_bounds)
+  : taint_diagnostic (sm, arg, has_bounds)
+  {}
+
+  const char *get_kind () const FINAL OVERRIDE { return "tainted_offset"; }
+
+  bool emit (rich_location *rich_loc) FINAL OVERRIDE
+  {
+    diagnostic_metadata m;
+    /* CWE-823: "Use of Out-of-range Pointer Offset".  */
+    m.add_cwe (823);
+    if (m_arg)
+      switch (m_has_bounds)
+	{
+	default:
+	  gcc_unreachable ();
+	case BOUNDS_NONE:
+	  return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_offset,
+			       "use of attacker-controlled value %qE as offset"
+			       " without bounds checking",
+			       m_arg);
+	  break;
+	case BOUNDS_UPPER:
+	  return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_offset,
+			       "use of attacker-controlled value %qE as offset"
+			       " without lower-bounds checking",
+			       m_arg);
+	  break;
+	case BOUNDS_LOWER:
+	  return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_offset,
+			       "use of attacker-controlled value %qE as offset"
+			       " without upper-bounds checking",
+			       m_arg);
+	  break;
+	}
+    else
+      switch (m_has_bounds)
+	{
+	default:
+	  gcc_unreachable ();
+	case BOUNDS_NONE:
+	  return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_offset,
+			       "use of attacker-controlled value as offset"
+			       " without bounds checking");
+	  break;
+	case BOUNDS_UPPER:
+	  return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_offset,
+			       "use of attacker-controlled value as offset"
+			       " without lower-bounds checking");
+	  break;
+	case BOUNDS_LOWER:
+	  return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_offset,
+			       "use of attacker-controlled value as offset"
+			       " without upper-bounds checking");
+	  break;
+	}
+  }
+
+  label_text describe_final_event (const evdesc::final_event &ev) FINAL OVERRIDE
+  {
+    if (m_arg)
+      switch (m_has_bounds)
+	{
+	default:
+	  gcc_unreachable ();
+	case BOUNDS_NONE:
+	  return ev.formatted_print ("use of attacker-controlled value %qE"
+				     " as offset without bounds checking",
+				     m_arg);
+	case BOUNDS_UPPER:
+	  return ev.formatted_print ("use of attacker-controlled value %qE"
+				     " as offset without lower-bounds checking",
+				     m_arg);
+	case BOUNDS_LOWER:
+	  return ev.formatted_print ("use of attacker-controlled value %qE"
+				     " as offset without upper-bounds checking",
+				     m_arg);
+	}
+    else
+      switch (m_has_bounds)
+	{
+	default:
+	  gcc_unreachable ();
+	case BOUNDS_NONE:
+	  return ev.formatted_print ("use of attacker-controlled value"
+				     " as offset without bounds checking");
+	case BOUNDS_UPPER:
+	  return ev.formatted_print ("use of attacker-controlled value"
+				     " as offset without lower-bounds"
+				     " checking");
+	case BOUNDS_LOWER:
+	  return ev.formatted_print ("use of attacker-controlled value"
+				     " as offset without upper-bounds"
+				     " checking");
+	}
+  }
+};
+
+/* Concrete taint_diagnostic subclass for reporting attacker-controlled
+   size.  */
+
+class tainted_size : public taint_diagnostic
+{
+public:
+  tainted_size (const taint_state_machine &sm, tree arg,
+		enum bounds has_bounds,
+		enum access_direction dir)
+  : taint_diagnostic (sm, arg, has_bounds),
+    m_dir (dir)
+  {}
+
+  const char *get_kind () const FINAL OVERRIDE { return "tainted_size"; }
+
+  bool emit (rich_location *rich_loc) FINAL OVERRIDE
+  {
+    diagnostic_metadata m;
+    m.add_cwe (129);
+    switch (m_has_bounds)
+      {
+      default:
+	gcc_unreachable ();
+      case BOUNDS_NONE:
+	return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_size,
+			     "use of attacker-controlled value %qE as size"
+			     " without bounds checking",
+			     m_arg);
+	break;
+      case BOUNDS_UPPER:
+	return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_size,
+			     "use of attacker-controlled value %qE as size"
+			     " without lower-bounds checking",
+			     m_arg);
+	break;
+      case BOUNDS_LOWER:
+	return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_size,
+			     "use of attacker-controlled value %qE as size"
+			     " without upper-bounds checking",
+			     m_arg);
+	break;
       }
-    else if (change.m_new_state == m_sm.m_has_lb)
-      return change.formatted_print ("%qE has its lower bound checked here",
-				     change.m_expr);
-    else if (change.m_new_state == m_sm.m_has_ub)
-      return change.formatted_print ("%qE has its upper bound checked here",
-				     change.m_expr);
-    return label_text ();
   }
 
   label_text describe_final_event (const evdesc::final_event &ev) FINAL OVERRIDE
@@ -161,24 +382,194 @@ public:
       default:
 	gcc_unreachable ();
       case BOUNDS_NONE:
-	return ev.formatted_print ("use of tainted value %qE in array lookup"
-				   " without bounds checking",
+	return ev.formatted_print ("use of attacker-controlled value %qE"
+				   " as size without bounds checking",
 				   m_arg);
       case BOUNDS_UPPER:
-	return ev.formatted_print ("use of tainted value %qE in array lookup"
-				   " without lower-bounds checking",
+	return ev.formatted_print ("use of attacker-controlled value %qE"
+				   " as size without lower-bounds checking",
 				   m_arg);
       case BOUNDS_LOWER:
-	return ev.formatted_print ("use of tainted value %qE in array lookup"
-				   " without upper-bounds checking",
+	return ev.formatted_print ("use of attacker-controlled value %qE"
+				   " as size without upper-bounds checking",
 				   m_arg);
       }
   }
 
 private:
-  const taint_state_machine &m_sm;
-  tree m_arg;
-  enum bounds m_has_bounds;
+  enum access_direction m_dir;
+};
+
+/* Concrete taint_diagnostic subclass for reporting attacker-controlled
+   divisor (so that an attacker can trigger a divide by zero).  */
+
+class tainted_divisor : public taint_diagnostic
+{
+public:
+  tainted_divisor (const taint_state_machine &sm, tree arg,
+		   enum bounds has_bounds)
+  : taint_diagnostic (sm, arg, has_bounds)
+  {}
+
+  const char *get_kind () const FINAL OVERRIDE { return "tainted_divisor"; }
+
+  bool emit (rich_location *rich_loc) FINAL OVERRIDE
+  {
+    diagnostic_metadata m;
+    /* CWE-369: "Divide By Zero".  */
+    m.add_cwe (369);
+    if (m_arg)
+      return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_divisor,
+			   "use of attacker-controlled value %qE as divisor"
+			   " without checking for zero",
+			   m_arg);
+    else
+      return warning_meta (rich_loc, m, OPT_Wanalyzer_tainted_divisor,
+			   "use of attacker-controlled value as divisor"
+			   " without checking for zero");
+  }
+
+  label_text describe_final_event (const evdesc::final_event &ev) FINAL OVERRIDE
+  {
+    if (m_arg)
+      return ev.formatted_print
+	("use of attacker-controlled value %qE as divisor"
+	 " without checking for zero",
+	 m_arg);
+    else
+      return ev.formatted_print
+	("use of attacker-controlled value as divisor"
+	 " without checking for zero");
+  }
+};
+
+/* Concrete taint_diagnostic subclass for reporting attacker-controlled
+   size of a dynamic allocation.  */
+
+class tainted_allocation_size : public taint_diagnostic
+{
+public:
+  tainted_allocation_size (const taint_state_machine &sm, tree arg,
+			   enum bounds has_bounds, enum memory_space mem_space)
+  : taint_diagnostic (sm, arg, has_bounds),
+    m_mem_space (mem_space)
+  {
+    gcc_assert (mem_space == MEMSPACE_STACK || mem_space == MEMSPACE_HEAP);
+  }
+
+  const char *get_kind () const FINAL OVERRIDE
+  {
+    return "tainted_allocation_size";
+  }
+
+  bool emit (rich_location *rich_loc) FINAL OVERRIDE
+  {
+    diagnostic_metadata m;
+    /* "CWE-789: Memory Allocation with Excessive Size Value".  */
+    m.add_cwe (789);
+    gcc_assert (m_mem_space == MEMSPACE_STACK || m_mem_space == MEMSPACE_HEAP);
+    // TODO: make use of m_mem_space
+    if (m_arg)
+      switch (m_has_bounds)
+	{
+	default:
+	  gcc_unreachable ();
+	case BOUNDS_NONE:
+	  return warning_meta (rich_loc, m,
+			       OPT_Wanalyzer_tainted_allocation_size,
+			       "use of attacker-controlled value %qE as"
+			       " allocation size without bounds checking",
+			       m_arg);
+	  break;
+	case BOUNDS_UPPER:
+	  return warning_meta (rich_loc, m,
+			       OPT_Wanalyzer_tainted_allocation_size,
+			       "use of attacker-controlled value %qE as"
+			       " allocation size without lower-bounds checking",
+			       m_arg);
+	  break;
+	case BOUNDS_LOWER:
+	  return warning_meta (rich_loc, m,
+			       OPT_Wanalyzer_tainted_allocation_size,
+			       "use of attacker-controlled value %qE as"
+			       " allocation size without upper-bounds checking",
+			     m_arg);
+	  break;
+	}
+    else
+      switch (m_has_bounds)
+	{
+	default:
+	  gcc_unreachable ();
+	case BOUNDS_NONE:
+	  return warning_meta (rich_loc, m,
+			       OPT_Wanalyzer_tainted_allocation_size,
+			       "use of attacker-controlled value as"
+			       " allocation size without bounds"
+			       " checking");
+	  break;
+	case BOUNDS_UPPER:
+	  return warning_meta (rich_loc, m,
+			       OPT_Wanalyzer_tainted_allocation_size,
+			       "use of attacker-controlled value as"
+			       " allocation size without lower-bounds"
+			       " checking");
+	  break;
+	case BOUNDS_LOWER:
+	  return warning_meta (rich_loc, m,
+			       OPT_Wanalyzer_tainted_allocation_size,
+			       "use of attacker-controlled value as"
+			       " allocation size without upper-bounds"
+			       " checking");
+	  break;
+	}
+  }
+
+  label_text describe_final_event (const evdesc::final_event &ev) FINAL OVERRIDE
+  {
+    if (m_arg)
+      switch (m_has_bounds)
+	{
+	default:
+	  gcc_unreachable ();
+	case BOUNDS_NONE:
+	  return ev.formatted_print
+	    ("use of attacker-controlled value %qE as allocation size"
+	     " without bounds checking",
+	     m_arg);
+	case BOUNDS_UPPER:
+	  return ev.formatted_print
+	    ("use of attacker-controlled value %qE as allocation size"
+	     " without lower-bounds checking",
+	     m_arg);
+	case BOUNDS_LOWER:
+	  return ev.formatted_print
+	    ("use of attacker-controlled value %qE as allocation size"
+	     " without upper-bounds checking",
+	     m_arg);
+	}
+    else
+      switch (m_has_bounds)
+	{
+	default:
+	  gcc_unreachable ();
+	case BOUNDS_NONE:
+	  return ev.formatted_print
+	    ("use of attacker-controlled value as allocation size"
+	     " without bounds checking");
+	case BOUNDS_UPPER:
+	  return ev.formatted_print
+	    ("use of attacker-controlled value as allocation size"
+	     " without lower-bounds checking");
+	case BOUNDS_LOWER:
+	  return ev.formatted_print
+	    ("use of attacker-controlled value as allocation size"
+	     " without upper-bounds checking");
+	}
+  }
+
+private:
+  enum memory_space m_mem_space;
 };
 
 /* taint_state_machine's ctor.  */
@@ -192,6 +583,79 @@ taint_state_machine::taint_state_machine (logger *logger)
   m_stop = add_state ("stop");
 }
 
+state_machine::state_t
+taint_state_machine::alt_get_inherited_state (const sm_state_map &map,
+					      const svalue *sval,
+					      const extrinsic_state &ext_state)
+  const
+{
+  switch (sval->get_kind ())
+    {
+    default:
+      break;
+    case SK_UNARYOP:
+      {
+	const unaryop_svalue *unaryop_sval
+	  = as_a <const unaryop_svalue *> (sval);
+	enum tree_code op = unaryop_sval->get_op ();
+	const svalue *arg = unaryop_sval->get_arg ();
+	switch (op)
+	  {
+	  case NOP_EXPR:
+	    {
+	      state_t arg_state = map.get_state (arg, ext_state);
+	      return arg_state;
+	    }
+	  default:
+	    gcc_unreachable ();
+	    break;
+	  }
+      }
+      break;
+    case SK_BINOP:
+      {
+	const binop_svalue *binop_sval = as_a <const binop_svalue *> (sval);
+	enum tree_code op = binop_sval->get_op ();
+	const svalue *arg0 = binop_sval->get_arg0 ();
+	const svalue *arg1 = binop_sval->get_arg1 ();
+	switch (op)
+	  {
+	  default:
+	    break;
+	  case PLUS_EXPR:
+	  case MINUS_EXPR:
+	  case MULT_EXPR:
+	  case POINTER_PLUS_EXPR:
+	  case TRUNC_DIV_EXPR:
+	  case TRUNC_MOD_EXPR:
+	    {
+	      state_t arg0_state = map.get_state (arg0, ext_state);
+	      state_t arg1_state = map.get_state (arg1, ext_state);
+	      return combine_states (arg0_state, arg1_state);
+	    }
+	    break;
+
+	  case EQ_EXPR:
+	  case GE_EXPR:
+	  case LE_EXPR:
+	  case NE_EXPR:
+	  case GT_EXPR:
+	  case LT_EXPR:
+	  case UNORDERED_EXPR:
+	  case ORDERED_EXPR:
+	    /* Comparisons are just booleans.  */
+	    return m_start;
+
+	  case BIT_AND_EXPR:
+	  case RSHIFT_EXPR:
+	    return NULL;
+	  }
+      }
+      break;
+    }
+  return NULL;
+}
+
 /* Implementation of state_machine::on_stmt vfunc for taint_state_machine.  */
 
 bool
@@ -220,53 +684,35 @@ taint_state_machine::on_stmt (sm_context *sm_ctxt,
 
   if (const gassign *assign = dyn_cast <const gassign *> (stmt))
     {
-      tree rhs1 = gimple_assign_rhs1 (assign);
       enum tree_code op = gimple_assign_rhs_code (assign);
 
-      /* Check array accesses.  */
-      if (op == ARRAY_REF)
+      switch (op)
 	{
-	  tree arg = TREE_OPERAND (rhs1, 1);
-
-	  /* Unsigned types have an implicit lower bound.  */
-	  bool is_unsigned = false;
-	  if (INTEGRAL_TYPE_P (TREE_TYPE (arg)))
-	    is_unsigned = TYPE_UNSIGNED (TREE_TYPE (arg));
-
-	  state_t state = sm_ctxt->get_state (stmt, arg);
-	  /* Can't use a switch as the states are non-const.  */
-	  if (state == m_tainted)
-	    {
-	      /* Complain about missing bounds.  */
-	      tree diag_arg = sm_ctxt->get_diagnostic_tree (arg);
-	      pending_diagnostic *d
-		= new tainted_array_index (*this, diag_arg,
-					   is_unsigned
-					   ? BOUNDS_LOWER : BOUNDS_NONE);
-	      sm_ctxt->warn (node, stmt, arg, d);
-	      sm_ctxt->set_next_state (stmt, arg, m_stop);
-	    }
-	  else if (state == m_has_lb)
-	    {
-	      /* Complain about missing upper bound.  */
-	      tree diag_arg = sm_ctxt->get_diagnostic_tree (arg);
-	      sm_ctxt->warn (node, stmt, arg,
-			      new tainted_array_index (*this, diag_arg,
-						       BOUNDS_LOWER));
-	      sm_ctxt->set_next_state (stmt, arg, m_stop);
-	    }
-	  else if (state == m_has_ub)
-	    {
-	      /* Complain about missing lower bound.  */
-	      if (!is_unsigned)
-		{
-		  tree diag_arg = sm_ctxt->get_diagnostic_tree (arg);
-		  sm_ctxt->warn  (node, stmt, arg,
-				  new tainted_array_index (*this, diag_arg,
-							   BOUNDS_UPPER));
-		  sm_ctxt->set_next_state (stmt, arg, m_stop);
-		}
-	    }
+	default:
+	  break;
+	case TRUNC_DIV_EXPR:
+	case CEIL_DIV_EXPR:
+	case FLOOR_DIV_EXPR:
+	case ROUND_DIV_EXPR:
+	case TRUNC_MOD_EXPR:
+	case CEIL_MOD_EXPR:
+	case FLOOR_MOD_EXPR:
+	case ROUND_MOD_EXPR:
+	case RDIV_EXPR:
+	case EXACT_DIV_EXPR:
+	  {
+	    tree divisor = gimple_assign_rhs2 (assign);;
+	    state_t state = sm_ctxt->get_state (stmt, divisor);
+	    enum bounds b;
+	    if (get_taint (state, TREE_TYPE (divisor), &b))
+	      {
+		tree diag_divisor = sm_ctxt->get_diagnostic_tree (divisor);
+		sm_ctxt->warn  (node, stmt, divisor,
+				new tainted_divisor (*this, diag_divisor, b));
+		sm_ctxt->set_next_state (stmt, divisor, m_stop);
+	      }
+	  }
+	  break;
 	}
     }
 
@@ -324,6 +770,62 @@ taint_state_machine::can_purge_p (state_t s ATTRIBUTE_UNUSED) const
   return true;
 }
 
+/* If STATE is a tainted state, write the bounds to *OUT and return true.
+   Otherwise return false.
+   Use the signedness of TYPE to determine if "has_ub" is tainted.  */
+
+bool
+taint_state_machine::get_taint (state_t state, tree type,
+				enum bounds *out) const
+{
+  /* Unsigned types have an implicit lower bound.  */
+  bool is_unsigned = false;
+  if (type)
+    if (INTEGRAL_TYPE_P (type))
+      is_unsigned = TYPE_UNSIGNED (type);
+
+  /* Can't use a switch as the states are non-const.  */
+  if (state == m_tainted)
+    {
+      *out = is_unsigned ? BOUNDS_LOWER : BOUNDS_NONE;
+      return true;
+    }
+  else if (state == m_has_lb)
+    {
+      *out = BOUNDS_LOWER;
+      return true;
+    }
+  else if (state == m_has_ub && !is_unsigned)
+    {
+      /* Missing lower bound.  */
+      *out = BOUNDS_UPPER;
+      return true;
+    }
+  return false;
+}
+
+/* Find the most tainted state of S0 and S1.  */
+
+state_machine::state_t
+taint_state_machine::combine_states (state_t s0, state_t s1) const
+{
+  gcc_assert (s0);
+  gcc_assert (s1);
+  if (s0 == s1)
+    return s0;
+  if (s0 == m_tainted || s1 == m_tainted)
+    return m_tainted;
+  if (s0 == m_stop)
+    return s1;
+  if (s1 == m_stop)
+    return s0;
+  if (s0 == m_start)
+    return s1;
+  if (s1 == m_start)
+    return s0;
+  gcc_unreachable ();
+}
+
 } // anonymous namespace
 
 /* Internal interface to this file. */
@@ -334,6 +836,152 @@ make_taint_state_machine (logger *logger)
   return new taint_state_machine (logger);
 }
 
+/* Complain to CTXT if accessing REG leads could lead to arbitrary
+   memory access under an attacker's control (due to taint).  */
+
+void
+region_model::check_region_for_taint (const region *reg,
+				      enum access_direction dir,
+				      region_model_context *ctxt) const
+{
+  gcc_assert (reg);
+  gcc_assert (ctxt);
+
+  LOG_SCOPE (ctxt->get_logger ());
+
+  sm_state_map *smap;
+  const state_machine *sm;
+  unsigned sm_idx;
+  if (!ctxt->get_taint_map (&smap, &sm, &sm_idx))
+    return;
+
+  gcc_assert (smap);
+  gcc_assert (sm);
+
+  const taint_state_machine &taint_sm = (const taint_state_machine &)*sm;
+
+  const extrinsic_state *ext_state = ctxt->get_ext_state ();
+  if (!ext_state)
+    return;
+
+  const region *iter_region = reg;
+  while (iter_region)
+    {
+      switch (iter_region->get_kind ())
+	{
+	default:
+	  break;
+
+	case RK_ELEMENT:
+	  {
+	    const element_region *element_reg
+	      = (const element_region *)iter_region;
+	    const svalue *index = element_reg->get_index ();
+	    const state_machine::state_t
+	      state = smap->get_state (index, *ext_state);
+	    gcc_assert (state);
+	    enum bounds b;
+	    if (taint_sm.get_taint (state, index->get_type (), &b))
+	    {
+	      tree arg = get_representative_tree (index);
+	      ctxt->warn (new tainted_array_index (taint_sm, arg, b));
+	    }
+	  }
+	  break;
+
+	case RK_OFFSET:
+	  {
+	    const offset_region *offset_reg
+	      = (const offset_region *)iter_region;
+	    const svalue *offset = offset_reg->get_byte_offset ();
+	    const state_machine::state_t
+	      state = smap->get_state (offset, *ext_state);
+	    gcc_assert (state);
+	    /* Handle implicit cast to sizetype.  */
+	    tree effective_type = offset->get_type ();
+	    if (const svalue *cast = offset->maybe_undo_cast ())
+	      if (cast->get_type ())
+		effective_type = cast->get_type ();
+	    enum bounds b;
+	    if (taint_sm.get_taint (state, effective_type, &b))
+	      {
+		tree arg = get_representative_tree (offset);
+		ctxt->warn (new tainted_offset (taint_sm, arg, b));
+	      }
+	  }
+	  break;
+
+	case RK_CAST:
+	  {
+	    const cast_region *cast_reg
+	      = as_a <const cast_region *> (iter_region);
+	    iter_region = cast_reg->get_original_region ();
+	    continue;
+	  }
+
+	case RK_SIZED:
+	  {
+	    const sized_region *sized_reg
+	      = (const sized_region *)iter_region;
+	    const svalue *size_sval = sized_reg->get_byte_size_sval (m_mgr);
+	    const state_machine::state_t
+	      state = smap->get_state (size_sval, *ext_state);
+	    gcc_assert (state);
+	    enum bounds b;
+	    if (taint_sm.get_taint (state, size_sval->get_type (), &b))
+	      {
+		tree arg = get_representative_tree (size_sval);
+		ctxt->warn (new tainted_size (taint_sm, arg, b, dir));
+	      }
+	  }
+	  break;
+	}
+
+      iter_region = iter_region->get_parent_region ();
+    }
+}
+
+/* Complain to CTXT about a tainted allocation size if SIZE_IN_BYTES is
+   under an attacker's control (due to taint), where the allocation
+   is happening within MEM_SPACE.  */
+
+void
+region_model::check_dynamic_size_for_taint (enum memory_space mem_space,
+					    const svalue *size_in_bytes,
+					    region_model_context *ctxt) const
+{
+  gcc_assert (mem_space == MEMSPACE_STACK || mem_space == MEMSPACE_HEAP);
+  gcc_assert (size_in_bytes);
+  gcc_assert (ctxt);
+
+  LOG_SCOPE (ctxt->get_logger ());
+
+  sm_state_map *smap;
+  const state_machine *sm;
+  unsigned sm_idx;
+  if (!ctxt->get_taint_map (&smap, &sm, &sm_idx))
+    return;
+
+  gcc_assert (smap);
+  gcc_assert (sm);
+
+  const taint_state_machine &taint_sm = (const taint_state_machine &)*sm;
+
+  const extrinsic_state *ext_state = ctxt->get_ext_state ();
+  if (!ext_state)
+    return;
+
+  const state_machine::state_t
+    state = smap->get_state (size_in_bytes, *ext_state);
+  gcc_assert (state);
+  enum bounds b;
+  if (taint_sm.get_taint (state, size_in_bytes->get_type (), &b))
+    {
+      tree arg = get_representative_tree (size_in_bytes);
+      ctxt->warn (new tainted_allocation_size (taint_sm, arg, b, mem_space));
+    }
+}
+
 } // namespace ana
 
 #endif /* #if ENABLE_ANALYZER */
diff --git a/gcc/analyzer/sm.h b/gcc/analyzer/sm.h
index 02faffb..b8570e8 100644
--- a/gcc/analyzer/sm.h
+++ b/gcc/analyzer/sm.h
@@ -69,6 +69,15 @@ public:
      within a heap-allocated struct.  */
   virtual bool inherited_state_p () const = 0;
 
+  /* A vfunc for more general handling of inheritance.  */
+  virtual state_t
+  alt_get_inherited_state (const sm_state_map &,
+			   const svalue *,
+			   const extrinsic_state &) const
+  {
+    return NULL;
+  }
+
   virtual state_machine::state_t get_default_state (const svalue *) const
   {
     return m_start;
diff --git a/gcc/analyzer/supergraph.cc b/gcc/analyzer/supergraph.cc
index 85acf44..be8cec3 100644
--- a/gcc/analyzer/supergraph.cc
+++ b/gcc/analyzer/supergraph.cc
@@ -41,6 +41,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "graphviz.h"
 #include "cgraph.h"
 #include "tree-dfa.h"
+#include "bitmap.h"
 #include "cfganal.h"
 #include "function.h"
 #include "json.h"
diff --git a/gcc/attr-fnspec.h b/gcc/attr-fnspec.h
index 1154c30..cd618cb 100644
--- a/gcc/attr-fnspec.h
+++ b/gcc/attr-fnspec.h
@@ -264,6 +264,29 @@ public:
     return str[1] == 'C' || str[1] == 'P';
   }
 
+  /* Return EAF flags for arg I.  */
+  int
+  arg_eaf_flags (unsigned int i)
+  {
+    int flags = 0;
+
+    if (!arg_specified_p (i))
+      ;
+    else if (!arg_used_p (i))
+      flags = EAF_UNUSED;
+    else
+      {
+	if (arg_direct_p (i))
+	  flags |= EAF_NO_INDIRECT_READ | EAF_NO_INDIRECT_ESCAPE
+		   | EAF_NOT_RETURNED_INDIRECTLY | EAF_NO_INDIRECT_CLOBBER;
+	if (arg_noescape_p (i))
+	  flags |= EAF_NO_DIRECT_ESCAPE | EAF_NO_INDIRECT_ESCAPE;
+	if (arg_readonly_p (i))
+	  flags |= EAF_NO_DIRECT_CLOBBER | EAF_NO_INDIRECT_CLOBBER;
+      }
+    return flags;
+  }
+
   /* Check validity of the string.  */
   void verify ();
 
diff --git a/gcc/attribs.c b/gcc/attribs.c
index 83fafc9..0284e5b 100644
--- a/gcc/attribs.c
+++ b/gcc/attribs.c
@@ -87,6 +87,8 @@ struct scoped_attributes
   const char *ns;
   vec<attribute_spec> attributes;
   hash_table<attribute_hasher> *attribute_hash;
+  /* True if we should not warn about unknown attributes in this NS.  */
+  bool ignored_p;
 };
 
 /* The table of scope attributes.  */
@@ -95,6 +97,8 @@ static vec<scoped_attributes> attributes_table;
 static scoped_attributes* find_attribute_namespace (const char*);
 static void register_scoped_attribute (const struct attribute_spec *,
 				       scoped_attributes *);
+static const struct attribute_spec *lookup_scoped_attribute_spec (const_tree,
+								  const_tree);
 
 static bool attributes_initialized = false;
 
@@ -121,12 +125,14 @@ extract_attribute_substring (struct substring *str)
 
 /* Insert an array of attributes ATTRIBUTES into a namespace.  This
    array must be NULL terminated.  NS is the name of attribute
-   namespace.  The function returns the namespace into which the
-   attributes have been registered.  */
+   namespace.  IGNORED_P is true iff all unknown attributes in this
+   namespace should be ignored for the purposes of -Wattributes.  The
+   function returns the namespace into which the attributes have been
+   registered.  */
 
 scoped_attributes *
 register_scoped_attributes (const struct attribute_spec *attributes,
-			    const char *ns)
+			    const char *ns, bool ignored_p /*=false*/)
 {
   scoped_attributes *result = NULL;
 
@@ -144,9 +150,12 @@ register_scoped_attributes (const struct attribute_spec *attributes,
       memset (&sa, 0, sizeof (sa));
       sa.ns = ns;
       sa.attributes.create (64);
+      sa.ignored_p = ignored_p;
       result = attributes_table.safe_push (sa);
       result->attribute_hash = new hash_table<attribute_hasher> (200);
     }
+  else
+    result->ignored_p |= ignored_p;
 
   /* Really add the attributes to their namespace now.  */
   for (unsigned i = 0; attributes[i].name != NULL; ++i)
@@ -224,6 +233,99 @@ check_attribute_tables (void)
 				 attribute_tables[j][l].name));
 }
 
+/* Used to stash pointers to allocated memory so that we can free them at
+   the end of parsing of all TUs. */
+static vec<attribute_spec *> ignored_attributes_table;
+
+/* Parse arguments V of -Wno-attributes=.
+   Currently we accept:
+     vendor::attr
+     vendor::
+   This functions also registers the parsed attributes so that we don't
+   warn that we don't recognize them.  */
+
+void
+handle_ignored_attributes_option (vec<char *> *v)
+{
+  if (v == nullptr)
+    return;
+
+  for (auto opt : v)
+    {
+      char *cln = strstr (opt, "::");
+      /* We don't accept '::attr'.  */
+      if (cln == nullptr || cln == opt)
+	{
+	  error ("wrong argument to ignored attributes");
+	  inform (input_location, "valid format is %<ns::attr%> or %<ns::%>");
+	  continue;
+	}
+      const char *vendor_start = opt;
+      ptrdiff_t vendor_len = cln - opt;
+      const char *attr_start = cln + 2;
+      /* This could really use rawmemchr :(.  */
+      ptrdiff_t attr_len = strchr (attr_start, '\0') - attr_start;
+      /* Verify that they look valid.  */
+      auto valid_p = [](const char *const s, ptrdiff_t len) {
+	bool ok = false;
+
+	for (int i = 0; i < len; ++i)
+	  if (ISALNUM (s[i]))
+	    ok = true;
+	  else if (s[i] != '_')
+	    return false;
+
+	return ok;
+      };
+      if (!valid_p (vendor_start, vendor_len))
+	{
+	  error ("wrong argument to ignored attributes");
+	  continue;
+	}
+      canonicalize_attr_name (vendor_start, vendor_len);
+      /* We perform all this hijinks so that we don't have to copy OPT.  */
+      tree vendor_id = get_identifier_with_length (vendor_start, vendor_len);
+      const char *attr;
+      /* In the "vendor::" case, we should ignore *any* attribute coming
+	 from this attribute namespace.  */
+      if (attr_len > 0)
+	{
+	  if (!valid_p (attr_start, attr_len))
+	    {
+	      error ("wrong argument to ignored attributes");
+	      continue;
+	    }
+	  canonicalize_attr_name (attr_start, attr_len);
+	  tree attr_id = get_identifier_with_length (attr_start, attr_len);
+	  attr = IDENTIFIER_POINTER (attr_id);
+	  /* If we've already seen this vendor::attr, ignore it.  Attempting to
+	     register it twice would lead to a crash.  */
+	  if (lookup_scoped_attribute_spec (vendor_id, attr_id))
+	    continue;
+	}
+      else
+	attr = nullptr;
+      /* Create a table with extra attributes which we will register.
+	 We can't free it here, so squirrel away the pointers.  */
+      attribute_spec *table = new attribute_spec[2];
+      ignored_attributes_table.safe_push (table);
+      table[0] = { attr, 0, 0, false, false, false, false, nullptr, nullptr };
+      table[1] = { nullptr, 0, 0, false, false, false, false, nullptr,
+		   nullptr };
+      register_scoped_attributes (table, IDENTIFIER_POINTER (vendor_id), !attr);
+    }
+}
+
+/* Free data we might have allocated when adding extra attributes.  */
+
+void
+free_attr_data ()
+{
+  for (auto x : ignored_attributes_table)
+    delete[] x;
+  ignored_attributes_table.release ();
+}
+
 /* Initialize attribute tables, and make some sanity checks if checking is
    enabled.  */
 
@@ -252,6 +354,9 @@ init_attributes (void)
     /* Put all the GNU attributes into the "gnu" namespace.  */
     register_scoped_attributes (attribute_tables[i], "gnu");
 
+  vec<char *> *ignored = (vec<char *> *) flag_ignored_attributes;
+  handle_ignored_attributes_option (ignored);
+
   invoke_plugin_callbacks (PLUGIN_ATTRIBUTES, NULL);
   attributes_initialized = true;
 }
@@ -456,6 +561,19 @@ diag_attr_exclusions (tree last_decl, tree node, tree attrname,
   return found;
 }
 
+/* Return true iff we should not complain about unknown attributes
+   coming from the attribute namespace NS.  This is the case for
+   the -Wno-attributes=ns:: command-line option.  */
+
+static bool
+attr_namespace_ignored_p (tree ns)
+{
+  if (ns == NULL_TREE)
+    return false;
+  scoped_attributes *r = find_attribute_namespace (IDENTIFIER_POINTER (ns));
+  return r && r->ignored_p;
+}
+
 /* Process the attributes listed in ATTRIBUTES and install them in *NODE,
    which is either a DECL (including a TYPE_DECL) or a TYPE.  If a DECL,
    it should be modified in place; if a TYPE, a copy should be created
@@ -556,7 +674,8 @@ decl_attributes (tree *node, tree attributes, int flags,
 
       if (spec == NULL)
 	{
-	  if (!(flags & (int) ATTR_FLAG_BUILT_IN))
+	  if (!(flags & (int) ATTR_FLAG_BUILT_IN)
+	      && !attr_namespace_ignored_p (ns))
 	    {
 	      if (ns == NULL_TREE || !cxx11_attr_p)
 		warning (OPT_Wattributes, "%qE attribute directive ignored",
diff --git a/gcc/attribs.h b/gcc/attribs.h
index 138c509..ba65788 100644
--- a/gcc/attribs.h
+++ b/gcc/attribs.h
@@ -21,6 +21,7 @@ along with GCC; see the file COPYING3.  If not see
 #define GCC_ATTRIBS_H
 
 extern const struct attribute_spec *lookup_attribute_spec (const_tree);
+extern void free_attr_data ();
 extern void init_attributes (void);
 
 /* Process the attributes listed in ATTRIBUTES and install them in *NODE,
@@ -40,12 +41,14 @@ extern void apply_tm_attr (tree, tree);
 extern tree make_attribute (const char *, const char *, tree);
 
 extern struct scoped_attributes* register_scoped_attributes (const struct attribute_spec *,
-							     const char *);
+							     const char *,
+							     bool = false);
 
 extern char *sorted_attr_string (tree);
 extern bool common_function_versions (tree, tree);
 extern tree make_dispatcher_decl (const tree);
 extern bool is_function_default_version (const tree);
+extern void handle_ignored_attributes_option (vec<char *> *);
 
 /* Return a type like TTYPE except that its TYPE_ATTRIBUTES
    is ATTRIBUTE.
@@ -114,17 +117,34 @@ extern unsigned decls_mismatched_attributes (tree, tree, tree,
 
 extern void maybe_diag_alias_attributes (tree, tree);
 
+/* For a given string S of length L, strip leading and trailing '_' characters
+   so that we have a canonical form of attribute names.  NB: This function may
+   change S and L.  */
+
+template <typename T>
+inline bool
+canonicalize_attr_name (const char *&s, T &l)
+{
+  if (l > 4 && s[0] == '_' && s[1] == '_' && s[l - 1] == '_' && s[l - 2] == '_')
+    {
+      s += 2;
+      l -= 4;
+      return true;
+    }
+  return false;
+}
+
 /* For a given IDENTIFIER_NODE, strip leading and trailing '_' characters
    so that we have a canonical form of attribute names.  */
 
 static inline tree
 canonicalize_attr_name (tree attr_name)
 {
-  const size_t l = IDENTIFIER_LENGTH (attr_name);
+  size_t l = IDENTIFIER_LENGTH (attr_name);
   const char *s = IDENTIFIER_POINTER (attr_name);
 
-  if (l > 4 && s[0] == '_' && s[1] == '_' && s[l - 1] == '_' && s[l - 2] == '_')
-    return get_identifier_with_length (s + 2, l - 4);
+  if (canonicalize_attr_name (s, l))
+    return get_identifier_with_length (s, l);
 
   return attr_name;
 }
diff --git a/gcc/bitmap.c b/gcc/bitmap.c
index 98c6d0b..1aa8e4b 100644
--- a/gcc/bitmap.c
+++ b/gcc/bitmap.c
@@ -983,7 +983,7 @@ bitmap_set_bit (bitmap head, int bit)
 
 /* Return whether a bit is set within a bitmap.  */
 
-int
+bool
 bitmap_bit_p (const_bitmap head, int bit)
 {
   unsigned int indx = bit / BITMAP_ELEMENT_ALL_BITS;
diff --git a/gcc/bitmap.h b/gcc/bitmap.h
index 1bddcdb..fc33877 100644
--- a/gcc/bitmap.h
+++ b/gcc/bitmap.h
@@ -436,7 +436,7 @@ extern bool bitmap_clear_bit (bitmap, int);
 extern bool bitmap_set_bit (bitmap, int);
 
 /* Return true if a bit is set in a bitmap.  */
-extern int bitmap_bit_p (const_bitmap, int);
+extern bool bitmap_bit_p (const_bitmap, int);
 
 /* Set and get multiple bit values in a sparse bitmap.  This allows a bitmap to
    function as a sparse array of bit patterns where the patterns are
diff --git a/gcc/builtin-types.def b/gcc/builtin-types.def
index d160826..3f1c81b 100644
--- a/gcc/builtin-types.def
+++ b/gcc/builtin-types.def
@@ -489,7 +489,6 @@ DEF_FUNCTION_TYPE_2 (BT_FN_BOOL_VPTR_INT, BT_BOOL, BT_VOLATILE_PTR, BT_INT)
 DEF_FUNCTION_TYPE_2 (BT_FN_BOOL_SIZE_CONST_VPTR, BT_BOOL, BT_SIZE,
 		     BT_CONST_VOLATILE_PTR)
 DEF_FUNCTION_TYPE_2 (BT_FN_BOOL_INT_BOOL, BT_BOOL, BT_INT, BT_BOOL)
-DEF_FUNCTION_TYPE_2 (BT_FN_VOID_UINT_UINT, BT_VOID, BT_UINT, BT_UINT)
 DEF_FUNCTION_TYPE_2 (BT_FN_UINT_UINT_PTR, BT_UINT, BT_UINT, BT_PTR)
 DEF_FUNCTION_TYPE_2 (BT_FN_UINT_UINT_CONST_PTR, BT_UINT, BT_UINT, BT_CONST_PTR)
 DEF_FUNCTION_TYPE_2 (BT_FN_PTR_CONST_PTR_SIZE, BT_PTR, BT_CONST_PTR, BT_SIZE)
@@ -680,6 +679,8 @@ DEF_FUNCTION_TYPE_4 (BT_FN_BOOL_UINT_ULLPTR_ULLPTR_ULLPTR,
 		     BT_PTR_ULONGLONG)
 DEF_FUNCTION_TYPE_4 (BT_FN_VOID_UINT_PTR_INT_PTR, BT_VOID, BT_INT, BT_PTR,
 		     BT_INT, BT_PTR)
+DEF_FUNCTION_TYPE_4 (BT_FN_BOOL_UINT_UINT_UINT_BOOL,
+		     BT_BOOL, BT_UINT, BT_UINT, BT_UINT, BT_BOOL)
 
 DEF_FUNCTION_TYPE_5 (BT_FN_INT_STRING_INT_SIZE_CONST_STRING_VALIST_ARG,
 		     BT_INT, BT_STRING, BT_INT, BT_SIZE, BT_CONST_STRING,
diff --git a/gcc/builtins.c b/gcc/builtins.c
index f1c3fea..384864b 100644
--- a/gcc/builtins.c
+++ b/gcc/builtins.c
@@ -3600,7 +3600,7 @@ check_strncat_sizes (tree exp, tree objsize)
   /* Try to verify that the destination is big enough for the shortest
      string.  */
 
-  access_data data (exp, access_read_write, maxread, true);
+  access_data data (nullptr, exp, access_read_write, maxread, true);
   if (!objsize && warn_stringop_overflow)
     {
       /* If it hasn't been provided by __strncat_chk, try to determine
@@ -4260,12 +4260,6 @@ expand_builtin_memcmp (tree exp, rtx target, bool result_eq)
   tree arg2 = CALL_EXPR_ARG (exp, 1);
   tree len = CALL_EXPR_ARG (exp, 2);
 
-  /* Diagnose calls where the specified length exceeds the size of either
-     object.  */
-  if (!check_read_access (exp, arg1, len, 0)
-      || !check_read_access (exp, arg2, len, 0))
-    return NULL_RTX;
-
   /* Due to the performance benefit, always inline the calls first
      when result_eq is false.  */
   rtx result = NULL_RTX;
@@ -5486,27 +5480,6 @@ expand_builtin_fork_or_exec (tree fn, tree exp, rtx target, int ignore)
   tree id, decl;
   tree call;
 
-  if (DECL_FUNCTION_CODE (fn) != BUILT_IN_FORK)
-    {
-      tree path = CALL_EXPR_ARG (exp, 0);
-      /* Detect unterminated path.  */
-      if (!check_read_access (exp, path))
-	return NULL_RTX;
-
-      /* Also detect unterminated first argument.  */
-      switch (DECL_FUNCTION_CODE (fn))
-	{
-	case BUILT_IN_EXECL:
-	case BUILT_IN_EXECLE:
-	case BUILT_IN_EXECLP:
-	  if (!check_read_access (exp, path))
-	    return NULL_RTX;
-	default:
-	  break;
-	}
-    }
-
-
   /* If we are not profiling, just call the function.  */
   if (!profile_arc_flag)
     return NULL_RTX;
@@ -10738,6 +10711,7 @@ is_simple_builtin (tree decl)
       case BUILT_IN_VA_END:
       case BUILT_IN_STACK_SAVE:
       case BUILT_IN_STACK_RESTORE:
+      case BUILT_IN_DWARF_CFA:
 	/* Exception state returns or moves registers around.  */
       case BUILT_IN_EH_FILTER:
       case BUILT_IN_EH_POINTER:
@@ -11126,6 +11100,19 @@ builtin_fnspec (tree callee)
       CASE_BUILT_IN_TM_STORE (M256):
 	return ".cO ";
       case BUILT_IN_STACK_SAVE:
+      case BUILT_IN_RETURN:
+      case BUILT_IN_EH_POINTER:
+      case BUILT_IN_EH_FILTER:
+      case BUILT_IN_UNWIND_RESUME:
+      case BUILT_IN_CXA_END_CLEANUP:
+      case BUILT_IN_EH_COPY_VALUES:
+      case BUILT_IN_FRAME_ADDRESS:
+      case BUILT_IN_APPLY_ARGS:
+      case BUILT_IN_ASAN_BEFORE_DYNAMIC_INIT:
+      case BUILT_IN_ASAN_AFTER_DYNAMIC_INIT:
+      case BUILT_IN_PREFETCH:
+      case BUILT_IN_DWARF_CFA:
+      case BUILT_IN_RETURN_ADDRESS:
 	return ".c";
       case BUILT_IN_ASSUME_ALIGNED:
 	return "1cX ";
diff --git a/gcc/c-family/ChangeLog b/gcc/c-family/ChangeLog
index c5f5915..7ed0102 100644
--- a/gcc/c-family/ChangeLog
+++ b/gcc/c-family/ChangeLog
@@ -1,3 +1,67 @@
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* c.opt: Add -fimplicit-constexpr.
+	* c-cppbuiltin.c: Define __cpp_implicit_constexpr.
+	* c-opts.c (c_common_post_options): Disable below C++14.
+
+2021-11-15  Jakub Jelinek  <jakub@redhat.com>
+
+	* c-omp.c (c_omp_split_clauses) <case OMP_CLAUSE_THREAD_LIMIT>:
+	Duplicate to both OMP_TARGET and OMP_TEAMS.
+
+2021-11-10  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/101940
+	* c-pragma.c (handle_pragma_diagnostic): Handle #pragma GCC diagnostic
+	ignored_attributes.
+
+2021-11-09  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-pragma.c (GCC_BAD_AT): New macro.
+	(GCC_BAD2_AT): New macro.
+	(handle_pragma_pack): Use the location of the pertinent token when
+	issuing diagnostics about invalid constants/actions, and trailing
+	junk.
+	(handle_pragma_target): Likewise for non-string "GCC option".
+	(handle_pragma_message): Likewise for trailing junk.
+
+2021-11-03  Joseph Myers  <joseph@codesourcery.com>
+
+	PR c/103031
+	* c-common.c (convert_and_check): Add argument init_const.  Call
+	convert_init if init_const.
+	* c-common.h (convert_and_check): Update prototype.
+	(convert_init): New prototype.
+
+2021-11-01  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-lex.c (c_lex_with_flags): When complaining about non-printable
+	CPP_OTHER tokens, set the "escape on output" flag.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	* c-omp.c (c_omp_check_loop_iv_r): Don't clear 3rd bit for
+	POINTER_PLUS_EXPR.
+	(c_omp_check_nonrect_loop_iv): Handle POINTER_PLUS_EXPR.
+	(c_omp_check_loop_iv): Set kind even if the iterator is non-integral.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/102854
+	* c-common.h (c_omp_check_loop_iv_exprs): Add enum tree_code argument.
+	* c-omp.c (c_omp_check_loop_iv_r): For trees other than decls,
+	TREE_VEC, PLUS_EXPR, MINUS_EXPR, MULT_EXPR, POINTER_PLUS_EXPR or
+	conversions temporarily clear the 3rd bit from d->kind while walking
+	subtrees.
+	(c_omp_check_loop_iv_exprs): Add CODE argument.  Or in 4 into data.kind
+	if possibly non-rectangular.
+
+2021-10-21  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/64888
+	* c-omp.c (c_omp_predefined_variable): Return true also for
+	ubsan_create_data created artificial variables.
+
 2021-10-14  Joseph Myers  <joseph@codesourcery.com>
 
 	* c-format.c (printf_length_specs, scanf_length_specs)
diff --git a/gcc/c-family/c-common.c b/gcc/c-family/c-common.c
index 32c7e3e..436df45 100644
--- a/gcc/c-family/c-common.c
+++ b/gcc/c-family/c-common.c
@@ -1739,10 +1739,13 @@ unsafe_conversion_p (tree type, tree expr, tree result, bool check_sign)
 
 /* Convert EXPR to TYPE, warning about conversion problems with constants.
    Invoke this function on every expression that is converted implicitly,
-   i.e. because of language rules and not because of an explicit cast.  */
+   i.e. because of language rules and not because of an explicit cast.
+   INIT_CONST is true if the conversion is for arithmetic types for a static
+   initializer and folding must apply accordingly (discarding floating-point
+   exceptions and assuming the default rounding mode is in effect).  */
 
 tree
-convert_and_check (location_t loc, tree type, tree expr)
+convert_and_check (location_t loc, tree type, tree expr, bool init_const)
 {
   tree result;
   tree expr_for_warning;
@@ -1754,7 +1757,9 @@ convert_and_check (location_t loc, tree type, tree expr)
     {
       tree orig_type = TREE_TYPE (expr);
       expr = TREE_OPERAND (expr, 0);
-      expr_for_warning = convert (orig_type, expr);
+      expr_for_warning = (init_const
+			  ? convert_init (orig_type, expr)
+			  : convert (orig_type, expr));
       if (orig_type == type)
 	return expr_for_warning;
     }
@@ -1764,7 +1769,7 @@ convert_and_check (location_t loc, tree type, tree expr)
   if (TREE_TYPE (expr) == type)
     return expr;
 
-  result = convert (type, expr);
+  result = init_const ? convert_init (type, expr) : convert (type, expr);
 
   if (c_inhibit_evaluation_warnings == 0
       && !TREE_OVERFLOW_P (expr)
diff --git a/gcc/c-family/c-common.h b/gcc/c-family/c-common.h
index 078730f..d5dad99 100644
--- a/gcc/c-family/c-common.h
+++ b/gcc/c-family/c-common.h
@@ -886,7 +886,7 @@ extern tree c_alignof_expr (location_t, tree);
    NOP_EXPR is used as a special case (see truthvalue_conversion).  */
 extern void binary_op_error (rich_location *, enum tree_code, tree, tree);
 extern tree fix_string_type (tree);
-extern tree convert_and_check (location_t, tree, tree);
+extern tree convert_and_check (location_t, tree, tree, bool = false);
 extern bool c_determine_visibility (tree);
 extern bool vector_types_compatible_elements_p (tree, tree);
 extern void mark_valid_location_for_stdc_pragma (bool);
@@ -908,6 +908,8 @@ extern tree c_common_get_narrower (tree, int *);
 extern bool get_attribute_operand (tree, unsigned HOST_WIDE_INT *);
 extern void c_common_finalize_early_debug (void);
 
+/* Used by convert_and_check; in front ends.  */
+extern tree convert_init (tree, tree);
 
 #define c_sizeof(LOC, T)  c_sizeof_or_alignof_type (LOC, T, true, false, 1)
 #define c_alignof(LOC, T) c_sizeof_or_alignof_type (LOC, T, false, false, 1)
@@ -1234,8 +1236,8 @@ extern void c_finish_omp_taskyield (location_t);
 extern tree c_finish_omp_for (location_t, enum tree_code, tree, tree, tree,
 			      tree, tree, tree, tree, bool);
 extern bool c_omp_check_loop_iv (tree, tree, walk_tree_lh);
-extern bool c_omp_check_loop_iv_exprs (location_t, tree, int, tree, tree, tree,
-				       walk_tree_lh);
+extern bool c_omp_check_loop_iv_exprs (location_t, enum tree_code, tree, int,
+				       tree, tree, tree, walk_tree_lh);
 extern tree c_finish_oacc_wait (location_t, tree, tree);
 extern tree c_oacc_split_loop_clauses (tree, tree *, bool);
 extern void c_omp_split_clauses (location_t, enum tree_code, omp_clause_mask,
diff --git a/gcc/c-family/c-cppbuiltin.c b/gcc/c-family/c-cppbuiltin.c
index 60e9e05..eb34d5a 100644
--- a/gcc/c-family/c-cppbuiltin.c
+++ b/gcc/c-family/c-cppbuiltin.c
@@ -1111,6 +1111,8 @@ c_cpp_builtins (cpp_reader *pfile)
       if (cxx_dialect >= cxx11 && strcmp (thread_model, "single") != 0)
 	cpp_define (pfile, "__STDCPP_THREADS__=1");
 #endif
+      if (flag_implicit_constexpr)
+	cpp_define (pfile, "__cpp_implicit_constexpr=20211111L");
     }
   /* Note that we define this for C as well, so that we know if
      __attribute__((cleanup)) will interface with EH.  */
diff --git a/gcc/c-family/c-lex.c b/gcc/c-family/c-lex.c
index 4b04e71..2651331 100644
--- a/gcc/c-family/c-lex.c
+++ b/gcc/c-family/c-lex.c
@@ -616,7 +616,11 @@ c_lex_with_flags (tree *value, location_t *loc, unsigned char *cpp_flags,
 	else if (ISGRAPH (c))
 	  error_at (*loc, "stray %qc in program", (int) c);
 	else
-	  error_at (*loc, "stray %<\\%o%> in program", (int) c);
+	  {
+	    rich_location rich_loc (line_table, *loc);
+	    rich_loc.set_escape_on_output (true);
+	    error_at (&rich_loc, "stray %<\\%o%> in program", (int) c);
+	  }
       }
       goto retry;
 
diff --git a/gcc/c-family/c-omp.c b/gcc/c-family/c-omp.c
index 5625be7..3f84fd1 100644
--- a/gcc/c-family/c-omp.c
+++ b/gcc/c-family/c-omp.c
@@ -1353,6 +1353,20 @@ c_omp_check_loop_iv_r (tree *tp, int *walk_subtrees, void *data)
 	}
       d->fail = true;
     }
+  else if ((d->kind & 4)
+	   && TREE_CODE (*tp) != TREE_VEC
+	   && TREE_CODE (*tp) != PLUS_EXPR
+	   && TREE_CODE (*tp) != MINUS_EXPR
+	   && TREE_CODE (*tp) != MULT_EXPR
+	   && TREE_CODE (*tp) != POINTER_PLUS_EXPR
+	   && !CONVERT_EXPR_P (*tp))
+    {
+      *walk_subtrees = 0;
+      d->kind &= 3;
+      walk_tree_1 (tp, c_omp_check_loop_iv_r, data, NULL, d->lh);
+      d->kind |= 4;
+      return NULL_TREE;
+    }
   else if (d->ppset->add (*tp))
     *walk_subtrees = 0;
   /* Don't walk dtors added by C++ wrap_cleanups_r.  */
@@ -1464,6 +1478,18 @@ c_omp_check_nonrect_loop_iv (tree *tp, struct c_omp_check_loop_iv_data *d,
 	}
       a2 = integer_zero_node;
       break;
+    case POINTER_PLUS_EXPR:
+      a1 = TREE_OPERAND (t, 0);
+      a2 = TREE_OPERAND (t, 1);
+      while (CONVERT_EXPR_P (a1))
+	a1 = TREE_OPERAND (a1, 0);
+      if (DECL_P (a1) && c_omp_is_loop_iterator (a1, d) >= 0)
+	{
+	  a2 = TREE_OPERAND (t, 1);
+	  t = a1;
+	  break;
+	}
+      break;
     default:
       break;
     }
@@ -1586,10 +1612,7 @@ c_omp_check_loop_iv (tree stmt, tree declv, walk_tree_lh lh)
 	  data.fail = true;
 	}
       /* Handle non-rectangular loop nests.  */
-      if (TREE_CODE (stmt) != OACC_LOOP
-	  && (TREE_CODE (TREE_OPERAND (init, 1)) == TREE_VEC
-	      || INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (init, 1))))
-	  && i > 0)
+      if (TREE_CODE (stmt) != OACC_LOOP && i > 0)
 	kind = 4;
       data.kind = kind;
       data.idx = i;
@@ -1651,11 +1674,13 @@ c_omp_check_loop_iv (tree stmt, tree declv, walk_tree_lh lh)
 /* Similar, but allows to check the init or cond expressions individually.  */
 
 bool
-c_omp_check_loop_iv_exprs (location_t stmt_loc, tree declv, int i, tree decl,
-			   tree init, tree cond, walk_tree_lh lh)
+c_omp_check_loop_iv_exprs (location_t stmt_loc, enum tree_code code,
+			   tree declv, int i, tree decl, tree init, tree cond,
+			   walk_tree_lh lh)
 {
   hash_set<tree> pset;
   struct c_omp_check_loop_iv_data data;
+  int kind = (code != OACC_LOOP && i > 0) ? 4 : 0;
 
   data.declv = declv;
   data.fail = false;
@@ -1674,7 +1699,7 @@ c_omp_check_loop_iv_exprs (location_t stmt_loc, tree declv, int i, tree decl,
   if (init)
     {
       data.expr_loc = EXPR_LOCATION (init);
-      data.kind = 0;
+      data.kind = kind;
       walk_tree_1 (&init,
 		   c_omp_check_loop_iv_r, &data, NULL, lh);
     }
@@ -1682,7 +1707,7 @@ c_omp_check_loop_iv_exprs (location_t stmt_loc, tree declv, int i, tree decl,
     {
       gcc_assert (COMPARISON_CLASS_P (cond));
       data.expr_loc = EXPR_LOCATION (init);
-      data.kind = 1;
+      data.kind = kind | 1;
       if (TREE_OPERAND (cond, 0) == decl)
 	walk_tree_1 (&TREE_OPERAND (cond, 1),
 		     c_omp_check_loop_iv_r, &data, NULL, lh);
@@ -1842,7 +1867,6 @@ c_omp_split_clauses (location_t loc, enum tree_code code,
 	  s = C_OMP_CLAUSE_SPLIT_TARGET;
 	  break;
 	case OMP_CLAUSE_NUM_TEAMS:
-	case OMP_CLAUSE_THREAD_LIMIT:
 	  s = C_OMP_CLAUSE_SPLIT_TEAMS;
 	  break;
 	case OMP_CLAUSE_DIST_SCHEDULE:
@@ -2506,6 +2530,30 @@ c_omp_split_clauses (location_t loc, enum tree_code code,
 	  else
 	    s = C_OMP_CLAUSE_SPLIT_FOR;
 	  break;
+	  /* thread_limit is allowed on target and teams.  Distribute it
+	     to all.  */
+	case OMP_CLAUSE_THREAD_LIMIT:
+	  if ((mask & (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_MAP))
+	      != 0)
+	    {
+	      if ((mask & (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_NUM_TEAMS))
+		  != 0)
+		{
+		  c = build_omp_clause (OMP_CLAUSE_LOCATION (clauses),
+					OMP_CLAUSE_THREAD_LIMIT);
+		  OMP_CLAUSE_THREAD_LIMIT_EXPR (c)
+		    = OMP_CLAUSE_THREAD_LIMIT_EXPR (clauses);
+		  OMP_CLAUSE_CHAIN (c) = cclauses[C_OMP_CLAUSE_SPLIT_TARGET];
+		  cclauses[C_OMP_CLAUSE_SPLIT_TARGET] = c;
+		}
+	      else
+		{
+		  s = C_OMP_CLAUSE_SPLIT_TARGET;
+		  break;
+		}
+	    }
+	  s = C_OMP_CLAUSE_SPLIT_TEAMS;
+	  break;
 	/* Allocate clause is allowed on target, teams, distribute, parallel,
 	   for, sections and taskloop.  Distribute it to all.  */
 	case OMP_CLAUSE_ALLOCATE:
diff --git a/gcc/c-family/c-opts.c b/gcc/c-family/c-opts.c
index 0225cba..62f5829 100644
--- a/gcc/c-family/c-opts.c
+++ b/gcc/c-family/c-opts.c
@@ -1052,6 +1052,9 @@ c_common_post_options (const char **pfilename)
       && flag_strong_eval_order == -1)
     flag_strong_eval_order = (cxx_dialect >= cxx17 ? 2 : 1);
 
+  if (flag_implicit_constexpr && cxx_dialect < cxx14)
+    flag_implicit_constexpr = false;
+
   /* Global sized deallocation is new in C++14.  */
   if (flag_sized_deallocation == -1)
     flag_sized_deallocation = (cxx_dialect >= cxx14);
diff --git a/gcc/c-family/c-pragma.c b/gcc/c-family/c-pragma.c
index a9be8df..3663eb1 100644
--- a/gcc/c-family/c-pragma.c
+++ b/gcc/c-family/c-pragma.c
@@ -39,6 +39,10 @@ along with GCC; see the file COPYING3.  If not see
   do { warning (OPT_Wpragmas, gmsgid); return; } while (0)
 #define GCC_BAD2(gmsgid, arg) \
   do { warning (OPT_Wpragmas, gmsgid, arg); return; } while (0)
+#define GCC_BAD_AT(loc, gmsgid)					\
+  do { warning_at (loc, OPT_Wpragmas, gmsgid); return; } while (0)
+#define GCC_BAD2_AT(loc, gmsgid, arg)			\
+  do { warning_at (loc, OPT_Wpragmas, gmsgid, arg); return; } while (0)
 
 struct GTY(()) align_stack {
   int		       alignment;
@@ -130,6 +134,7 @@ pop_alignment (tree id)
 static void
 handle_pragma_pack (cpp_reader * ARG_UNUSED (dummy))
 {
+  location_t loc;
   tree x, id = 0;
   int align = -1;
   enum cpp_ttype token;
@@ -138,7 +143,7 @@ handle_pragma_pack (cpp_reader * ARG_UNUSED (dummy))
   if (pragma_lex (&x) != CPP_OPEN_PAREN)
     GCC_BAD ("missing %<(%> after %<#pragma pack%> - ignored");
 
-  token = pragma_lex (&x);
+  token = pragma_lex (&x, &loc);
   if (token == CPP_CLOSE_PAREN)
     {
       action = set;
@@ -147,7 +152,7 @@ handle_pragma_pack (cpp_reader * ARG_UNUSED (dummy))
   else if (token == CPP_NUMBER)
     {
       if (TREE_CODE (x) != INTEGER_CST)
-	GCC_BAD ("invalid constant in %<#pragma pack%> - ignored");
+	GCC_BAD_AT (loc, "invalid constant in %<#pragma pack%> - ignored");
       align = TREE_INT_CST_LOW (x);
       action = set;
       if (pragma_lex (&x) != CPP_CLOSE_PAREN)
@@ -167,11 +172,12 @@ handle_pragma_pack (cpp_reader * ARG_UNUSED (dummy))
       else if (!strcmp (op, "pop"))
 	action = pop;
       else
-	GCC_BAD2 ("unknown action %qE for %<#pragma pack%> - ignored", x);
+	GCC_BAD2_AT (loc, "unknown action %qE for %<#pragma pack%> - ignored",
+		     x);
 
       while ((token = pragma_lex (&x)) == CPP_COMMA)
 	{
-	  token = pragma_lex (&x);
+	  token = pragma_lex (&x, &loc);
 	  if (token == CPP_NAME && id == 0)
 	    {
 	      id = x;
@@ -179,7 +185,8 @@ handle_pragma_pack (cpp_reader * ARG_UNUSED (dummy))
 	  else if (token == CPP_NUMBER && action == push && align == -1)
 	    {
 	      if (TREE_CODE (x) != INTEGER_CST)
-		GCC_BAD ("invalid constant in %<#pragma pack%> - ignored");
+		GCC_BAD_AT (loc,
+			    "invalid constant in %<#pragma pack%> - ignored");
 	      align = TREE_INT_CST_LOW (x);
 	      if (align == -1)
 		action = set;
@@ -195,8 +202,8 @@ handle_pragma_pack (cpp_reader * ARG_UNUSED (dummy))
   else
     GCC_BAD ("malformed %<#pragma pack%> - ignored");
 
-  if (pragma_lex (&x) != CPP_EOF)
-    warning (OPT_Wpragmas, "junk at end of %<#pragma pack%>");
+  if (pragma_lex (&x, &loc) != CPP_EOF)
+    warning_at (loc, OPT_Wpragmas, "junk at end of %<#pragma pack%>");
 
   if (flag_pack_struct)
     GCC_BAD ("%<#pragma pack%> has no effect with %<-fpack-struct%> - ignored");
@@ -764,7 +771,7 @@ handle_pragma_diagnostic(cpp_reader *ARG_UNUSED(dummy))
   if (token != CPP_NAME)
     {
       warning_at (loc, OPT_Wpragmas,
-		  "missing [error|warning|ignored|push|pop]"
+		  "missing [error|warning|ignored|push|pop|ignored_attributes]"
 		  " after %<#pragma GCC diagnostic%>");
       return;
     }
@@ -787,10 +794,43 @@ handle_pragma_diagnostic(cpp_reader *ARG_UNUSED(dummy))
       diagnostic_pop_diagnostics (global_dc, input_location);
       return;
     }
+  else if (strcmp (kind_string, "ignored_attributes") == 0)
+    {
+      token = pragma_lex (&x, &loc);
+      if (token != CPP_STRING)
+	{
+	  warning_at (loc, OPT_Wpragmas,
+		      "missing attribute name after %<#pragma GCC diagnostic "
+		      "ignored_attributes%>");
+	  return;
+	}
+      char *args = xstrdup (TREE_STRING_POINTER (x));
+      const size_t l = strlen (args);
+      if (l == 0)
+	{
+	  warning_at (loc, OPT_Wpragmas, "missing argument to %<#pragma GCC "
+		      "diagnostic ignored_attributes%>");
+	  free (args);
+	  return;
+	}
+      else if (args[l - 1] == ',')
+	{
+	  warning_at (loc, OPT_Wpragmas, "trailing %<,%> in arguments for "
+		      "%<#pragma GCC diagnostic ignored_attributes%>");
+	  free (args);
+	  return;
+	}
+      auto_vec<char *> v;
+      for (char *p = strtok (args, ","); p; p = strtok (NULL, ","))
+	v.safe_push (p);
+      handle_ignored_attributes_option (&v);
+      free (args);
+      return;
+    }
   else
     {
       warning_at (loc, OPT_Wpragmas,
-		  "expected [error|warning|ignored|push|pop]"
+		  "expected [error|warning|ignored|push|pop|ignored_attributes]"
 		  " after %<#pragma GCC diagnostic%>");
       return;
     }
@@ -857,6 +897,7 @@ handle_pragma_diagnostic(cpp_reader *ARG_UNUSED(dummy))
 static void
 handle_pragma_target(cpp_reader *ARG_UNUSED(dummy))
 {
+  location_t loc;
   enum cpp_ttype token;
   tree x;
   bool close_paren_needed_p = false;
@@ -867,16 +908,16 @@ handle_pragma_target(cpp_reader *ARG_UNUSED(dummy))
       return;
     }
 
-  token = pragma_lex (&x);
+  token = pragma_lex (&x, &loc);
   if (token == CPP_OPEN_PAREN)
     {
       close_paren_needed_p = true;
-      token = pragma_lex (&x);
+      token = pragma_lex (&x, &loc);
     }
 
   if (token != CPP_STRING)
     {
-      GCC_BAD ("%<#pragma GCC option%> is not a string");
+      GCC_BAD_AT (loc, "%<#pragma GCC option%> is not a string");
       return;
     }
 
@@ -1149,6 +1190,7 @@ handle_pragma_reset_options (cpp_reader *ARG_UNUSED(dummy))
 static void
 handle_pragma_message (cpp_reader *ARG_UNUSED(dummy))
 {
+  location_t loc;
   enum cpp_ttype token;
   tree x, message = 0;
 
@@ -1170,8 +1212,8 @@ handle_pragma_message (cpp_reader *ARG_UNUSED(dummy))
 
   gcc_assert (message);
 
-  if (pragma_lex (&x) != CPP_EOF)
-    warning (OPT_Wpragmas, "junk at end of %<#pragma message%>");
+  if (pragma_lex (&x, &loc) != CPP_EOF)
+    warning_at (loc, OPT_Wpragmas, "junk at end of %<#pragma message%>");
 
   if (TREE_STRING_LENGTH (message) > 1)
     inform (input_location, "%<#pragma message: %s%>",
diff --git a/gcc/c-family/c.opt b/gcc/c-family/c.opt
index 06457ac..8a4cd63 100644
--- a/gcc/c-family/c.opt
+++ b/gcc/c-family/c.opt
@@ -1712,6 +1712,10 @@ fimplement-inlines
 C++ ObjC++ Var(flag_implement_inlines) Init(1)
 Export functions even if they can be inlined.
 
+fimplicit-constexpr
+C++ ObjC++ Var(flag_implicit_constexpr)
+Make inline functions constexpr by default.
+
 fimplicit-inline-templates
 C++ ObjC++ Var(flag_implicit_inline_templates) Init(1)
 Emit implicit instantiations of inline templates.
diff --git a/gcc/c/ChangeLog b/gcc/c/ChangeLog
index 1387318..3ce48f1 100644
--- a/gcc/c/ChangeLog
+++ b/gcc/c/ChangeLog
@@ -1,3 +1,44 @@
+2021-11-15  Jakub Jelinek  <jakub@redhat.com>
+
+	* c-parser.c (OMP_TARGET_CLAUSE_MASK): Add
+	PRAGMA_OMP_CLAUSE_THREAD_LIMIT.
+
+2021-11-11  Jakub Jelinek  <jakub@redhat.com>
+
+	* c-parser.c (c_parser_omp_clause_num_teams): Parse optional
+	lower-bound and store it into OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR.
+	Use OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR instead of
+	OMP_CLAUSE_NUM_TEAMS_EXPR.
+	(c_parser_omp_target): For OMP_CLAUSE_NUM_TEAMS evaluate before
+	combined target teams even lower-bound expression.
+
+2021-11-11  Richard Biener  <rguenther@suse.de>
+
+	* gimple-parser.c: Shuffle bitmap.h include.
+
+2021-11-03  Joseph Myers  <joseph@codesourcery.com>
+
+	PR c/103031
+	* c-convert.c (c_convert): New function, based on convert.
+	(convert): Make into wrapper of c_convert.
+	(convert_init): New function.
+	* c-typeck.c (enum impl_conv): Add ic_init_const.
+	(convert_for_assignment): Handle ic_init_const like ic_init.  Add
+	new argument to convert_and_check call.
+	(digest_init): Pass ic_init_const to convert_for_assignment for
+	initializers required to be constant.
+
+2021-11-02  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* c-tree.h (c_simulate_record_decl): Declare.
+	* c-objc-common.h (LANG_HOOKS_SIMULATE_RECORD_DECL): Override.
+	* c-decl.c (c_simulate_record_decl): New function.
+
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Make-lang.in: Add dummy c.install-dvi target.
+
 2021-10-15  Richard Biener  <rguenther@suse.de>
 
 	PR c/102763
diff --git a/gcc/c/Make-lang.in b/gcc/c/Make-lang.in
index a1cdee8..67a40fc 100644
--- a/gcc/c/Make-lang.in
+++ b/gcc/c/Make-lang.in
@@ -99,6 +99,7 @@ c.dvi:
 c.pdf:
 c.html:
 c.install-info:
+c.install-dvi:
 c.install-pdf:
 c.install-html:
 c.all.cross:
diff --git a/gcc/c/c-convert.c b/gcc/c/c-convert.c
index d0035a3..905b26a 100644
--- a/gcc/c/c-convert.c
+++ b/gcc/c/c-convert.c
@@ -60,10 +60,13 @@ along with GCC; see the file COPYING3.  If not see
    converted to type TYPE.  The TREE_TYPE of the value
    is always TYPE.  This function implements all reasonable
    conversions; callers should filter out those that are
-   not permitted by the language being compiled.  */
+   not permitted by the language being compiled.
+   INIT_CONST is true if the conversion is for arithmetic types for a static
+   initializer and folding must apply accordingly (discarding floating-point
+   exceptions and assuming the default rounding mode is in effect).  */
 
-tree
-convert (tree type, tree expr)
+static tree
+c_convert (tree type, tree expr, bool init_const)
 {
   tree e = expr;
   enum tree_code code = TREE_CODE (type);
@@ -115,7 +118,7 @@ convert (tree type, tree expr)
 	  && COMPLETE_TYPE_P (type))
 	{
 	  expr = save_expr (expr);
-	  expr = c_fully_fold (expr, false, NULL);
+	  expr = c_fully_fold (expr, init_const, NULL);
 	  tree check = ubsan_instrument_float_cast (loc, type, expr);
 	  expr = fold_build1 (FIX_TRUNC_EXPR, type, expr);
 	  if (check == NULL_TREE)
@@ -173,10 +176,32 @@ convert (tree type, tree expr)
 
     maybe_fold:
       if (TREE_CODE (ret) != C_MAYBE_CONST_EXPR)
-	ret = fold (ret);
+	ret = init_const ? fold_init (ret) : fold (ret);
       return ret;
     }
 
   error ("conversion to non-scalar type requested");
   return error_mark_node;
 }
+
+/* Create an expression whose value is that of EXPR, converted to type TYPE.
+   The TREE_TYPE of the value is always TYPE.  This function implements all
+   reasonable conversions; callers should filter out those that are not
+   permitted by the language being compiled.  */
+
+tree
+convert (tree type, tree expr)
+{
+  return c_convert (type, expr, false);
+}
+
+/* Create an expression whose value is that of EXPR, converted to type TYPE, in
+   a static initializer.  The TREE_TYPE of the value is always TYPE.  This
+   function implements all reasonable conversions; callers should filter out
+   those that are not permitted by the language being compiled.  */
+
+tree
+convert_init (tree type, tree expr)
+{
+  return c_convert (type, expr, true);
+}
diff --git a/gcc/c/c-decl.c b/gcc/c/c-decl.c
index 771efa3..186fa16 100644
--- a/gcc/c/c-decl.c
+++ b/gcc/c/c-decl.c
@@ -9436,6 +9436,36 @@ c_simulate_enum_decl (location_t loc, const char *name,
   input_location = saved_loc;
   return enumtype;
 }
+
+/* Implement LANG_HOOKS_SIMULATE_RECORD_DECL.  */
+
+tree
+c_simulate_record_decl (location_t loc, const char *name,
+			array_slice<const tree> fields)
+{
+  location_t saved_loc = input_location;
+  input_location = loc;
+
+  class c_struct_parse_info *struct_info;
+  tree ident = get_identifier (name);
+  tree type = start_struct (loc, RECORD_TYPE, ident, &struct_info);
+
+  for (unsigned int i = 0; i < fields.size (); ++i)
+    {
+      DECL_FIELD_CONTEXT (fields[i]) = type;
+      if (i > 0)
+	DECL_CHAIN (fields[i - 1]) = fields[i];
+    }
+
+  finish_struct (loc, type, fields[0], NULL_TREE, struct_info);
+
+  tree decl = build_decl (loc, TYPE_DECL, ident, type);
+  set_underlying_type (decl);
+  lang_hooks.decls.pushdecl (decl);
+
+  input_location = saved_loc;
+  return type;
+}
 
 /* Create the FUNCTION_DECL for a function definition.
    DECLSPECS, DECLARATOR and ATTRIBUTES are the parts of
diff --git a/gcc/c/c-objc-common.h b/gcc/c/c-objc-common.h
index 7d35a06..f4e8271 100644
--- a/gcc/c/c-objc-common.h
+++ b/gcc/c/c-objc-common.h
@@ -81,6 +81,8 @@ along with GCC; see the file COPYING3.  If not see
 
 #undef LANG_HOOKS_SIMULATE_ENUM_DECL
 #define LANG_HOOKS_SIMULATE_ENUM_DECL c_simulate_enum_decl
+#undef LANG_HOOKS_SIMULATE_RECORD_DECL
+#define LANG_HOOKS_SIMULATE_RECORD_DECL c_simulate_record_decl
 #undef LANG_HOOKS_TYPE_FOR_MODE
 #define LANG_HOOKS_TYPE_FOR_MODE c_common_type_for_mode
 #undef LANG_HOOKS_TYPE_FOR_SIZE
diff --git a/gcc/c/c-parser.c b/gcc/c/c-parser.c
index 80dd61d..83e837c 100644
--- a/gcc/c/c-parser.c
+++ b/gcc/c/c-parser.c
@@ -15175,7 +15175,10 @@ c_parser_omp_clause_orderedkind (c_parser *parser ATTRIBUTE_UNUSED,
 }
 
 /* OpenMP 4.0:
-   num_teams ( expression ) */
+   num_teams ( expression )
+
+   OpenMP 5.1:
+   num_teams ( expression : expression ) */
 
 static tree
 c_parser_omp_clause_num_teams (c_parser *parser, tree list)
@@ -15184,34 +15187,68 @@ c_parser_omp_clause_num_teams (c_parser *parser, tree list)
   matching_parens parens;
   if (parens.require_open (parser))
     {
-      location_t expr_loc = c_parser_peek_token (parser)->location;
+      location_t upper_loc = c_parser_peek_token (parser)->location;
+      location_t lower_loc = UNKNOWN_LOCATION;
       c_expr expr = c_parser_expr_no_commas (parser, NULL);
-      expr = convert_lvalue_to_rvalue (expr_loc, expr, false, true);
-      tree c, t = expr.value;
-      t = c_fully_fold (t, false, NULL);
+      expr = convert_lvalue_to_rvalue (upper_loc, expr, false, true);
+      tree c, upper = expr.value, lower = NULL_TREE;
+      upper = c_fully_fold (upper, false, NULL);
+
+      if (c_parser_next_token_is (parser, CPP_COLON))
+	{
+	  c_parser_consume_token (parser);
+	  lower_loc = upper_loc;
+	  lower = upper;
+	  upper_loc = c_parser_peek_token (parser)->location;
+	  expr = c_parser_expr_no_commas (parser, NULL);
+	  expr = convert_lvalue_to_rvalue (upper_loc, expr, false, true);
+	  upper = expr.value;
+	  upper = c_fully_fold (upper, false, NULL);
+	}
 
       parens.skip_until_found_close (parser);
 
-      if (!INTEGRAL_TYPE_P (TREE_TYPE (t)))
+      if (!INTEGRAL_TYPE_P (TREE_TYPE (upper))
+	  || (lower && !INTEGRAL_TYPE_P (TREE_TYPE (lower))))
 	{
 	  c_parser_error (parser, "expected integer expression");
 	  return list;
 	}
 
       /* Attempt to statically determine when the number isn't positive.  */
-      c = fold_build2_loc (expr_loc, LE_EXPR, boolean_type_node, t,
-			   build_int_cst (TREE_TYPE (t), 0));
-      protected_set_expr_location (c, expr_loc);
+      c = fold_build2_loc (upper_loc, LE_EXPR, boolean_type_node, upper,
+			   build_int_cst (TREE_TYPE (upper), 0));
+      protected_set_expr_location (c, upper_loc);
       if (c == boolean_true_node)
 	{
-	  warning_at (expr_loc, 0, "%<num_teams%> value must be positive");
-	  t = integer_one_node;
+	  warning_at (upper_loc, 0, "%<num_teams%> value must be positive");
+	  upper = integer_one_node;
+	}
+      if (lower)
+	{
+	  c = fold_build2_loc (lower_loc, LE_EXPR, boolean_type_node, lower,
+			       build_int_cst (TREE_TYPE (lower), 0));
+	  protected_set_expr_location (c, lower_loc);
+	  if (c == boolean_true_node)
+	    {
+	      warning_at (lower_loc, 0, "%<num_teams%> value must be positive");
+	      lower = NULL_TREE;
+	    }
+	  else if (TREE_CODE (lower) == INTEGER_CST
+		   && TREE_CODE (upper) == INTEGER_CST
+		   && tree_int_cst_lt (upper, lower))
+	    {
+	      warning_at (lower_loc, 0, "%<num_teams%> lower bound %qE bigger "
+					"than upper bound %qE", lower, upper);
+	      lower = NULL_TREE;
+	    }
 	}
 
       check_no_duplicate_clause (list, OMP_CLAUSE_NUM_TEAMS, "num_teams");
 
       c = build_omp_clause (num_teams_loc, OMP_CLAUSE_NUM_TEAMS);
-      OMP_CLAUSE_NUM_TEAMS_EXPR (c) = t;
+      OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (c) = upper;
+      OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c) = lower;
       OMP_CLAUSE_CHAIN (c) = list;
       list = c;
     }
@@ -20926,6 +20963,7 @@ c_parser_omp_target_exit_data (location_t loc, c_parser *parser,
 	| (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_ALLOCATE)	\
 	| (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_DEFAULTMAP)	\
 	| (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_IN_REDUCTION)	\
+	| (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_THREAD_LIMIT)	\
 	| (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_IS_DEVICE_PTR))
 
 static bool
@@ -21016,31 +21054,31 @@ c_parser_omp_target (c_parser *parser, enum pragma_context context, bool *if_p)
 	  if (ret == NULL_TREE)
 	    return false;
 	  if (ccode == OMP_TEAMS)
-	    {
-	      /* For combined target teams, ensure the num_teams and
-		 thread_limit clause expressions are evaluated on the host,
-		 before entering the target construct.  */
-	      tree c;
-	      for (c = cclauses[C_OMP_CLAUSE_SPLIT_TEAMS];
-		   c; c = OMP_CLAUSE_CHAIN (c))
-		if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS
-		     || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREAD_LIMIT)
-		    && TREE_CODE (OMP_CLAUSE_OPERAND (c, 0)) != INTEGER_CST)
-		  {
-		    tree expr = OMP_CLAUSE_OPERAND (c, 0);
-		    tree tmp = create_tmp_var_raw (TREE_TYPE (expr));
-		    expr = build4 (TARGET_EXPR, TREE_TYPE (expr), tmp,
-				   expr, NULL_TREE, NULL_TREE);
-		    add_stmt (expr);
-		    OMP_CLAUSE_OPERAND (c, 0) = expr;
-		    tree tc = build_omp_clause (OMP_CLAUSE_LOCATION (c),
-						OMP_CLAUSE_FIRSTPRIVATE);
-		    OMP_CLAUSE_DECL (tc) = tmp;
-		    OMP_CLAUSE_CHAIN (tc)
-		      = cclauses[C_OMP_CLAUSE_SPLIT_TARGET];
-		    cclauses[C_OMP_CLAUSE_SPLIT_TARGET] = tc;
-		  }
-	    }
+	    /* For combined target teams, ensure the num_teams and
+	       thread_limit clause expressions are evaluated on the host,
+	       before entering the target construct.  */
+	    for (tree c = cclauses[C_OMP_CLAUSE_SPLIT_TEAMS];
+		 c; c = OMP_CLAUSE_CHAIN (c))
+	      if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS
+		  || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREAD_LIMIT)
+		for (int i = 0;
+		     i <= (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS); ++i)
+		  if (OMP_CLAUSE_OPERAND (c, i)
+		      && TREE_CODE (OMP_CLAUSE_OPERAND (c, i)) != INTEGER_CST)
+		    {
+		      tree expr = OMP_CLAUSE_OPERAND (c, i);
+		      tree tmp = create_tmp_var_raw (TREE_TYPE (expr));
+		      expr = build4 (TARGET_EXPR, TREE_TYPE (expr), tmp,
+				     expr, NULL_TREE, NULL_TREE);
+		      add_stmt (expr);
+		      OMP_CLAUSE_OPERAND (c, i) = expr;
+		      tree tc = build_omp_clause (OMP_CLAUSE_LOCATION (c),
+						  OMP_CLAUSE_FIRSTPRIVATE);
+		      OMP_CLAUSE_DECL (tc) = tmp;
+		      OMP_CLAUSE_CHAIN (tc)
+			= cclauses[C_OMP_CLAUSE_SPLIT_TARGET];
+		      cclauses[C_OMP_CLAUSE_SPLIT_TARGET] = tc;
+		    }
 	  tree stmt = make_node (OMP_TARGET);
 	  TREE_TYPE (stmt) = void_type_node;
 	  OMP_TARGET_CLAUSES (stmt) = cclauses[C_OMP_CLAUSE_SPLIT_TARGET];
diff --git a/gcc/c/c-tree.h b/gcc/c/c-tree.h
index a046c6b..f1dbbd5 100644
--- a/gcc/c/c-tree.h
+++ b/gcc/c/c-tree.h
@@ -598,6 +598,8 @@ extern tree finish_struct (location_t, tree, tree, tree,
 			   class c_struct_parse_info *);
 extern tree c_simulate_enum_decl (location_t, const char *,
 				  vec<string_int_pair> *);
+extern tree c_simulate_record_decl (location_t, const char *,
+				    array_slice<const tree>);
 extern struct c_arg_info *build_arg_info (void);
 extern struct c_arg_info *get_parm_info (bool, tree);
 extern tree grokfield (location_t, struct c_declarator *,
diff --git a/gcc/c/c-typeck.c b/gcc/c/c-typeck.c
index 0aac978..782414f 100644
--- a/gcc/c/c-typeck.c
+++ b/gcc/c/c-typeck.c
@@ -53,12 +53,13 @@ along with GCC; see the file COPYING3.  If not see
 #include "attribs.h"
 #include "asan.h"
 
-/* Possible cases of implicit bad conversions.  Used to select
-   diagnostic messages in convert_for_assignment.  */
+/* Possible cases of implicit conversions.  Used to select diagnostic messages
+   and control folding initializers in convert_for_assignment.  */
 enum impl_conv {
   ic_argpass,
   ic_assign,
   ic_init,
+  ic_init_const,
   ic_return
 };
 
@@ -6802,6 +6803,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
         pedwarn (LOCATION, OPT, AS);                                     \
         break;                                                           \
       case ic_init:                                                      \
+      case ic_init_const:                                                \
         pedwarn_init (LOCATION, OPT, IN);                                \
         break;                                                           \
       case ic_return:                                                    \
@@ -6838,6 +6840,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 	  warning_at (LOCATION, OPT, AS, QUALS);                         \
         break;                                                           \
       case ic_init:                                                      \
+      case ic_init_const:                                                \
 	if (PEDWARN)							 \
 	  pedwarn (LOCATION, OPT, IN, QUALS);                            \
 	else								 \
@@ -6886,6 +6889,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 	  break;
 
 	case ic_init:
+	case ic_init_const:
 	  parmno = -2;
 	  break;
 
@@ -6919,6 +6923,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 		     "%qT in assignment is invalid in C++", rhstype, type);
 	    break;
 	  case ic_init:
+	  case ic_init_const:
 	    pedwarn_init (location, OPT_Wc___compat, "enum conversion from "
 			  "%qT to %qT in initialization is invalid in C++",
 			  rhstype, type);
@@ -7029,7 +7034,8 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 	      && sanitize_flags_p (SANITIZE_FLOAT_CAST)))
 	in_late_binary_op = true;
       tree ret = convert_and_check (expr_loc != UNKNOWN_LOCATION
-				    ? expr_loc : location, type, orig_rhs);
+				    ? expr_loc : location, type, orig_rhs,
+				    errtype == ic_init_const);
       in_late_binary_op = save;
       return ret;
     }
@@ -7252,6 +7258,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 		break;
 	      }
 	    case ic_init:
+	    case ic_init_const:
 	      {
 		const char msg[] = G_("initialization from pointer to "
 				      "non-enclosed address space");
@@ -7296,6 +7303,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 			"a candidate for a format attribute");
 	    break;
 	  case ic_init:
+	  case ic_init_const:
 	    warning_at (location, OPT_Wsuggest_attribute_format,
 			"initialization left-hand side might be "
 			"a candidate for a format attribute");
@@ -7339,6 +7347,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 			  "incompatible scalar storage order", type, rhstype);
 	    break;
 	  case ic_init:
+	  case ic_init_const:
 	    /* Likewise.  */
 	    if (TREE_CODE (rhs) != CALL_EXPR
 		|| (t = get_callee_fndecl (rhs)) == NULL_TREE
@@ -7465,6 +7474,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 			     "differ in signedness", rhstype, type);
 		    break;
 		  case ic_init:
+		  case ic_init_const:
 		    pedwarn_init (location, OPT_Wpointer_sign,
 				  "pointer targets in initialization of %qT "
 				  "from %qT differ in signedness", type,
@@ -7530,6 +7540,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 			 type, rhstype);
 	      break;
 	    case ic_init:
+	    case ic_init_const:
 	      if (bltin)
 		pedwarn_init (location, OPT_Wincompatible_pointer_types,
 			      "initialization of %qT from pointer to "
@@ -7599,6 +7610,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 		     "without a cast", type, rhstype);
 	    break;
 	  case ic_init:
+	  case ic_init_const:
 	    pedwarn_init (location, OPT_Wint_conversion,
 			  "initialization of %qT from %qT makes pointer from "
 			  "integer without a cast", type, rhstype);
@@ -7635,6 +7647,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 		   "without a cast", type, rhstype);
 	  break;
 	case ic_init:
+	case ic_init_const:
 	  pedwarn_init (location, OPT_Wint_conversion,
 			"initialization of %qT from %qT makes integer from "
 			"pointer without a cast", type, rhstype);
@@ -7686,6 +7699,7 @@ convert_for_assignment (location_t location, location_t expr_loc, tree type,
 	break;
       }
     case ic_init:
+    case ic_init_const:
       {
 	const char msg[]
 	  = G_("incompatible types when initializing type %qT using type %qT");
@@ -8195,7 +8209,9 @@ digest_init (location_t init_loc, tree type, tree init, tree origtype,
       if (TREE_CODE (TREE_TYPE (inside_init)) == POINTER_TYPE)
 	inside_init = convert_for_assignment (init_loc, UNKNOWN_LOCATION,
 					      type, inside_init, origtype,
-					      ic_init, null_pointer_constant,
+					      (require_constant
+					       ? ic_init_const
+					       : ic_init), null_pointer_constant,
 					      NULL_TREE, NULL_TREE, 0);
       return inside_init;
     }
@@ -8215,7 +8231,8 @@ digest_init (location_t init_loc, tree type, tree init, tree origtype,
 			      inside_init);
       inside_init
 	= convert_for_assignment (init_loc, UNKNOWN_LOCATION, type,
-				  inside_init, origtype, ic_init,
+				  inside_init, origtype,
+				  require_constant ? ic_init_const : ic_init,
 				  null_pointer_constant, NULL_TREE, NULL_TREE,
 				  0);
 
diff --git a/gcc/c/gimple-parser.c b/gcc/c/gimple-parser.c
index f3d9935..32f22db 100644
--- a/gcc/c/gimple-parser.c
+++ b/gcc/c/gimple-parser.c
@@ -56,13 +56,13 @@ along with GCC; see the file COPYING3.  If not see
 #include "internal-fn.h"
 #include "cfg.h"
 #include "cfghooks.h"
+#include "bitmap.h"
 #include "cfganal.h"
 #include "tree-cfg.h"
 #include "gimple-iterator.h"
 #include "cfgloop.h"
 #include "tree-phinodes.h"
 #include "tree-into-ssa.h"
-#include "bitmap.h"
 
 
 /* GIMPLE parser state.  */
diff --git a/gcc/cfganal.c b/gcc/cfganal.c
index cec5abe..0cba612 100644
--- a/gcc/cfganal.c
+++ b/gcc/cfganal.c
@@ -362,33 +362,14 @@ control_dependences::set_control_dependence_map_bit (basic_block bb,
   if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
     return;
   gcc_assert (bb != EXIT_BLOCK_PTR_FOR_FN (cfun));
-  bitmap_set_bit (control_dependence_map[bb->index], edge_index);
+  bitmap_set_bit (&control_dependence_map[bb->index], edge_index);
 }
 
 /* Clear all control dependences for block BB.  */
 void
 control_dependences::clear_control_dependence_bitmap (basic_block bb)
 {
-  bitmap_clear (control_dependence_map[bb->index]);
-}
-
-/* Find the immediate postdominator PDOM of the specified basic block BLOCK.
-   This function is necessary because some blocks have negative numbers.  */
-
-static inline basic_block
-find_pdom (basic_block block)
-{
-  gcc_assert (block != ENTRY_BLOCK_PTR_FOR_FN (cfun));
-
-  if (block == EXIT_BLOCK_PTR_FOR_FN (cfun))
-    return EXIT_BLOCK_PTR_FOR_FN (cfun);
-  else
-    {
-      basic_block bb = get_immediate_dominator (CDI_POST_DOMINATORS, block);
-      if (! bb)
-	return EXIT_BLOCK_PTR_FOR_FN (cfun);
-      return bb;
-    }
+  bitmap_clear (&control_dependence_map[bb->index]);
 }
 
 /* Determine all blocks' control dependences on the given edge with edge_list
@@ -402,22 +383,14 @@ control_dependences::find_control_dependence (int edge_index)
 
   gcc_assert (get_edge_src (edge_index) != EXIT_BLOCK_PTR_FOR_FN (cfun));
 
-  /* For abnormal edges, we don't make current_block control
-     dependent because instructions that throw are always necessary
-     anyway.  */
-  edge e = find_edge (get_edge_src (edge_index), get_edge_dest (edge_index));
-  if (e->flags & EDGE_ABNORMAL)
-    return;
-
-  if (get_edge_src (edge_index) == ENTRY_BLOCK_PTR_FOR_FN (cfun))
-    ending_block = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
-  else
-    ending_block = find_pdom (get_edge_src (edge_index));
+  ending_block = get_immediate_dominator (CDI_POST_DOMINATORS,
+					  get_edge_src (edge_index));
 
   for (current_block = get_edge_dest (edge_index);
        current_block != ending_block
        && current_block != EXIT_BLOCK_PTR_FOR_FN (cfun);
-       current_block = find_pdom (current_block))
+       current_block = get_immediate_dominator (CDI_POST_DOMINATORS,
+						current_block))
     set_control_dependence_map_bit (current_block, edge_index);
 }
 
@@ -440,11 +413,23 @@ control_dependences::control_dependences ()
   FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun),
 		  EXIT_BLOCK_PTR_FOR_FN (cfun), next_bb)
     FOR_EACH_EDGE (e, ei, bb->succs)
-      m_el.quick_push (std::make_pair (e->src->index, e->dest->index));
+      {
+	/* For abnormal edges, we don't make current_block control
+	   dependent because instructions that throw are always necessary
+	   anyway.  */
+	if (e->flags & EDGE_ABNORMAL)
+	  {
+	    num_edges--;
+	    continue;
+	  }
+	m_el.quick_push (std::make_pair (e->src->index, e->dest->index));
+      }
 
+  bitmap_obstack_initialize (&m_bitmaps);
   control_dependence_map.create (last_basic_block_for_fn (cfun));
+  control_dependence_map.quick_grow (last_basic_block_for_fn (cfun));
   for (int i = 0; i < last_basic_block_for_fn (cfun); ++i)
-    control_dependence_map.quick_push (BITMAP_ALLOC (NULL));
+    bitmap_initialize (&control_dependence_map[i], &m_bitmaps);
   for (int i = 0; i < num_edges; ++i)
     find_control_dependence (i);
 
@@ -455,10 +440,9 @@ control_dependences::control_dependences ()
 
 control_dependences::~control_dependences ()
 {
-  for (unsigned i = 0; i < control_dependence_map.length (); ++i)
-    BITMAP_FREE (control_dependence_map[i]);
   control_dependence_map.release ();
   m_el.release ();
+  bitmap_obstack_release (&m_bitmaps);
 }
 
 /* Returns the bitmap of edges the basic-block I is dependent on.  */
@@ -466,7 +450,7 @@ control_dependences::~control_dependences ()
 bitmap
 control_dependences::get_edges_dependent_on (int i)
 {
-  return control_dependence_map[i];
+  return &control_dependence_map[i];
 }
 
 /* Returns the edge source with index I from the edge list.  */
diff --git a/gcc/cfganal.h b/gcc/cfganal.h
index 31ce56c..7ef4319 100644
--- a/gcc/cfganal.h
+++ b/gcc/cfganal.h
@@ -44,8 +44,9 @@ private:
   void set_control_dependence_map_bit (basic_block, int);
   void clear_control_dependence_bitmap (basic_block);
   void find_control_dependence (int);
-  vec<bitmap> control_dependence_map;
+  vec<bitmap_head> control_dependence_map;
   vec<std::pair<int, int> > m_el;
+  bitmap_obstack m_bitmaps;
 };
 
 extern bool mark_dfs_back_edges (void);
diff --git a/gcc/cfgexpand.c b/gcc/cfgexpand.c
index 8b067f9..eb6466f 100644
--- a/gcc/cfgexpand.c
+++ b/gcc/cfgexpand.c
@@ -4341,11 +4341,8 @@ avoid_deep_ter_for_debug (gimple *stmt, int depth)
 	  tree &vexpr = deep_ter_debug_map->get_or_insert (use);
 	  if (vexpr != NULL)
 	    continue;
-	  vexpr = make_node (DEBUG_EXPR_DECL);
+	  vexpr = build_debug_expr_decl (TREE_TYPE (use));
 	  gimple *def_temp = gimple_build_debug_bind (vexpr, use, g);
-	  DECL_ARTIFICIAL (vexpr) = 1;
-	  TREE_TYPE (vexpr) = TREE_TYPE (use);
-	  SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (use)));
 	  gimple_stmt_iterator gsi = gsi_for_stmt (g);
 	  gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
 	  avoid_deep_ter_for_debug (def_temp, 0);
@@ -5756,6 +5753,7 @@ expand_gimple_basic_block (basic_block bb, bool disable_tail_calls)
   rtx_insn *last;
   edge e;
   edge_iterator ei;
+  bool nondebug_stmt_seen = false;
 
   if (dump_file)
     fprintf (dump_file, "\n;; Generating RTL for gimple basic block %d\n",
@@ -5836,6 +5834,8 @@ expand_gimple_basic_block (basic_block bb, bool disable_tail_calls)
       basic_block new_bb;
 
       stmt = gsi_stmt (gsi);
+      if (!is_gimple_debug (stmt))
+	nondebug_stmt_seen = true;
 
       /* If this statement is a non-debug one, and we generate debug
 	 insns, then this one might be the last real use of a TERed
@@ -5898,18 +5898,17 @@ expand_gimple_basic_block (basic_block bb, bool disable_tail_calls)
 		       temporary.  */
 		    gimple *debugstmt;
 		    tree value = gimple_assign_rhs_to_tree (def);
-		    tree vexpr = make_node (DEBUG_EXPR_DECL);
+		    tree vexpr = build_debug_expr_decl (TREE_TYPE (value));
 		    rtx val;
 		    machine_mode mode;
 
 		    set_curr_insn_location (gimple_location (def));
 
-		    DECL_ARTIFICIAL (vexpr) = 1;
-		    TREE_TYPE (vexpr) = TREE_TYPE (value);
 		    if (DECL_P (value))
 		      mode = DECL_MODE (value);
 		    else
 		      mode = TYPE_MODE (TREE_TYPE (value));
+		    /* FIXME: Is setting the mode really necessary? */
 		    SET_DECL_MODE (vexpr, mode);
 
 		    val = gen_rtx_VAR_LOCATION
@@ -6090,7 +6089,7 @@ expand_gimple_basic_block (basic_block bb, bool disable_tail_calls)
   /* Expand implicit goto and convert goto_locus.  */
   FOR_EACH_EDGE (e, ei, bb->succs)
     {
-      if (e->goto_locus != UNKNOWN_LOCATION || !stmt || is_gimple_debug (stmt))
+      if (e->goto_locus != UNKNOWN_LOCATION || !nondebug_stmt_seen)
 	set_curr_insn_location (e->goto_locus);
       if ((e->flags & EDGE_FALLTHRU) && e->dest != bb->next_bb)
 	{
diff --git a/gcc/cfghooks.c b/gcc/cfghooks.c
index 6446e16..fa2dae2 100644
--- a/gcc/cfghooks.c
+++ b/gcc/cfghooks.c
@@ -1239,25 +1239,22 @@ lv_flush_pending_stmts (edge e)
     cfg_hooks->flush_pending_stmts (e);
 }
 
-/* Loop versioning uses the duplicate_loop_to_header_edge to create
+/* Loop versioning uses the duplicate_loop_body_to_header_edge to create
    a new version of the loop basic-blocks, the parameters here are
-   exactly the same as in duplicate_loop_to_header_edge or
-   tree_duplicate_loop_to_header_edge; while in tree-ssa there is
+   exactly the same as in duplicate_loop_body_to_header_edge or
+   tree_duplicate_loop_body_to_header_edge; while in tree-ssa there is
    additional work to maintain ssa information that's why there is
-   a need to call the tree_duplicate_loop_to_header_edge rather
-   than duplicate_loop_to_header_edge when we are in tree mode.  */
+   a need to call the tree_duplicate_loop_body_to_header_edge rather
+   than duplicate_loop_body_to_header_edge when we are in tree mode.  */
 bool
-cfg_hook_duplicate_loop_to_header_edge (class loop *loop, edge e,
-					unsigned int ndupl,
-					sbitmap wont_exit, edge orig,
-					vec<edge> *to_remove,
-					int flags)
+cfg_hook_duplicate_loop_body_to_header_edge (class loop *loop, edge e,
+					     unsigned int ndupl,
+					     sbitmap wont_exit, edge orig,
+					     vec<edge> *to_remove, int flags)
 {
-  gcc_assert (cfg_hooks->cfg_hook_duplicate_loop_to_header_edge);
-  return cfg_hooks->cfg_hook_duplicate_loop_to_header_edge (loop, e,
-							    ndupl, wont_exit,
-							    orig, to_remove,
-							    flags);
+  gcc_assert (cfg_hooks->cfg_hook_duplicate_loop_body_to_header_edge);
+  return cfg_hooks->cfg_hook_duplicate_loop_body_to_header_edge (
+    loop, e, ndupl, wont_exit, orig, to_remove, flags);
 }
 
 /* Conditional jumps are represented differently in trees and RTL,
diff --git a/gcc/cfghooks.h b/gcc/cfghooks.h
index 8645fe5..6df651f 100644
--- a/gcc/cfghooks.h
+++ b/gcc/cfghooks.h
@@ -166,10 +166,9 @@ struct cfg_hooks
 
   /* A hook for duplicating loop in CFG, currently this is used
      in loop versioning.  */
-  bool (*cfg_hook_duplicate_loop_to_header_edge) (class loop *, edge,
-						  unsigned, sbitmap,
-						  edge, vec<edge> *,
-						  int);
+  bool (*cfg_hook_duplicate_loop_body_to_header_edge) (class loop *, edge,
+						       unsigned, sbitmap, edge,
+						       vec<edge> *, int);
 
   /* Add condition to new basic block and update CFG used in loop
      versioning.  */
@@ -250,12 +249,11 @@ extern bool block_ends_with_condjump_p (const_basic_block bb);
 extern int flow_call_edges_add (sbitmap);
 extern void execute_on_growing_pred (edge);
 extern void execute_on_shrinking_pred (edge);
-extern bool cfg_hook_duplicate_loop_to_header_edge (class loop *loop, edge,
-						    unsigned int ndupl,
-						    sbitmap wont_exit,
-						    edge orig,
-						    vec<edge> *to_remove,
-						    int flags);
+extern bool
+cfg_hook_duplicate_loop_body_to_header_edge (class loop *loop, edge,
+					     unsigned int ndupl,
+					     sbitmap wont_exit, edge orig,
+					     vec<edge> *to_remove, int flags);
 
 extern void lv_flush_pending_stmts (edge);
 extern void extract_cond_bb_edges (basic_block, edge *, edge*);
diff --git a/gcc/cfgloop.c b/gcc/cfgloop.c
index 2ba9918..20c24c1 100644
--- a/gcc/cfgloop.c
+++ b/gcc/cfgloop.c
@@ -1567,19 +1567,17 @@ verify_loop_structure (void)
   /* Check irreducible loops.  */
   if (loops_state_satisfies_p (LOOPS_HAVE_MARKED_IRREDUCIBLE_REGIONS))
     {
-      auto_edge_flag saved_irr_mask (cfun);
-      /* Record old info.  */
-      auto_sbitmap irreds (last_basic_block_for_fn (cfun));
+      auto_edge_flag saved_edge_irr (cfun);
+      auto_bb_flag saved_bb_irr (cfun);
+      /* Save old info.  */
       FOR_EACH_BB_FN (bb, cfun)
 	{
 	  edge_iterator ei;
 	  if (bb->flags & BB_IRREDUCIBLE_LOOP)
-	    bitmap_set_bit (irreds, bb->index);
-	  else
-	    bitmap_clear_bit (irreds, bb->index);
+	    bb->flags |= saved_bb_irr;
 	  FOR_EACH_EDGE (e, ei, bb->succs)
 	    if (e->flags & EDGE_IRREDUCIBLE_LOOP)
-	      e->flags |= saved_irr_mask;
+	      e->flags |= saved_edge_irr;
 	}
 
       /* Recount it.  */
@@ -1591,34 +1589,35 @@ verify_loop_structure (void)
 	  edge_iterator ei;
 
 	  if ((bb->flags & BB_IRREDUCIBLE_LOOP)
-	      && !bitmap_bit_p (irreds, bb->index))
+	      && !(bb->flags & saved_bb_irr))
 	    {
 	      error ("basic block %d should be marked irreducible", bb->index);
 	      err = 1;
 	    }
 	  else if (!(bb->flags & BB_IRREDUCIBLE_LOOP)
-	      && bitmap_bit_p (irreds, bb->index))
+		   && (bb->flags & saved_bb_irr))
 	    {
 	      error ("basic block %d should not be marked irreducible", bb->index);
 	      err = 1;
 	    }
+	  bb->flags &= ~saved_bb_irr;
 	  FOR_EACH_EDGE (e, ei, bb->succs)
 	    {
 	      if ((e->flags & EDGE_IRREDUCIBLE_LOOP)
-		  && !(e->flags & saved_irr_mask))
+		  && !(e->flags & saved_edge_irr))
 		{
 		  error ("edge from %d to %d should be marked irreducible",
 			 e->src->index, e->dest->index);
 		  err = 1;
 		}
 	      else if (!(e->flags & EDGE_IRREDUCIBLE_LOOP)
-		       && (e->flags & saved_irr_mask))
+		       && (e->flags & saved_edge_irr))
 		{
 		  error ("edge from %d to %d should not be marked irreducible",
 			 e->src->index, e->dest->index);
 		  err = 1;
 		}
-	      e->flags &= ~saved_irr_mask;
+	      e->flags &= ~saved_edge_irr;
 	    }
 	}
     }
diff --git a/gcc/cfgloopmanip.c b/gcc/cfgloopmanip.c
index 82c242d..aa538a2 100644
--- a/gcc/cfgloopmanip.c
+++ b/gcc/cfgloopmanip.c
@@ -846,71 +846,6 @@ create_empty_loop_on_edge (edge entry_edge,
   return loop;
 }
 
-/* Make area between HEADER_EDGE and LATCH_EDGE a loop by connecting
-   latch to header and update loop tree and dominators
-   accordingly. Everything between them plus LATCH_EDGE destination must
-   be dominated by HEADER_EDGE destination, and back-reachable from
-   LATCH_EDGE source.  HEADER_EDGE is redirected to basic block SWITCH_BB,
-   FALSE_EDGE of SWITCH_BB to original destination of HEADER_EDGE and
-   TRUE_EDGE of SWITCH_BB to original destination of LATCH_EDGE.
-   Returns the newly created loop.  Frequencies and counts in the new loop
-   are scaled by FALSE_SCALE and in the old one by TRUE_SCALE.  */
-
-class loop *
-loopify (edge latch_edge, edge header_edge,
-	 basic_block switch_bb, edge true_edge, edge false_edge,
-	 bool redirect_all_edges, profile_probability true_scale,
-	 profile_probability false_scale)
-{
-  basic_block succ_bb = latch_edge->dest;
-  basic_block pred_bb = header_edge->src;
-  class loop *loop = alloc_loop ();
-  class loop *outer = loop_outer (succ_bb->loop_father);
-  profile_count cnt;
-
-  loop->header = header_edge->dest;
-  loop->latch = latch_edge->src;
-
-  cnt = header_edge->count ();
-
-  /* Redirect edges.  */
-  loop_redirect_edge (latch_edge, loop->header);
-  loop_redirect_edge (true_edge, succ_bb);
-
-  /* During loop versioning, one of the switch_bb edge is already properly
-     set. Do not redirect it again unless redirect_all_edges is true.  */
-  if (redirect_all_edges)
-    {
-      loop_redirect_edge (header_edge, switch_bb);
-      loop_redirect_edge (false_edge, loop->header);
-
-      /* Update dominators.  */
-      set_immediate_dominator (CDI_DOMINATORS, switch_bb, pred_bb);
-      set_immediate_dominator (CDI_DOMINATORS, loop->header, switch_bb);
-    }
-
-  set_immediate_dominator (CDI_DOMINATORS, succ_bb, switch_bb);
-
-  /* Compute new loop.  */
-  add_loop (loop, outer);
-
-  /* Add switch_bb to appropriate loop.  */
-  if (switch_bb->loop_father)
-    remove_bb_from_loops (switch_bb);
-  add_bb_to_loop (switch_bb, outer);
-
-  /* Fix counts.  */
-  if (redirect_all_edges)
-    {
-      switch_bb->count = cnt;
-    }
-  scale_loop_frequencies (loop, false_scale);
-  scale_loop_frequencies (succ_bb->loop_father, true_scale);
-  update_dominators_in_loop (loop);
-
-  return loop;
-}
-
 /* Remove the latch edge of a LOOP and update loops to indicate that
    the LOOP was removed.  After this function, original loop latch will
    have no successor, which caller is expected to fix somehow.
@@ -1124,10 +1059,9 @@ can_duplicate_loop_p (const class loop *loop)
    impossible.  */
 
 bool
-duplicate_loop_to_header_edge (class loop *loop, edge e,
-			       unsigned int ndupl, sbitmap wont_exit,
-			       edge orig, vec<edge> *to_remove,
-			       int flags)
+duplicate_loop_body_to_header_edge (class loop *loop, edge e,
+				    unsigned int ndupl, sbitmap wont_exit,
+				    edge orig, vec<edge> *to_remove, int flags)
 {
   class loop *target, *aloop;
   class loop **orig_loops;
@@ -1681,7 +1615,7 @@ loop_version (class loop *loop,
 	      bool place_after)
 {
   basic_block first_head, second_head;
-  edge entry, latch_edge, true_edge, false_edge;
+  edge entry, latch_edge;
   int irred_flag;
   class loop *nloop;
   basic_block cond_bb;
@@ -1694,19 +1628,36 @@ loop_version (class loop *loop,
   /* Note down head of loop as first_head.  */
   first_head = entry->dest;
 
-  /* Duplicate loop.  */
-  if (!cfg_hook_duplicate_loop_to_header_edge (loop, entry, 1,
-					       NULL, NULL, NULL, 0))
+  /* 1) Duplicate loop on the entry edge.  */
+  if (!cfg_hook_duplicate_loop_body_to_header_edge (loop, entry, 1, NULL, NULL,
+						    NULL, 0))
     {
       entry->flags |= irred_flag;
       return NULL;
     }
 
+  /* 2) loopify the duplicated new loop. */
+  latch_edge = single_succ_edge (get_bb_copy (loop->latch));
+  nloop = alloc_loop ();
+  class loop *outer = loop_outer (latch_edge->dest->loop_father);
+  edge new_header_edge = single_pred_edge (get_bb_copy (loop->header));
+  nloop->header = new_header_edge->dest;
+  nloop->latch = latch_edge->src;
+  loop_redirect_edge (latch_edge, nloop->header);
+
+  /* Compute new loop.  */
+  add_loop (nloop, outer);
+  copy_loop_info (loop, nloop);
+  set_loop_copy (loop, nloop);
+
+  /* loopify redirected latch_edge. Update its PENDING_STMTS.  */
+  lv_flush_pending_stmts (latch_edge);
+
   /* After duplication entry edge now points to new loop head block.
      Note down new head as second_head.  */
   second_head = entry->dest;
 
-  /* Split loop entry edge and insert new block with cond expr.  */
+  /* 3) Split loop entry edge and insert new block with cond expr.  */
   cond_bb =  lv_adjust_loop_entry_edge (first_head, second_head,
 					entry, cond_expr, then_prob, else_prob);
   if (condition_bb)
@@ -1718,24 +1669,17 @@ loop_version (class loop *loop,
       return NULL;
     }
 
-  latch_edge = single_succ_edge (get_bb_copy (loop->latch));
-
-  extract_cond_bb_edges (cond_bb, &true_edge, &false_edge);
-  nloop = loopify (latch_edge,
-		   single_pred_edge (get_bb_copy (loop->header)),
-		   cond_bb, true_edge, false_edge,
-		   false /* Do not redirect all edges.  */,
-		   then_scale, else_scale);
-
-  copy_loop_info (loop, nloop);
-  set_loop_copy (loop, nloop);
+  /* Add cond_bb to appropriate loop.  */
+  if (cond_bb->loop_father)
+    remove_bb_from_loops (cond_bb);
+  add_bb_to_loop (cond_bb, outer);
 
-  /* loopify redirected latch_edge. Update its PENDING_STMTS.  */
-  lv_flush_pending_stmts (latch_edge);
+  /* 4) Scale the original loop and new loop frequency.  */
+  scale_loop_frequencies (loop, then_scale);
+  scale_loop_frequencies (nloop, else_scale);
+  update_dominators_in_loop (loop);
+  update_dominators_in_loop (nloop);
 
-  /* loopify redirected condition_bb's succ edge. Update its PENDING_STMTS.  */
-  extract_cond_bb_edges (cond_bb, &true_edge, &false_edge);
-  lv_flush_pending_stmts (false_edge);
   /* Adjust irreducible flag.  */
   if (irred_flag)
     {
diff --git a/gcc/cfgloopmanip.h b/gcc/cfgloopmanip.h
index 07d5f92..8dc5de9 100644
--- a/gcc/cfgloopmanip.h
+++ b/gcc/cfgloopmanip.h
@@ -42,18 +42,15 @@ extern void scale_loop_profile (class loop *, profile_probability, gcov_type);
 extern edge create_empty_if_region_on_edge (edge, tree);
 extern class loop *create_empty_loop_on_edge (edge, tree, tree, tree, tree,
 					       tree *, tree *, class loop *);
-extern class loop *loopify (edge, edge,
-			     basic_block, edge, edge, bool,
-			     profile_probability, profile_probability);
 extern void unloop (class loop *, bool *, bitmap);
 extern void copy_loop_info (class loop *loop, class loop *target);
 extern class loop * duplicate_loop (class loop *, class loop *,
 				     class loop * = NULL);
 extern void duplicate_subloops (class loop *, class loop *);
 extern bool can_duplicate_loop_p (const class loop *loop);
-extern bool duplicate_loop_to_header_edge (class loop *, edge,
-					   unsigned, sbitmap, edge,
- 					   vec<edge> *, int);
+extern bool
+duplicate_loop_body_to_header_edge (class loop *, edge, unsigned, sbitmap, edge,
+				    vec<edge> *, int);
 extern bool mfb_keep_just (edge);
 basic_block create_preheader (class loop *, int);
 extern void create_preheaders (int);
diff --git a/gcc/cfgrtl.c b/gcc/cfgrtl.c
index 4f3b1e8..e3a724b 100644
--- a/gcc/cfgrtl.c
+++ b/gcc/cfgrtl.c
@@ -5344,7 +5344,7 @@ struct cfg_hooks cfg_layout_rtl_cfg_hooks = {
   rtl_flow_call_edges_add,
   NULL, /* execute_on_growing_pred */
   NULL, /* execute_on_shrinking_pred */
-  duplicate_loop_to_header_edge, /* duplicate loop for trees */
+  duplicate_loop_body_to_header_edge, /* duplicate loop for trees */
   rtl_lv_add_condition_to_bb, /* lv_add_condition_to_bb */
   NULL, /* lv_adjust_loop_header_phi*/
   rtl_extract_cond_bb_edges, /* extract_cond_bb_edges */
diff --git a/gcc/cgraph.c b/gcc/cgraph.c
index de07865..466b66d 100644
--- a/gcc/cgraph.c
+++ b/gcc/cgraph.c
@@ -2203,6 +2203,9 @@ cgraph_node::dump (FILE *f)
     fprintf (f, " %soperator_delete",
 	     DECL_IS_REPLACEABLE_OPERATOR (decl) ? "replaceable_" : "");
 
+  if (DECL_STATIC_CHAIN (decl))
+    fprintf (f, " static_chain");
+
   fprintf (f, "\n");
 
   if (thunk)
@@ -2611,6 +2614,53 @@ cgraph_node::set_malloc_flag (bool malloc_p)
   return changed;
 }
 
+/* Worker to set noreturng flag.  */
+static void
+set_noreturn_flag_1 (cgraph_node *node, bool noreturn_p, bool *changed)
+{
+  if (noreturn_p && !TREE_THIS_VOLATILE (node->decl))
+    {
+      TREE_THIS_VOLATILE (node->decl) = true;
+      *changed = true;
+    }
+
+  ipa_ref *ref;
+  FOR_EACH_ALIAS (node, ref)
+    {
+      cgraph_node *alias = dyn_cast<cgraph_node *> (ref->referring);
+      if (!noreturn_p || alias->get_availability () > AVAIL_INTERPOSABLE)
+	set_noreturn_flag_1 (alias, noreturn_p, changed);
+    }
+
+  for (cgraph_edge *e = node->callers; e; e = e->next_caller)
+    if (e->caller->thunk
+	&& (!noreturn_p || e->caller->get_availability () > AVAIL_INTERPOSABLE))
+      set_noreturn_flag_1 (e->caller, noreturn_p, changed);
+}
+
+/* Set TREE_THIS_VOLATILE on NODE's decl and on NODE's aliases if any.  */
+
+bool
+cgraph_node::set_noreturn_flag (bool noreturn_p)
+{
+  bool changed = false;
+
+  if (!noreturn_p || get_availability () > AVAIL_INTERPOSABLE)
+    set_noreturn_flag_1 (this, noreturn_p, &changed);
+  else
+    {
+      ipa_ref *ref;
+
+      FOR_EACH_ALIAS (this, ref)
+	{
+	  cgraph_node *alias = dyn_cast<cgraph_node *> (ref->referring);
+	  if (!noreturn_p || alias->get_availability () > AVAIL_INTERPOSABLE)
+	    set_noreturn_flag_1 (alias, noreturn_p, &changed);
+	}
+    }
+  return changed;
+}
+
 /* Worker to set_const_flag.  */
 
 static void
diff --git a/gcc/cgraph.h b/gcc/cgraph.h
index 4cdb373..e42e305 100644
--- a/gcc/cgraph.h
+++ b/gcc/cgraph.h
@@ -404,7 +404,8 @@ public:
   inline bool
   can_be_discarded_p (void)
   {
-    return (DECL_EXTERNAL (decl)
+    return ((DECL_EXTERNAL (decl)
+	     && !in_other_partition)
 	    || ((get_comdat_group ()
 		 || DECL_COMMON (decl)
 		 || (DECL_SECTION_NAME (decl) && DECL_WEAK (decl)))
@@ -1166,6 +1167,10 @@ struct GTY((tag ("SYMTAB_FUNCTION"))) cgraph_node : public symtab_node
      if any.  */
   bool set_malloc_flag (bool malloc_p);
 
+  /* SET TREE_THIS_VOLATILE on cgraph_node's decl and on aliases of the node
+     if any.  */
+  bool set_noreturn_flag (bool noreturn_p);
+
   /* If SET_CONST is true, mark function, aliases and thunks to be ECF_CONST.
     If SET_CONST if false, clear the flag.
 
diff --git a/gcc/common.opt b/gcc/common.opt
index a2af7fb..de9b848 100644
--- a/gcc/common.opt
+++ b/gcc/common.opt
@@ -83,7 +83,7 @@ int flag_gen_aux_info = 0
 Variable
 int flag_shlib
 
-; These two are really VEC(char_p,heap) *.
+; These three are really VEC(char_p,heap) *.
 
 Variable
 void *flag_instrument_functions_exclude_functions
@@ -91,6 +91,9 @@ void *flag_instrument_functions_exclude_functions
 Variable
 void *flag_instrument_functions_exclude_files
 
+Variable
+void *flag_ignored_attributes
+
 ; Generic structs (e.g. templates not explicitly specialized)
 ; may not have a compilation unit associated with them, and so
 ; may need to be treated differently from ordinary structs.
@@ -549,6 +552,10 @@ Wattributes
 Common Var(warn_attributes) Init(1) Warning
 Warn about inappropriate attribute usage.
 
+Wattributes=
+Common Joined
+Do not warn about specified attributes.
+
 Wattribute-alias
 Common Alias(Wattribute_alias=, 1, 0) Warning
 Warn about type safety and similar errors and mismatches in declarations with alias attributes.
@@ -1365,6 +1372,10 @@ fdiagnostics-format=
 Common Joined RejectNegative Enum(diagnostics_output_format)
 -fdiagnostics-format=[text|json]	Select output format.
 
+fdiagnostics-escape-format=
+Common Joined RejectNegative Enum(diagnostics_escape_format)
+-fdiagnostics-escape-format=[unicode|bytes]	Select how to escape non-printable-ASCII bytes in the source for diagnostics that suggest it.
+
 ; Required for these enum values.
 SourceInclude
 diagnostic.h
@@ -1379,6 +1390,15 @@ EnumValue
 Enum(diagnostics_column_unit) String(byte) Value(DIAGNOSTICS_COLUMN_UNIT_BYTE)
 
 Enum
+Name(diagnostics_escape_format) Type(int)
+
+EnumValue
+Enum(diagnostics_escape_format) String(unicode) Value(DIAGNOSTICS_ESCAPE_FORMAT_UNICODE)
+
+EnumValue
+Enum(diagnostics_escape_format) String(bytes) Value(DIAGNOSTICS_ESCAPE_FORMAT_BYTES)
+
+Enum
 Name(diagnostics_output_format) Type(int)
 
 EnumValue
@@ -1722,6 +1742,14 @@ fguess-branch-probability
 Common Var(flag_guess_branch_prob) Optimization
 Enable guessing of branch probabilities.
 
+fharden-compares
+Common Var(flag_harden_compares) Optimization
+Harden conditionals not used in branches, checking reversed conditions.
+
+fharden-conditional-branches
+Common Var(flag_harden_conditional_branches) Optimization
+Harden conditional branches by checking reversed conditions.
+
 ; Nonzero means ignore `#ident' directives.  0 means handle them.
 ; Generate position-independent code for executables if possible
 ; On SVR4 targets, it also controls whether or not to emit a
@@ -3284,7 +3312,7 @@ Common Driver JoinedOrMissing Negative(gvms)
 Generate debug information in extended STABS format.
 
 gstatement-frontiers
-Common Driver Var(debug_nonbind_markers_p)
+Common Driver Var(debug_nonbind_markers_p) PerFunction
 Emit progressive recommended breakpoint locations.
 
 gstrict-dwarf
diff --git a/gcc/common/config/riscv/riscv-common.c b/gcc/common/config/riscv/riscv-common.c
index 10868fd4..b8dd0ae 100644
--- a/gcc/common/config/riscv/riscv-common.c
+++ b/gcc/common/config/riscv/riscv-common.c
@@ -101,6 +101,11 @@ static const struct riscv_ext_version riscv_ext_version_table[] =
   {"zifencei", ISA_SPEC_CLASS_20191213, 2, 0},
   {"zifencei", ISA_SPEC_CLASS_20190608, 2, 0},
 
+  {"zba", ISA_SPEC_CLASS_NONE, 1, 0},
+  {"zbb", ISA_SPEC_CLASS_NONE, 1, 0},
+  {"zbc", ISA_SPEC_CLASS_NONE, 1, 0},
+  {"zbs", ISA_SPEC_CLASS_NONE, 1, 0},
+
   /* Terminate the list.  */
   {NULL, ISA_SPEC_CLASS_NONE, 0, 0}
 };
@@ -197,7 +202,6 @@ multi_letter_subset_rank (const std::string &subset)
       high_order = 1;
       break;
     case 'z':
-      gcc_assert (subset.length () > 2);
       high_order = 2;
       break;
     case 'x':
@@ -418,7 +422,7 @@ riscv_subset_list::to_string (bool version_p) const
   /* Skip since older binutils doesn't recognize zicsr.  */
   skip_zicsr = true;
 #endif
-#ifndef HAVE_AS_MARCH_ZIFENCE
+#ifndef HAVE_AS_MARCH_ZIFENCEI
   /* Skip since older binutils doesn't recognize zifencei, we made
      a mistake in that binutils 2.35 supports zicsr but not zifencei.  */
   skip_zifencei = true;
@@ -906,6 +910,11 @@ static const riscv_ext_flag_table_t riscv_ext_flag_table[] =
   {"zicsr",    &gcc_options::x_riscv_zi_subext, MASK_ZICSR},
   {"zifencei", &gcc_options::x_riscv_zi_subext, MASK_ZIFENCEI},
 
+  {"zba",    &gcc_options::x_riscv_zb_subext, MASK_ZBA},
+  {"zbb",    &gcc_options::x_riscv_zb_subext, MASK_ZBB},
+  {"zbc",    &gcc_options::x_riscv_zb_subext, MASK_ZBC},
+  {"zbs",    &gcc_options::x_riscv_zb_subext, MASK_ZBS},
+
   {NULL, NULL, 0}
 };
 
diff --git a/gcc/config.gcc b/gcc/config.gcc
index fb1f06f..edd1265 100644
--- a/gcc/config.gcc
+++ b/gcc/config.gcc
@@ -490,7 +490,7 @@ powerpc*-*-*)
 	extra_headers="${extra_headers} xmmintrin.h mm_malloc.h emmintrin.h"
 	extra_headers="${extra_headers} mmintrin.h x86intrin.h"
 	extra_headers="${extra_headers} pmmintrin.h tmmintrin.h smmintrin.h"
-	extra_headers="${extra_headers} nmmintrin.h"
+	extra_headers="${extra_headers} nmmintrin.h immintrin.h x86gprintrin.h"
 	extra_headers="${extra_headers} ppu_intrinsics.h spu2vmx.h vec_types.h si2vmx.h"
 	extra_headers="${extra_headers} amo.h"
 	case x$with_cpu in
@@ -681,9 +681,9 @@ case ${target} in
       *-*-darwin[4-9]* | *-*-darwin1[0-9]*)
         macos_min=`expr $darwin_maj - 4`
         ;;
-      *-*-darwin20*)
-        # Darwin 20 corresponds to macOS 11.
-        macos_maj=11
+      *-*-darwin2*)
+        # Darwin 20 corresponds to macOS 11, Darwin 21 to macOS 12.
+        macos_maj=`expr $darwin_maj - 9`
         def_ld64=609.0
         ;;
       *-*-darwin)
@@ -5088,16 +5088,6 @@ case "${target}" in
 			(at="/opt/$with_advance_toolchain"
 			 echo "/* Use Advance Toolchain $at */"
 			 echo
-			 echo "#undef  LINK_OS_EXTRA_SPEC32"
-			 echo "#define LINK_OS_EXTRA_SPEC32" \
-			      "\"%(link_os_new_dtags)" \
-			      "-rpath $prefix/lib -rpath $at/lib\""
-			 echo
-			 echo "#undef  LINK_OS_EXTRA_SPEC64"
-			 echo "#define LINK_OS_EXTRA_SPEC64" \
-			      "\"%(link_os_new_dtags)" \
-			      "-rpath $prefix/lib64 -rpath $at/lib64\""
-			 echo
 			 echo "#undef  LINK_OS_NEW_DTAGS_SPEC"
 			 echo "#define LINK_OS_NEW_DTAGS_SPEC" \
 			      "\"--enable-new-dtags\""
diff --git a/gcc/config.host b/gcc/config.host
index 0a02c33..81ff7ed 100644
--- a/gcc/config.host
+++ b/gcc/config.host
@@ -99,7 +99,8 @@ case ${host} in
 esac
 
 case ${host} in
-  aarch64*-*-freebsd* | aarch64*-*-linux* | aarch64*-*-fuchsia*)
+  aarch64*-*-freebsd* | aarch64*-*-linux* | aarch64*-*-fuchsia* |\
+  aarch64*-*-darwin*)
     case ${target} in
       aarch64*-*-*)
 	host_extra_gcc_objs="driver-aarch64.o"
@@ -251,6 +252,10 @@ case ${host} in
     host_extra_gcc_objs="${host_extra_gcc_objs} driver-mingw32.o"
     host_lto_plugin_soname=liblto_plugin.dll
     ;;
+  aarch64*-*-darwin*)
+    out_host_hook_obj="${out_host_hook_obj} host-aarch64-darwin.o"
+    host_xmake_file="${host_xmake_file} aarch64/x-darwin"
+    ;;
   i[34567]86-*-darwin* | x86_64-*-darwin*)
     out_host_hook_obj="${out_host_hook_obj} host-i386-darwin.o"
     host_xmake_file="${host_xmake_file} i386/x-darwin"
diff --git a/gcc/config.in b/gcc/config.in
index b5bec39..d6adc72 100644
--- a/gcc/config.in
+++ b/gcc/config.in
@@ -2226,6 +2226,12 @@
 #endif
 
 
+/* Define to 1 if ld64 supports '-platform_version'. */
+#ifndef USED_FOR_TARGET
+#undef LD64_HAS_PLATFORM_VERSION
+#endif
+
+
 /* Define to ld64 version. */
 #ifndef USED_FOR_TARGET
 #undef LD64_VERSION
diff --git a/gcc/config/aarch64/aarch64-builtins.c b/gcc/config/aarch64/aarch64-builtins.c
index a815e4c..303e1e5 100644
--- a/gcc/config/aarch64/aarch64-builtins.c
+++ b/gcc/config/aarch64/aarch64-builtins.c
@@ -75,6 +75,54 @@
 #define bf_UP    E_BFmode
 #define v4bf_UP  E_V4BFmode
 #define v8bf_UP  E_V8BFmode
+#define v2x8qi_UP  E_V2x8QImode
+#define v2x4hi_UP  E_V2x4HImode
+#define v2x4hf_UP  E_V2x4HFmode
+#define v2x4bf_UP  E_V2x4BFmode
+#define v2x2si_UP  E_V2x2SImode
+#define v2x2sf_UP  E_V2x2SFmode
+#define v2x1di_UP  E_V2x1DImode
+#define v2x1df_UP  E_V2x1DFmode
+#define v2x16qi_UP E_V2x16QImode
+#define v2x8hi_UP  E_V2x8HImode
+#define v2x8hf_UP  E_V2x8HFmode
+#define v2x8bf_UP  E_V2x8BFmode
+#define v2x4si_UP  E_V2x4SImode
+#define v2x4sf_UP  E_V2x4SFmode
+#define v2x2di_UP  E_V2x2DImode
+#define v2x2df_UP  E_V2x2DFmode
+#define v3x8qi_UP  E_V3x8QImode
+#define v3x4hi_UP  E_V3x4HImode
+#define v3x4hf_UP  E_V3x4HFmode
+#define v3x4bf_UP  E_V3x4BFmode
+#define v3x2si_UP  E_V3x2SImode
+#define v3x2sf_UP  E_V3x2SFmode
+#define v3x1di_UP  E_V3x1DImode
+#define v3x1df_UP  E_V3x1DFmode
+#define v3x16qi_UP E_V3x16QImode
+#define v3x8hi_UP  E_V3x8HImode
+#define v3x8hf_UP  E_V3x8HFmode
+#define v3x8bf_UP  E_V3x8BFmode
+#define v3x4si_UP  E_V3x4SImode
+#define v3x4sf_UP  E_V3x4SFmode
+#define v3x2di_UP  E_V3x2DImode
+#define v3x2df_UP  E_V3x2DFmode
+#define v4x8qi_UP  E_V4x8QImode
+#define v4x4hi_UP  E_V4x4HImode
+#define v4x4hf_UP  E_V4x4HFmode
+#define v4x4bf_UP  E_V4x4BFmode
+#define v4x2si_UP  E_V4x2SImode
+#define v4x2sf_UP  E_V4x2SFmode
+#define v4x1di_UP  E_V4x1DImode
+#define v4x1df_UP  E_V4x1DFmode
+#define v4x16qi_UP E_V4x16QImode
+#define v4x8hi_UP  E_V4x8HImode
+#define v4x8hf_UP  E_V4x8HFmode
+#define v4x8bf_UP  E_V4x8BFmode
+#define v4x4si_UP  E_V4x4SImode
+#define v4x4sf_UP  E_V4x4SFmode
+#define v4x2di_UP  E_V4x2DImode
+#define v4x2df_UP  E_V4x2DFmode
 #define UP(X) X##_UP
 
 #define SIMD_MAX_BUILTIN_ARGS 5
@@ -228,7 +276,6 @@ aarch64_types_binop_pppu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
   = { qualifier_poly, qualifier_poly, qualifier_poly, qualifier_unsigned };
 #define TYPES_TERNOP_PPPU (aarch64_types_binop_pppu_qualifiers)
 
-
 static enum aarch64_type_qualifiers
 aarch64_types_quadop_lane_pair_qualifiers[SIMD_MAX_BUILTIN_ARGS]
   = { qualifier_none, qualifier_none, qualifier_none,
@@ -265,10 +312,6 @@ aarch64_types_quadopu_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
 #define TYPES_QUADOPUI (aarch64_types_quadopu_imm_qualifiers)
 
 static enum aarch64_type_qualifiers
-aarch64_types_binop_imm_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
-  = { qualifier_poly, qualifier_none, qualifier_immediate };
-#define TYPES_GETREGP (aarch64_types_binop_imm_p_qualifiers)
-static enum aarch64_type_qualifiers
 aarch64_types_binop_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
   = { qualifier_none, qualifier_none, qualifier_immediate };
 #define TYPES_GETREG (aarch64_types_binop_imm_qualifiers)
@@ -292,10 +335,6 @@ aarch64_types_shift2_to_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS]
 #define TYPES_SHIFT2IMM_UUSS (aarch64_types_shift2_to_unsigned_qualifiers)
 
 static enum aarch64_type_qualifiers
-aarch64_types_ternop_s_imm_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
-  = { qualifier_none, qualifier_none, qualifier_poly, qualifier_immediate};
-#define TYPES_SETREGP (aarch64_types_ternop_s_imm_p_qualifiers)
-static enum aarch64_type_qualifiers
 aarch64_types_ternop_s_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
   = { qualifier_none, qualifier_none, qualifier_none, qualifier_immediate};
 #define TYPES_SETREG (aarch64_types_ternop_s_imm_qualifiers)
@@ -314,27 +353,37 @@ aarch64_types_unsigned_shiftacc_qualifiers[SIMD_MAX_BUILTIN_ARGS]
       qualifier_immediate };
 #define TYPES_USHIFTACC (aarch64_types_unsigned_shiftacc_qualifiers)
 
-
-static enum aarch64_type_qualifiers
-aarch64_types_combine_qualifiers[SIMD_MAX_BUILTIN_ARGS]
-  = { qualifier_none, qualifier_none, qualifier_none };
-#define TYPES_COMBINE (aarch64_types_combine_qualifiers)
-
-static enum aarch64_type_qualifiers
-aarch64_types_combine_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
-  = { qualifier_poly, qualifier_poly, qualifier_poly };
-#define TYPES_COMBINEP (aarch64_types_combine_p_qualifiers)
-
 static enum aarch64_type_qualifiers
 aarch64_types_load1_qualifiers[SIMD_MAX_BUILTIN_ARGS]
   = { qualifier_none, qualifier_const_pointer_map_mode };
 #define TYPES_LOAD1 (aarch64_types_load1_qualifiers)
 #define TYPES_LOADSTRUCT (aarch64_types_load1_qualifiers)
 static enum aarch64_type_qualifiers
+aarch64_types_load1_u_qualifiers[SIMD_MAX_BUILTIN_ARGS]
+  = { qualifier_unsigned, qualifier_const_pointer_map_mode };
+#define TYPES_LOAD1_U (aarch64_types_load1_u_qualifiers)
+#define TYPES_LOADSTRUCT_U (aarch64_types_load1_u_qualifiers)
+static enum aarch64_type_qualifiers
+aarch64_types_load1_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
+  = { qualifier_poly, qualifier_const_pointer_map_mode };
+#define TYPES_LOAD1_P (aarch64_types_load1_p_qualifiers)
+#define TYPES_LOADSTRUCT_P (aarch64_types_load1_p_qualifiers)
+
+static enum aarch64_type_qualifiers
 aarch64_types_loadstruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
   = { qualifier_none, qualifier_const_pointer_map_mode,
       qualifier_none, qualifier_struct_load_store_lane_index };
 #define TYPES_LOADSTRUCT_LANE (aarch64_types_loadstruct_lane_qualifiers)
+static enum aarch64_type_qualifiers
+aarch64_types_loadstruct_lane_u_qualifiers[SIMD_MAX_BUILTIN_ARGS]
+  = { qualifier_unsigned, qualifier_const_pointer_map_mode,
+      qualifier_unsigned, qualifier_struct_load_store_lane_index };
+#define TYPES_LOADSTRUCT_LANE_U (aarch64_types_loadstruct_lane_u_qualifiers)
+static enum aarch64_type_qualifiers
+aarch64_types_loadstruct_lane_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
+  = { qualifier_poly, qualifier_const_pointer_map_mode,
+      qualifier_poly, qualifier_struct_load_store_lane_index };
+#define TYPES_LOADSTRUCT_LANE_P (aarch64_types_loadstruct_lane_p_qualifiers)
 
 static enum aarch64_type_qualifiers
 aarch64_types_bsl_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
@@ -358,19 +407,36 @@ aarch64_types_bsl_u_qualifiers[SIMD_MAX_BUILTIN_ARGS]
    qualifier_map_mode | qualifier_pointer to build a pointer to the
    element type of the vector.  */
 static enum aarch64_type_qualifiers
-aarch64_types_store1_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
-  = { qualifier_void, qualifier_pointer_map_mode, qualifier_poly };
-#define TYPES_STORE1P (aarch64_types_store1_p_qualifiers)
-static enum aarch64_type_qualifiers
 aarch64_types_store1_qualifiers[SIMD_MAX_BUILTIN_ARGS]
   = { qualifier_void, qualifier_pointer_map_mode, qualifier_none };
 #define TYPES_STORE1 (aarch64_types_store1_qualifiers)
 #define TYPES_STORESTRUCT (aarch64_types_store1_qualifiers)
 static enum aarch64_type_qualifiers
+aarch64_types_store1_u_qualifiers[SIMD_MAX_BUILTIN_ARGS]
+  = { qualifier_void, qualifier_pointer_map_mode, qualifier_unsigned };
+#define TYPES_STORE1_U (aarch64_types_store1_u_qualifiers)
+#define TYPES_STORESTRUCT_U (aarch64_types_store1_u_qualifiers)
+static enum aarch64_type_qualifiers
+aarch64_types_store1_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
+  = { qualifier_void, qualifier_pointer_map_mode, qualifier_poly };
+#define TYPES_STORE1_P (aarch64_types_store1_p_qualifiers)
+#define TYPES_STORESTRUCT_P (aarch64_types_store1_p_qualifiers)
+
+static enum aarch64_type_qualifiers
 aarch64_types_storestruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
   = { qualifier_void, qualifier_pointer_map_mode,
       qualifier_none, qualifier_struct_load_store_lane_index };
 #define TYPES_STORESTRUCT_LANE (aarch64_types_storestruct_lane_qualifiers)
+static enum aarch64_type_qualifiers
+aarch64_types_storestruct_lane_u_qualifiers[SIMD_MAX_BUILTIN_ARGS]
+  = { qualifier_void, qualifier_pointer_map_mode,
+      qualifier_unsigned, qualifier_struct_load_store_lane_index };
+#define TYPES_STORESTRUCT_LANE_U (aarch64_types_storestruct_lane_u_qualifiers)
+static enum aarch64_type_qualifiers
+aarch64_types_storestruct_lane_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
+  = { qualifier_void, qualifier_pointer_map_mode,
+      qualifier_poly, qualifier_struct_load_store_lane_index };
+#define TYPES_STORESTRUCT_LANE_P (aarch64_types_storestruct_lane_p_qualifiers)
 
 #define CF0(N, X) CODE_FOR_aarch64_##N##X
 #define CF1(N, X) CODE_FOR_##N##X##1
@@ -644,6 +710,8 @@ static GTY(()) struct aarch64_simd_type_info aarch64_simd_types [] = {
 };
 #undef ENTRY
 
+static GTY(()) tree aarch64_simd_tuple_types[ARM_NEON_H_TYPES_LAST][3];
+
 static GTY(()) tree aarch64_simd_intOI_type_node = NULL_TREE;
 static GTY(()) tree aarch64_simd_intCI_type_node = NULL_TREE;
 static GTY(()) tree aarch64_simd_intXI_type_node = NULL_TREE;
@@ -764,9 +832,16 @@ aarch64_lookup_simd_builtin_type (machine_mode mode,
     return aarch64_simd_builtin_std_type (mode, q);
 
   for (i = 0; i < nelts; i++)
-    if (aarch64_simd_types[i].mode == mode
-	&& aarch64_simd_types[i].q == q)
-      return aarch64_simd_types[i].itype;
+    {
+      if (aarch64_simd_types[i].mode == mode
+	  && aarch64_simd_types[i].q == q)
+	return aarch64_simd_types[i].itype;
+      if (aarch64_simd_tuple_types[i][0] != NULL_TREE)
+	for (int j = 0; j < 3; j++)
+	  if (TYPE_MODE (aarch64_simd_tuple_types[i][j]) == mode
+	      && aarch64_simd_types[i].q == q)
+	    return aarch64_simd_tuple_types[i][j];
+    }
 
   return NULL_TREE;
 }
@@ -1046,32 +1121,22 @@ aarch64_init_fcmla_laneq_builtins (void)
 }
 
 void
-aarch64_init_simd_builtins (void)
+aarch64_init_simd_builtin_functions (bool called_from_pragma)
 {
   unsigned int i, fcode = AARCH64_SIMD_PATTERN_START;
 
-  if (aarch64_simd_builtins_initialized_p)
-    return;
-
-  aarch64_simd_builtins_initialized_p = true;
-
-  aarch64_init_simd_builtin_types ();
-
-  /* Strong-typing hasn't been implemented for all AdvSIMD builtin intrinsics.
-     Therefore we need to preserve the old __builtin scalar types.  It can be
-     removed once all the intrinsics become strongly typed using the qualifier
-     system.  */
-  aarch64_init_simd_builtin_scalar_types ();
- 
-  tree lane_check_fpr = build_function_type_list (void_type_node,
-						  size_type_node,
-						  size_type_node,
-						  intSI_type_node,
-						  NULL);
-  aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_LANE_CHECK]
-    = aarch64_general_add_builtin ("__builtin_aarch64_im_lane_boundsi",
-				   lane_check_fpr,
-				   AARCH64_SIMD_BUILTIN_LANE_CHECK);
+  if (!called_from_pragma)
+    {
+      tree lane_check_fpr = build_function_type_list (void_type_node,
+						      size_type_node,
+						      size_type_node,
+						      intSI_type_node,
+						      NULL);
+      aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_LANE_CHECK]
+	= aarch64_general_add_builtin ("__builtin_aarch64_im_lane_boundsi",
+				       lane_check_fpr,
+				       AARCH64_SIMD_BUILTIN_LANE_CHECK);
+    }
 
   for (i = 0; i < ARRAY_SIZE (aarch64_simd_builtin_data); i++, fcode++)
     {
@@ -1101,6 +1166,18 @@ aarch64_init_simd_builtins (void)
       tree return_type = void_type_node, args = void_list_node;
       tree eltype;
 
+      int struct_mode_args = 0;
+      for (int j = op_num; j >= 0; j--)
+	{
+	  machine_mode op_mode = insn_data[d->code].operand[j].mode;
+	  if (aarch64_advsimd_struct_mode_p (op_mode))
+	    struct_mode_args++;
+	}
+
+      if ((called_from_pragma && struct_mode_args == 0)
+	  || (!called_from_pragma && struct_mode_args > 0))
+	continue;
+
       /* Build a function type directly from the insn_data for this
 	 builtin.  The build_function_type () function takes care of
 	 removing duplicates for us.  */
@@ -1171,12 +1248,112 @@ aarch64_init_simd_builtins (void)
 
       tree attrs = aarch64_get_attributes (d->flags, d->mode);
 
-      fndecl = aarch64_general_add_builtin (namebuf, ftype, fcode, attrs);
+      if (called_from_pragma)
+	{
+	  unsigned int raw_code
+		= (fcode << AARCH64_BUILTIN_SHIFT) | AARCH64_BUILTIN_GENERAL;
+	  fndecl = simulate_builtin_function_decl (input_location, namebuf,
+						   ftype, raw_code, NULL,
+						   attrs);
+	}
+      else
+	fndecl = aarch64_general_add_builtin (namebuf, ftype, fcode, attrs);
+
       aarch64_builtin_decls[fcode] = fndecl;
     }
+}
 
-   /* Initialize the remaining fcmla_laneq intrinsics.  */
-   aarch64_init_fcmla_laneq_builtins ();
+/* Register the tuple type that contains NUM_VECTORS of the AdvSIMD type
+   indexed by TYPE_INDEX.  */
+static void
+register_tuple_type (unsigned int num_vectors, unsigned int type_index)
+{
+  aarch64_simd_type_info *type = &aarch64_simd_types[type_index];
+
+  /* Synthesize the name of the user-visible vector tuple type.  */
+  const char *vector_type_name = type->name;
+  char tuple_type_name[sizeof ("bfloat16x4x2_t")];
+  snprintf (tuple_type_name, sizeof (tuple_type_name), "%.*sx%d_t",
+	    (int) strlen (vector_type_name) - 4, vector_type_name + 2,
+	    num_vectors);
+  tuple_type_name[0] = TOLOWER (tuple_type_name[0]);
+
+  tree vector_type = type->itype;
+  tree array_type = build_array_type_nelts (vector_type, num_vectors);
+  if (type->mode == DImode)
+    {
+      if (num_vectors == 2)
+	SET_TYPE_MODE (array_type, V2x1DImode);
+      else if (num_vectors == 3)
+	SET_TYPE_MODE (array_type, V3x1DImode);
+      else if (num_vectors == 4)
+	SET_TYPE_MODE (array_type, V4x1DImode);
+    }
+
+  unsigned int alignment
+	= (known_eq (GET_MODE_SIZE (type->mode), 16) ? 128 : 64);
+  gcc_assert (TYPE_MODE_RAW (array_type) == TYPE_MODE (array_type)
+	      && TYPE_ALIGN (array_type) == alignment);
+
+  tree field = build_decl (input_location, FIELD_DECL,
+			   get_identifier ("val"), array_type);
+
+  tree t = lang_hooks.types.simulate_record_decl (input_location,
+						  tuple_type_name,
+						  make_array_slice (&field,
+								    1));
+  gcc_assert (TYPE_MODE_RAW (t) == TYPE_MODE (t)
+	      && TYPE_ALIGN (t) == alignment);
+
+  if (num_vectors == 2)
+    aarch64_simd_tuple_types[type_index][0] = t;
+  else if (num_vectors == 3)
+    aarch64_simd_tuple_types[type_index][1] = t;
+  else if (num_vectors == 4)
+    aarch64_simd_tuple_types[type_index][2] = t;
+}
+
+static bool
+aarch64_scalar_builtin_type_p (aarch64_simd_type t)
+{
+  return (t == Poly8_t || t == Poly16_t || t == Poly64_t || t == Poly128_t);
+}
+
+/* Implement #pragma GCC aarch64 "arm_neon.h".  */
+void
+handle_arm_neon_h (void)
+{
+  /* Register the AdvSIMD vector tuple types.  */
+  for (unsigned int i = 0; i < ARM_NEON_H_TYPES_LAST; i++)
+    for (unsigned int count = 2; count <= 4; ++count)
+      if (!aarch64_scalar_builtin_type_p (aarch64_simd_types[i].type))
+	register_tuple_type (count, i);
+
+  aarch64_init_simd_builtin_functions (true);
+}
+
+void
+aarch64_init_simd_builtins (void)
+{
+  if (aarch64_simd_builtins_initialized_p)
+    return;
+
+  aarch64_simd_builtins_initialized_p = true;
+
+  aarch64_init_simd_builtin_types ();
+
+  /* Strong-typing hasn't been implemented for all AdvSIMD builtin intrinsics.
+     Therefore we need to preserve the old __builtin scalar types.  It can be
+     removed once all the intrinsics become strongly typed using the qualifier
+     system.  */
+  aarch64_init_simd_builtin_scalar_types ();
+
+  aarch64_init_simd_builtin_functions (false);
+  if (in_lto_p)
+    handle_arm_neon_h ();
+
+  /* Initialize the remaining fcmla_laneq intrinsics.  */
+  aarch64_init_fcmla_laneq_builtins ();
 }
 
 static void
@@ -2405,47 +2582,83 @@ get_mem_type_for_load_store (unsigned int fcode)
 {
   switch (fcode)
   {
-    VAR1 (LOAD1, ld1 , 0, LOAD, v8qi)
-    VAR1 (STORE1, st1 , 0, STORE, v8qi)
+    VAR1 (LOAD1, ld1, 0, LOAD, v8qi)
+    VAR1 (STORE1, st1, 0, STORE, v8qi)
       return Int8x8_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v16qi)
-    VAR1 (STORE1, st1 , 0, STORE, v16qi)
+    VAR1 (LOAD1, ld1, 0, LOAD, v16qi)
+    VAR1 (STORE1, st1, 0, STORE, v16qi)
       return Int8x16_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v4hi)
-    VAR1 (STORE1, st1 , 0, STORE, v4hi)
+    VAR1 (LOAD1, ld1, 0, LOAD, v4hi)
+    VAR1 (STORE1, st1, 0, STORE, v4hi)
       return Int16x4_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v8hi)
-    VAR1 (STORE1, st1 , 0, STORE, v8hi)
+    VAR1 (LOAD1, ld1, 0, LOAD, v8hi)
+    VAR1 (STORE1, st1, 0, STORE, v8hi)
       return Int16x8_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v2si)
-    VAR1 (STORE1, st1 , 0, STORE, v2si)
+    VAR1 (LOAD1, ld1, 0, LOAD, v2si)
+    VAR1 (STORE1, st1, 0, STORE, v2si)
       return Int32x2_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v4si)
-    VAR1 (STORE1, st1 , 0, STORE, v4si)
+    VAR1 (LOAD1, ld1, 0, LOAD, v4si)
+    VAR1 (STORE1, st1, 0, STORE, v4si)
       return Int32x4_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v2di)
-    VAR1 (STORE1, st1 , 0, STORE, v2di)
+    VAR1 (LOAD1, ld1, 0, LOAD, v2di)
+    VAR1 (STORE1, st1, 0, STORE, v2di)
       return Int64x2_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v4hf)
-    VAR1 (STORE1, st1 , 0, STORE, v4hf)
+    VAR1 (LOAD1_U, ld1, 0, LOAD, v8qi)
+    VAR1 (STORE1_U, st1, 0, STORE, v8qi)
+      return Uint8x8_t;
+    VAR1 (LOAD1_U, ld1, 0, LOAD, v16qi)
+    VAR1 (STORE1_U, st1, 0, STORE, v16qi)
+      return Uint8x16_t;
+    VAR1 (LOAD1_U, ld1, 0, LOAD, v4hi)
+    VAR1 (STORE1_U, st1, 0, STORE, v4hi)
+      return Uint16x4_t;
+    VAR1 (LOAD1_U, ld1, 0, LOAD, v8hi)
+    VAR1 (STORE1_U, st1, 0, STORE, v8hi)
+      return Uint16x8_t;
+    VAR1 (LOAD1_U, ld1, 0, LOAD, v2si)
+    VAR1 (STORE1_U, st1, 0, STORE, v2si)
+      return Uint32x2_t;
+    VAR1 (LOAD1_U, ld1, 0, LOAD, v4si)
+    VAR1 (STORE1_U, st1, 0, STORE, v4si)
+      return Uint32x4_t;
+    VAR1 (LOAD1_U, ld1, 0, LOAD, v2di)
+    VAR1 (STORE1_U, st1, 0, STORE, v2di)
+      return Uint64x2_t;
+    VAR1 (LOAD1_P, ld1, 0, LOAD, v8qi)
+    VAR1 (STORE1_P, st1, 0, STORE, v8qi)
+      return Poly8x8_t;
+    VAR1 (LOAD1_P, ld1, 0, LOAD, v16qi)
+    VAR1 (STORE1_P, st1, 0, STORE, v16qi)
+      return Poly8x16_t;
+    VAR1 (LOAD1_P, ld1, 0, LOAD, v4hi)
+    VAR1 (STORE1_P, st1, 0, STORE, v4hi)
+      return Poly16x4_t;
+    VAR1 (LOAD1_P, ld1, 0, LOAD, v8hi)
+    VAR1 (STORE1_P, st1, 0, STORE, v8hi)
+      return Poly16x8_t;
+    VAR1 (LOAD1_P, ld1, 0, LOAD, v2di)
+    VAR1 (STORE1_P, st1, 0, STORE, v2di)
+      return Poly64x2_t;
+    VAR1 (LOAD1, ld1, 0, LOAD, v4hf)
+    VAR1 (STORE1, st1, 0, STORE, v4hf)
       return Float16x4_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v8hf)
-    VAR1 (STORE1, st1 , 0, STORE, v8hf)
+    VAR1 (LOAD1, ld1, 0, LOAD, v8hf)
+    VAR1 (STORE1, st1, 0, STORE, v8hf)
       return Float16x8_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v4bf)
-    VAR1 (STORE1, st1 , 0, STORE, v4bf)
+    VAR1 (LOAD1, ld1, 0, LOAD, v4bf)
+    VAR1 (STORE1, st1, 0, STORE, v4bf)
       return Bfloat16x4_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v8bf)
-    VAR1 (STORE1, st1 , 0, STORE, v8bf)
+    VAR1 (LOAD1, ld1, 0, LOAD, v8bf)
+    VAR1 (STORE1, st1, 0, STORE, v8bf)
       return Bfloat16x8_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v2sf)
-    VAR1 (STORE1, st1 , 0, STORE, v2sf)
+    VAR1 (LOAD1, ld1, 0, LOAD, v2sf)
+    VAR1 (STORE1, st1, 0, STORE, v2sf)
       return Float32x2_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v4sf)
-    VAR1 (STORE1, st1 , 0, STORE, v4sf)
+    VAR1 (LOAD1, ld1, 0, LOAD, v4sf)
+    VAR1 (STORE1, st1, 0, STORE, v4sf)
       return Float32x4_t;
-    VAR1 (LOAD1, ld1 , 0, LOAD, v2df)
-    VAR1 (STORE1, st1 , 0, STORE, v2df)
+    VAR1 (LOAD1, ld1, 0, LOAD, v2df)
+    VAR1 (STORE1, st1, 0, STORE, v2df)
       return Float64x2_t;
     default:
       gcc_unreachable ();
@@ -2458,7 +2671,7 @@ get_mem_type_for_load_store (unsigned int fcode)
    failure.  */
 gimple *
 aarch64_general_gimple_fold_builtin (unsigned int fcode, gcall *stmt,
-				     gimple_stmt_iterator *gsi)
+				     gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED)
 {
   gimple *new_stmt = NULL;
   unsigned nargs = gimple_call_num_args (stmt);
@@ -2479,46 +2692,49 @@ aarch64_general_gimple_fold_builtin (unsigned int fcode, gcall *stmt,
 
      /*lower store and load neon builtins to gimple.  */
      BUILTIN_VALL_F16 (LOAD1, ld1, 0, LOAD)
+     BUILTIN_VDQ_I (LOAD1_U, ld1, 0, LOAD)
+     BUILTIN_VALLP_NO_DI (LOAD1_P, ld1, 0, LOAD)
 	if (!BYTES_BIG_ENDIAN)
 	  {
 	    enum aarch64_simd_type mem_type
 	      = get_mem_type_for_load_store(fcode);
 	    aarch64_simd_type_info simd_type
 	      = aarch64_simd_types[mem_type];
-	    tree elt_ptr_type = build_pointer_type (simd_type.eltype);
+	    tree elt_ptr_type = build_pointer_type_for_mode (simd_type.eltype,
+							     VOIDmode, true);
 	    tree zero = build_zero_cst (elt_ptr_type);
-	    gimple_seq stmts = NULL;
-	    tree base = gimple_convert (&stmts, elt_ptr_type,
-					args[0]);
-	    if (stmts)
-	      gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
+	    /* Use element type alignment.  */
+	    tree access_type
+	      = build_aligned_type (simd_type.itype,
+				    TYPE_ALIGN (simd_type.eltype));
 	    new_stmt
 	      = gimple_build_assign (gimple_get_lhs (stmt),
 				     fold_build2 (MEM_REF,
-						  simd_type.itype,
-						  base, zero));
+						  access_type,
+						  args[0], zero));
 	  }
 	break;
 
       BUILTIN_VALL_F16 (STORE1, st1, 0, STORE)
+      BUILTIN_VDQ_I (STORE1_U, st1, 0, STORE)
+      BUILTIN_VALLP_NO_DI (STORE1_P, st1, 0, STORE)
 	if (!BYTES_BIG_ENDIAN)
 	  {
 	    enum aarch64_simd_type mem_type
 	      = get_mem_type_for_load_store(fcode);
 	    aarch64_simd_type_info simd_type
 	      = aarch64_simd_types[mem_type];
-	    tree elt_ptr_type = build_pointer_type (simd_type.eltype);
+	    tree elt_ptr_type = build_pointer_type_for_mode (simd_type.eltype,
+							     VOIDmode, true);
 	    tree zero = build_zero_cst (elt_ptr_type);
-	    gimple_seq stmts = NULL;
-	    tree base = gimple_convert (&stmts, elt_ptr_type,
-					args[0]);
-	    if (stmts)
-	      gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
+	    /* Use element type alignment.  */
+	    tree access_type
+	      = build_aligned_type (simd_type.itype,
+				    TYPE_ALIGN (simd_type.eltype));
 	    new_stmt
-	      = gimple_build_assign (fold_build2 (MEM_REF,
-				     simd_type.itype,
-				     base,
-				     zero), args[1]);
+	      = gimple_build_assign (fold_build2 (MEM_REF, access_type,
+						  args[0], zero),
+				     args[1]);
 	  }
 	break;
 
@@ -2534,6 +2750,54 @@ aarch64_general_gimple_fold_builtin (unsigned int fcode, gcall *stmt,
 					       1, args[0]);
 	gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
 	break;
+      BUILTIN_VSDQ_I_DI (BINOP, ashl, 3, NONE)
+	if (TREE_CODE (args[1]) == INTEGER_CST
+	    && wi::ltu_p (wi::to_wide (args[1]), element_precision (args[0])))
+	  new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
+					  LSHIFT_EXPR, args[0], args[1]);
+	break;
+      BUILTIN_VSDQ_I_DI (BINOP, sshl, 0, NONE)
+      BUILTIN_VSDQ_I_DI (BINOP_UUS, ushl, 0, NONE)
+	{
+	  tree cst = args[1];
+	  tree ctype = TREE_TYPE (cst);
+	  /* Left shifts can be both scalar or vector, e.g. uint64x1_t is
+	     treated as a scalar type not a vector one.  */
+	  if ((cst = uniform_integer_cst_p (cst)) != NULL_TREE)
+	    {
+	      wide_int wcst = wi::to_wide (cst);
+	      tree unit_ty = TREE_TYPE (cst);
+
+	      wide_int abs_cst = wi::abs (wcst);
+	      if (wi::geu_p (abs_cst, element_precision (args[0])))
+		break;
+
+	      if (wi::neg_p (wcst, TYPE_SIGN (ctype)))
+		{
+		  tree final_cst;
+		  final_cst = wide_int_to_tree (unit_ty, abs_cst);
+		  if (TREE_CODE (cst) != INTEGER_CST)
+		    final_cst = build_uniform_cst (ctype, final_cst);
+
+		  new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
+						  RSHIFT_EXPR, args[0],
+						  final_cst);
+		}
+	      else
+		new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
+						LSHIFT_EXPR, args[0], args[1]);
+	    }
+	}
+	break;
+      BUILTIN_VDQ_I (SHIFTIMM, ashr, 3, NONE)
+      VAR1 (SHIFTIMM, ashr_simd, 0, NONE, di)
+      BUILTIN_VDQ_I (USHIFTIMM, lshr, 3, NONE)
+      VAR1 (USHIFTIMM, lshr_simd, 0, NONE, di)
+	if (TREE_CODE (args[1]) == INTEGER_CST
+	    && wi::ltu_p (wi::to_wide (args[1]), element_precision (args[0])))
+	  new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
+					  RSHIFT_EXPR, args[0], args[1]);
+	break;
       BUILTIN_GPF (BINOP, fmulx, 0, ALL)
 	{
 	  gcc_assert (nargs == 2);
diff --git a/gcc/config/aarch64/aarch64-c.c b/gcc/config/aarch64/aarch64-c.c
index f9ddffa..d6653e4 100644
--- a/gcc/config/aarch64/aarch64-c.c
+++ b/gcc/config/aarch64/aarch64-c.c
@@ -296,6 +296,8 @@ aarch64_pragma_aarch64 (cpp_reader *)
   const char *name = TREE_STRING_POINTER (x);
   if (strcmp (name, "arm_sve.h") == 0)
     aarch64_sve::handle_arm_sve_h ();
+  else if (strcmp (name, "arm_neon.h") == 0)
+    handle_arm_neon_h ();
   else
     error ("unknown %<#pragma GCC aarch64%> option %qs", name);
 }
diff --git a/gcc/config/aarch64/aarch64-cores.def b/gcc/config/aarch64/aarch64-cores.def
index 77da310..617cde42 100644
--- a/gcc/config/aarch64/aarch64-cores.def
+++ b/gcc/config/aarch64/aarch64-cores.def
@@ -68,7 +68,8 @@ AARCH64_CORE("octeontx83",    octeontxt83,   thunderx,  8A,  AARCH64_FL_FOR_ARCH
 AARCH64_CORE("thunderxt81",   thunderxt81,   thunderx,  8A,  AARCH64_FL_FOR_ARCH8 | AARCH64_FL_CRC | AARCH64_FL_CRYPTO, thunderx,  0x43, 0x0a2, -1)
 AARCH64_CORE("thunderxt83",   thunderxt83,   thunderx,  8A,  AARCH64_FL_FOR_ARCH8 | AARCH64_FL_CRC | AARCH64_FL_CRYPTO, thunderx,  0x43, 0x0a3, -1)
 
-/* Ampere Computing cores. */
+/* Ampere Computing ('\xC0') cores. */
+AARCH64_CORE("ampere1", ampere1, cortexa57, 8_6A, AARCH64_FL_FOR_ARCH8_6, ampere1, 0xC0, 0xac3, -1)
 /* Do not swap around "emag" and "xgene1",
    this order is required to handle variant correctly. */
 AARCH64_CORE("emag",        emag,      xgene1,    8A,  AARCH64_FL_FOR_ARCH8 | AARCH64_FL_CRC | AARCH64_FL_CRYPTO, emag, 0x50, 0x000, 3)
diff --git a/gcc/config/aarch64/aarch64-cost-tables.h b/gcc/config/aarch64/aarch64-cost-tables.h
index dd2e7e7..e6ded65 100644
--- a/gcc/config/aarch64/aarch64-cost-tables.h
+++ b/gcc/config/aarch64/aarch64-cost-tables.h
@@ -124,7 +124,10 @@ const struct cpu_cost_table qdf24xx_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),  /* alu.  */
-    COSTS_N_INSNS (4)   /* mult.  */
+    COSTS_N_INSNS (4),  /* mult.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (2),  /* dup.  */
+    COSTS_N_INSNS (2)   /* extract.  */
   }
 };
 
@@ -229,7 +232,10 @@ const struct cpu_cost_table thunderx_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* Alu.  */
-    COSTS_N_INSNS (4)	/* mult.  */
+    COSTS_N_INSNS (4),	/* mult.  */
+    COSTS_N_INSNS (1),	/* movi.  */
+    COSTS_N_INSNS (2),	/* dup.  */
+    COSTS_N_INSNS (2)	/* extract.  */
   }
 };
 
@@ -333,7 +339,10 @@ const struct cpu_cost_table thunderx2t99_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* Alu.  */
-    COSTS_N_INSNS (4)	/* Mult.  */
+    COSTS_N_INSNS (4),	/* Mult.  */
+    COSTS_N_INSNS (1),	/* movi.  */
+    COSTS_N_INSNS (2),	/* dup.  */
+    COSTS_N_INSNS (2)	/* extract.  */
   }
 };
 
@@ -437,7 +446,10 @@ const struct cpu_cost_table thunderx3t110_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* Alu.  */
-    COSTS_N_INSNS (4)	/* Mult.  */
+    COSTS_N_INSNS (4),	/* Mult.  */
+    COSTS_N_INSNS (1),	/* movi.  */
+    COSTS_N_INSNS (2),	/* dup.  */
+    COSTS_N_INSNS (2)	/* extract.  */
   }
 };
 
@@ -542,7 +554,10 @@ const struct cpu_cost_table tsv110_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),  /* alu.  */
-    COSTS_N_INSNS (4)   /* mult.  */
+    COSTS_N_INSNS (4),  /* mult.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (2),  /* dup.  */
+    COSTS_N_INSNS (2)   /* extract.  */
   }
 };
 
@@ -646,7 +661,117 @@ const struct cpu_cost_table a64fx_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),  /* alu.  */
-    COSTS_N_INSNS (4)   /* mult.  */
+    COSTS_N_INSNS (4),  /* mult.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (2),  /* dup.  */
+    COSTS_N_INSNS (2)   /* extract.  */
+  }
+};
+
+const struct cpu_cost_table ampere1_extra_costs =
+{
+  /* ALU */
+  {
+    0,                 /* arith.  */
+    0,                 /* logical.  */
+    0,                 /* shift.  */
+    COSTS_N_INSNS (1), /* shift_reg.  */
+    0,                 /* arith_shift.  */
+    COSTS_N_INSNS (1), /* arith_shift_reg.  */
+    0,                 /* log_shift.  */
+    COSTS_N_INSNS (1), /* log_shift_reg.  */
+    0,                 /* extend.  */
+    COSTS_N_INSNS (1), /* extend_arith.  */
+    0,                 /* bfi.  */
+    0,                 /* bfx.  */
+    0,                 /* clz.  */
+    0,                 /* rev.  */
+    0,                 /* non_exec.  */
+    true               /* non_exec_costs_exec.  */
+  },
+  {
+    /* MULT SImode */
+    {
+      COSTS_N_INSNS (3),       /* simple.  */
+      COSTS_N_INSNS (3),       /* flag_setting.  */
+      COSTS_N_INSNS (3),       /* extend.  */
+      COSTS_N_INSNS (4),       /* add.  */
+      COSTS_N_INSNS (4),       /* extend_add.  */
+      COSTS_N_INSNS (18)       /* idiv.  */
+    },
+    /* MULT DImode */
+    {
+      COSTS_N_INSNS (3),       /* simple.  */
+      0,                       /* flag_setting (N/A).  */
+      COSTS_N_INSNS (3),       /* extend.  */
+      COSTS_N_INSNS (4),       /* add.  */
+      COSTS_N_INSNS (4),       /* extend_add.  */
+      COSTS_N_INSNS (34)       /* idiv.  */
+    }
+  },
+  /* LD/ST */
+  {
+    COSTS_N_INSNS (4),         /* load.  */
+    COSTS_N_INSNS (4),         /* load_sign_extend.  */
+    0,                         /* ldrd (n/a).  */
+    0,                         /* ldm_1st.  */
+    0,                         /* ldm_regs_per_insn_1st.  */
+    0,                         /* ldm_regs_per_insn_subsequent.  */
+    COSTS_N_INSNS (5),         /* loadf.  */
+    COSTS_N_INSNS (5),         /* loadd.  */
+    COSTS_N_INSNS (5),         /* load_unaligned.  */
+    0,                         /* store.  */
+    0,                         /* strd.  */
+    0,                         /* stm_1st.  */
+    0,                         /* stm_regs_per_insn_1st.  */
+    0,                         /* stm_regs_per_insn_subsequent.  */
+    COSTS_N_INSNS (2),         /* storef.  */
+    COSTS_N_INSNS (2),         /* stored.  */
+    COSTS_N_INSNS (2),         /* store_unaligned.  */
+    COSTS_N_INSNS (3),         /* loadv.  */
+    COSTS_N_INSNS (3)          /* storev.  */
+  },
+  {
+    /* FP SFmode */
+    {
+      COSTS_N_INSNS (25),      /* div.  */
+      COSTS_N_INSNS (4),       /* mult.  */
+      COSTS_N_INSNS (4),       /* mult_addsub.  */
+      COSTS_N_INSNS (4),       /* fma.  */
+      COSTS_N_INSNS (4),       /* addsub.  */
+      COSTS_N_INSNS (2),       /* fpconst.  */
+      COSTS_N_INSNS (4),       /* neg.  */
+      COSTS_N_INSNS (4),       /* compare.  */
+      COSTS_N_INSNS (4),       /* widen.  */
+      COSTS_N_INSNS (4),       /* narrow.  */
+      COSTS_N_INSNS (4),       /* toint.  */
+      COSTS_N_INSNS (4),       /* fromint.  */
+      COSTS_N_INSNS (4)        /* roundint.  */
+    },
+    /* FP DFmode */
+    {
+      COSTS_N_INSNS (34),      /* div.  */
+      COSTS_N_INSNS (5),       /* mult.  */
+      COSTS_N_INSNS (5),       /* mult_addsub.  */
+      COSTS_N_INSNS (5),       /* fma.  */
+      COSTS_N_INSNS (5),       /* addsub.  */
+      COSTS_N_INSNS (2),       /* fpconst.  */
+      COSTS_N_INSNS (5),       /* neg.  */
+      COSTS_N_INSNS (5),       /* compare.  */
+      COSTS_N_INSNS (5),       /* widen.  */
+      COSTS_N_INSNS (5),       /* narrow.  */
+      COSTS_N_INSNS (6),       /* toint.  */
+      COSTS_N_INSNS (6),       /* fromint.  */
+      COSTS_N_INSNS (5)        /* roundint.  */
+    }
+  },
+  /* Vector */
+  {
+    COSTS_N_INSNS (3),  /* alu.  */
+    COSTS_N_INSNS (3),  /* mult.  */
+    COSTS_N_INSNS (2),  /* movi.  */
+    COSTS_N_INSNS (2),  /* dup.  */
+    COSTS_N_INSNS (2)   /* extract.  */
   }
 };
 
diff --git a/gcc/config/aarch64/aarch64-modes.def b/gcc/config/aarch64/aarch64-modes.def
index 1a07bc1..ac97d22 100644
--- a/gcc/config/aarch64/aarch64-modes.def
+++ b/gcc/config/aarch64/aarch64-modes.def
@@ -81,13 +81,69 @@ INT_MODE (OI, 32);
 INT_MODE (CI, 48);
 INT_MODE (XI, 64);
 
+/* Define Advanced SIMD modes for structures of 2, 3 and 4 d-registers.  */
+#define ADV_SIMD_D_REG_STRUCT_MODES(NVECS, VB, VH, VS, VD) \
+  VECTOR_MODES_WITH_PREFIX (V##NVECS##x, INT, 8, 3); \
+  VECTOR_MODES_WITH_PREFIX (V##NVECS##x, FLOAT, 8, 3); \
+  VECTOR_MODE_WITH_PREFIX (V##NVECS##x, FLOAT, DF, 1, 3); \
+  VECTOR_MODE_WITH_PREFIX (V##NVECS##x, INT, DI, 1, 3); \
+  \
+  ADJUST_NUNITS (VB##QI, NVECS * 8); \
+  ADJUST_NUNITS (VH##HI, NVECS * 4); \
+  ADJUST_NUNITS (VS##SI, NVECS * 2); \
+  ADJUST_NUNITS (VD##DI, NVECS); \
+  ADJUST_NUNITS (VH##BF, NVECS * 4); \
+  ADJUST_NUNITS (VH##HF, NVECS * 4); \
+  ADJUST_NUNITS (VS##SF, NVECS * 2); \
+  ADJUST_NUNITS (VD##DF, NVECS); \
+  \
+  ADJUST_ALIGNMENT (VB##QI, 8); \
+  ADJUST_ALIGNMENT (VH##HI, 8); \
+  ADJUST_ALIGNMENT (VS##SI, 8); \
+  ADJUST_ALIGNMENT (VD##DI, 8); \
+  ADJUST_ALIGNMENT (VH##BF, 8); \
+  ADJUST_ALIGNMENT (VH##HF, 8); \
+  ADJUST_ALIGNMENT (VS##SF, 8); \
+  ADJUST_ALIGNMENT (VD##DF, 8);
+
+ADV_SIMD_D_REG_STRUCT_MODES (2, V2x8, V2x4, V2x2, V2x1)
+ADV_SIMD_D_REG_STRUCT_MODES (3, V3x8, V3x4, V3x2, V3x1)
+ADV_SIMD_D_REG_STRUCT_MODES (4, V4x8, V4x4, V4x2, V4x1)
+
+/* Define Advanced SIMD modes for structures of 2, 3 and 4 q-registers.  */
+#define ADV_SIMD_Q_REG_STRUCT_MODES(NVECS, VB, VH, VS, VD) \
+  VECTOR_MODES_WITH_PREFIX (V##NVECS##x, INT, 16, 3); \
+  VECTOR_MODES_WITH_PREFIX (V##NVECS##x, FLOAT, 16, 3); \
+  \
+  ADJUST_NUNITS (VB##QI, NVECS * 16); \
+  ADJUST_NUNITS (VH##HI, NVECS * 8); \
+  ADJUST_NUNITS (VS##SI, NVECS * 4); \
+  ADJUST_NUNITS (VD##DI, NVECS * 2); \
+  ADJUST_NUNITS (VH##BF, NVECS * 8); \
+  ADJUST_NUNITS (VH##HF, NVECS * 8); \
+  ADJUST_NUNITS (VS##SF, NVECS * 4); \
+  ADJUST_NUNITS (VD##DF, NVECS * 2); \
+  \
+  ADJUST_ALIGNMENT (VB##QI, 16); \
+  ADJUST_ALIGNMENT (VH##HI, 16); \
+  ADJUST_ALIGNMENT (VS##SI, 16); \
+  ADJUST_ALIGNMENT (VD##DI, 16); \
+  ADJUST_ALIGNMENT (VH##BF, 16); \
+  ADJUST_ALIGNMENT (VH##HF, 16); \
+  ADJUST_ALIGNMENT (VS##SF, 16); \
+  ADJUST_ALIGNMENT (VD##DF, 16);
+
+ADV_SIMD_Q_REG_STRUCT_MODES (2, V2x16, V2x8, V2x4, V2x2)
+ADV_SIMD_Q_REG_STRUCT_MODES (3, V3x16, V3x8, V3x4, V3x2)
+ADV_SIMD_Q_REG_STRUCT_MODES (4, V4x16, V4x8, V4x4, V4x2)
+
 /* Define SVE modes for NVECS vectors.  VB, VH, VS and VD are the prefixes
    for 8-bit, 16-bit, 32-bit and 64-bit elements respectively.  It isn't
    strictly necessary to set the alignment here, since the default would
    be clamped to BIGGEST_ALIGNMENT anyhow, but it seems clearer.  */
 #define SVE_MODES(NVECS, VB, VH, VS, VD) \
-  VECTOR_MODES_WITH_PREFIX (VNx, INT, 16 * NVECS, 0); \
-  VECTOR_MODES_WITH_PREFIX (VNx, FLOAT, 16 * NVECS, 0); \
+  VECTOR_MODES_WITH_PREFIX (VNx, INT, 16 * NVECS, NVECS == 1 ? 1 : 4); \
+  VECTOR_MODES_WITH_PREFIX (VNx, FLOAT, 16 * NVECS, NVECS == 1 ? 1 : 4); \
   \
   ADJUST_NUNITS (VB##QI, aarch64_sve_vg * NVECS * 8); \
   ADJUST_NUNITS (VH##HI, aarch64_sve_vg * NVECS * 4); \
@@ -123,14 +179,14 @@ SVE_MODES (4, VNx64, VNx32, VNx16, VNx8)
    In memory they occupy contiguous locations, in the same way as fixed-length
    vectors.  E.g. VNx8QImode is half the size of VNx16QImode.
 
-   Passing 1 as the final argument ensures that the modes come after all
-   other modes in the GET_MODE_WIDER chain, so that we never pick them
-   in preference to a full vector mode.  */
-VECTOR_MODES_WITH_PREFIX (VNx, INT, 2, 1);
-VECTOR_MODES_WITH_PREFIX (VNx, INT, 4, 1);
-VECTOR_MODES_WITH_PREFIX (VNx, INT, 8, 1);
-VECTOR_MODES_WITH_PREFIX (VNx, FLOAT, 4, 1);
-VECTOR_MODES_WITH_PREFIX (VNx, FLOAT, 8, 1);
+   Passing 2 as the final argument ensures that the modes come after all
+   other single-vector modes in the GET_MODE_WIDER chain, so that we never
+   pick them in preference to a full vector mode.  */
+VECTOR_MODES_WITH_PREFIX (VNx, INT, 2, 2);
+VECTOR_MODES_WITH_PREFIX (VNx, INT, 4, 2);
+VECTOR_MODES_WITH_PREFIX (VNx, INT, 8, 2);
+VECTOR_MODES_WITH_PREFIX (VNx, FLOAT, 4, 2);
+VECTOR_MODES_WITH_PREFIX (VNx, FLOAT, 8, 2);
 
 ADJUST_NUNITS (VNx2QI, aarch64_sve_vg);
 ADJUST_NUNITS (VNx2HI, aarch64_sve_vg);
diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h
index 768e8fa..f7887d0 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -743,6 +743,7 @@ unsigned HOST_WIDE_INT aarch64_and_split_imm2 (HOST_WIDE_INT val_in);
 bool aarch64_and_bitmask_imm (unsigned HOST_WIDE_INT val_in, machine_mode mode);
 int aarch64_branch_cost (bool, bool);
 enum aarch64_symbol_type aarch64_classify_symbolic_expression (rtx);
+bool aarch64_advsimd_struct_mode_p (machine_mode mode);
 opt_machine_mode aarch64_vq_mode (scalar_mode);
 opt_machine_mode aarch64_full_sve_mode (scalar_mode);
 bool aarch64_can_const_movi_rtx_p (rtx x, machine_mode mode);
@@ -968,6 +969,7 @@ rtx aarch64_general_expand_builtin (unsigned int, tree, rtx, int);
 tree aarch64_general_builtin_decl (unsigned, bool);
 tree aarch64_general_builtin_rsqrt (unsigned int);
 tree aarch64_builtin_vectorized_function (unsigned int, tree, tree);
+void handle_arm_neon_h (void);
 
 namespace aarch64_sve {
   void init_builtins ();
diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def b/gcc/config/aarch64/aarch64-simd-builtins.def
index be1c5b5..b4b95d7 100644
--- a/gcc/config/aarch64/aarch64-simd-builtins.def
+++ b/gcc/config/aarch64/aarch64-simd-builtins.def
@@ -43,15 +43,18 @@
    help describe the attributes (for example, pure) for the intrinsic
    function.  */
 
-  BUILTIN_VDC (COMBINE, combine, 0, AUTO_FP)
-  VAR1 (COMBINEP, combine, 0, NONE, di)
-  BUILTIN_VB (BINOP, pmul, 0, NONE)
-  VAR1 (BINOP, pmull, 0, NONE, v8qi)
-  VAR1 (BINOP, pmull_hi, 0, NONE, v16qi)
+  BUILTIN_VDC (BINOP, combine, 0, AUTO_FP)
+  BUILTIN_VD_I (BINOPU, combine, 0, NONE)
+  BUILTIN_VDC_P (BINOPP, combine, 0, NONE)
+  BUILTIN_VB (BINOPP, pmul, 0, NONE)
+  VAR1 (BINOPP, pmull, 0, NONE, v8qi)
+  VAR1 (BINOPP, pmull_hi, 0, NONE, v16qi)
   BUILTIN_VHSDF_HSDF (BINOP, fmulx, 0, FP)
   BUILTIN_VHSDF_DF (UNOP, sqrt, 2, FP)
   BUILTIN_VDQ_I (BINOP, addp, 0, NONE)
+  BUILTIN_VDQ_I (BINOPU, addp, 0, NONE)
   VAR1 (UNOP, addp, 0, NONE, di)
+  VAR1 (UNOPU, addp, 0, NONE, di)
   BUILTIN_VDQ_BHSI (UNOP, clrsb, 2, NONE)
   BUILTIN_VDQ_BHSI (UNOP, clz, 2, NONE)
   BUILTIN_VS (UNOP, ctz, 2, NONE)
@@ -76,85 +79,112 @@
   BUILTIN_VSDQ_I (BINOP_SSU, suqadd, 0, NONE)
   BUILTIN_VSDQ_I (BINOP_UUS, usqadd, 0, NONE)
 
-  /* Implemented by aarch64_get_dreg<VSTRUCT:mode><VDC:mode>.  */
-  BUILTIN_VDC (GETREG, get_dregoi, 0, AUTO_FP)
-  BUILTIN_VDC (GETREG, get_dregci, 0, AUTO_FP)
-  BUILTIN_VDC (GETREG, get_dregxi, 0, AUTO_FP)
-  VAR1 (GETREGP, get_dregoi, 0, AUTO_FP, di)
-  VAR1 (GETREGP, get_dregci, 0, AUTO_FP, di)
-  VAR1 (GETREGP, get_dregxi, 0, AUTO_FP, di)
-  /* Implemented by aarch64_get_qreg<VSTRUCT:mode><VQ:mode>.  */
-  BUILTIN_VQ (GETREG, get_qregoi, 0, AUTO_FP)
-  BUILTIN_VQ (GETREG, get_qregci, 0, AUTO_FP)
-  BUILTIN_VQ (GETREG, get_qregxi, 0, AUTO_FP)
-  VAR1 (GETREGP, get_qregoi, 0, AUTO_FP, v2di)
-  VAR1 (GETREGP, get_qregci, 0, AUTO_FP, v2di)
-  VAR1 (GETREGP, get_qregxi, 0, AUTO_FP, v2di)
-  /* Implemented by aarch64_set_qreg<VSTRUCT:mode><VQ:mode>.  */
-  BUILTIN_VQ (SETREG, set_qregoi, 0, AUTO_FP)
-  BUILTIN_VQ (SETREG, set_qregci, 0, AUTO_FP)
-  BUILTIN_VQ (SETREG, set_qregxi, 0, AUTO_FP)
-  VAR1 (SETREGP, set_qregoi, 0, AUTO_FP, v2di)
-  VAR1 (SETREGP, set_qregci, 0, AUTO_FP, v2di)
-  VAR1 (SETREGP, set_qregxi, 0, AUTO_FP, v2di)
-  /* Implemented by aarch64_ld1x2<VQ:mode>. */
-  BUILTIN_VQ (LOADSTRUCT, ld1x2, 0, LOAD)
-  /* Implemented by aarch64_ld1x2<VDC:mode>. */
-  BUILTIN_VDC (LOADSTRUCT, ld1x2, 0, LOAD)
-  /* Implemented by aarch64_ld<VSTRUCT:nregs><VDC:mode>.  */
-  BUILTIN_VDC (LOADSTRUCT, ld2, 0, LOAD)
-  BUILTIN_VDC (LOADSTRUCT, ld3, 0, LOAD)
-  BUILTIN_VDC (LOADSTRUCT, ld4, 0, LOAD)
-  /* Implemented by aarch64_ld<VSTRUCT:nregs><VQ:mode>.  */
-  BUILTIN_VQ (LOADSTRUCT, ld2, 0, LOAD)
-  BUILTIN_VQ (LOADSTRUCT, ld3, 0, LOAD)
-  BUILTIN_VQ (LOADSTRUCT, ld4, 0, LOAD)
-  /* Implemented by aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>.  */
+  /* Implemented by aarch64_ld1x2<vstruct_elt>. */
+  BUILTIN_VALLDIF (LOADSTRUCT, ld1x2, 0, LOAD)
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld1x2, 0, LOAD)
+  BUILTIN_VALLP (LOADSTRUCT_P, ld1x2, 0, LOAD)
+  /* Implemented by aarch64_ld1x3<vstruct_elt>.  */
+  BUILTIN_VALLDIF (LOADSTRUCT, ld1x3, 0, LOAD)
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld1x3, 0, LOAD)
+  BUILTIN_VALLP (LOADSTRUCT_P, ld1x3, 0, LOAD)
+  /* Implemented by aarch64_ld1x4<vstruct_elt>.  */
+  BUILTIN_VALLDIF (LOADSTRUCT, ld1x4, 0, LOAD)
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld1x4, 0, LOAD)
+  BUILTIN_VALLP (LOADSTRUCT_P, ld1x4, 0, LOAD)
+
+  /* Implemented by aarch64_st1x2<vstruct_elt>.  */
+  BUILTIN_VALLDIF (STORESTRUCT, st1x2, 0, STORE)
+  BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st1x2, 0, STORE)
+  BUILTIN_VALLP (STORESTRUCT_P, st1x2, 0, STORE)
+  /* Implemented by aarch64_st1x3<vstruct_elt>.  */
+  BUILTIN_VALLDIF (STORESTRUCT, st1x3, 0, STORE)
+  BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st1x3, 0, STORE)
+  BUILTIN_VALLP (STORESTRUCT_P, st1x3, 0, STORE)
+  /* Implemented by aarch64_st1x4<vstruct_elt>.  */
+  BUILTIN_VALLDIF (STORESTRUCT, st1x4, 0, STORE)
+  BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st1x4, 0, STORE)
+  BUILTIN_VALLP (STORESTRUCT_P, st1x4, 0, STORE)
+
+  /* Implemented by aarch64_ld<nregs><vstruct_elt>.  */
+  BUILTIN_VALLDIF (LOADSTRUCT, ld2, 0, LOAD)
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld2, 0, LOAD)
+  BUILTIN_VALLP (LOADSTRUCT_P, ld2, 0, LOAD)
+  BUILTIN_VALLDIF (LOADSTRUCT, ld3, 0, LOAD)
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld3, 0, LOAD)
+  BUILTIN_VALLP (LOADSTRUCT_P, ld3, 0, LOAD)
+  BUILTIN_VALLDIF (LOADSTRUCT, ld4, 0, LOAD)
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld4, 0, LOAD)
+  BUILTIN_VALLP (LOADSTRUCT_P, ld4, 0, LOAD)
+
+  /* Implemented by aarch64_st<nregs><vstruct_elt>.  */
+  BUILTIN_VALLDIF (STORESTRUCT, st2, 0, STORE)
+  BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st2, 0, STORE)
+  BUILTIN_VALLP (STORESTRUCT_P, st2, 0, STORE)
+  BUILTIN_VALLDIF (STORESTRUCT, st3, 0, STORE)
+  BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st3, 0, STORE)
+  BUILTIN_VALLP (STORESTRUCT_P, st3, 0, STORE)
+  BUILTIN_VALLDIF (STORESTRUCT, st4, 0, STORE)
+  BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st4, 0, STORE)
+  BUILTIN_VALLP (STORESTRUCT_P, st4, 0, STORE)
+
+  /* Implemented by aarch64_ld<nregs>r<vstruct_elt>.  */
   BUILTIN_VALLDIF (LOADSTRUCT, ld2r, 0, LOAD)
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld2r, 0, LOAD)
+  BUILTIN_VALLP (LOADSTRUCT_P, ld2r, 0, LOAD)
   BUILTIN_VALLDIF (LOADSTRUCT, ld3r, 0, LOAD)
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld3r, 0, LOAD)
+  BUILTIN_VALLP (LOADSTRUCT_P, ld3r, 0, LOAD)
   BUILTIN_VALLDIF (LOADSTRUCT, ld4r, 0, LOAD)
-  /* Implemented by aarch64_ld<VSTRUCT:nregs>_lane<VQ:mode>.  */
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld4r, 0, LOAD)
+  BUILTIN_VALLP (LOADSTRUCT_P, ld4r, 0, LOAD)
+
+  /* Implemented by aarch64_ld<nregs>_lane<vstruct_elt>.  */
   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld2_lane, 0, ALL)
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_LANE_U, ld2_lane, 0, ALL)
+  BUILTIN_VALLP (LOADSTRUCT_LANE_P, ld2_lane, 0, ALL)
   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld3_lane, 0, ALL)
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_LANE_U, ld3_lane, 0, ALL)
+  BUILTIN_VALLP (LOADSTRUCT_LANE_P, ld3_lane, 0, ALL)
   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld4_lane, 0, ALL)
-  /* Implemented by aarch64_st<VSTRUCT:nregs><VDC:mode>.  */
-  BUILTIN_VDC (STORESTRUCT, st2, 0, STORE)
-  BUILTIN_VDC (STORESTRUCT, st3, 0, STORE)
-  BUILTIN_VDC (STORESTRUCT, st4, 0, STORE)
-  /* Implemented by aarch64_st<VSTRUCT:nregs><VQ:mode>.  */
-  BUILTIN_VQ (STORESTRUCT, st2, 0, STORE)
-  BUILTIN_VQ (STORESTRUCT, st3, 0, STORE)
-  BUILTIN_VQ (STORESTRUCT, st4, 0, STORE)
+  BUILTIN_VSDQ_I_DI (LOADSTRUCT_LANE_U, ld4_lane, 0, ALL)
+  BUILTIN_VALLP (LOADSTRUCT_LANE_P, ld4_lane, 0, ALL)
 
+  /* Implemented by aarch64_st<nregs>_lane<vstruct_elt>.  */
   BUILTIN_VALLDIF (STORESTRUCT_LANE, st2_lane, 0, ALL)
+  BUILTIN_VSDQ_I_DI (STORESTRUCT_LANE_U, st2_lane, 0, ALL)
+  BUILTIN_VALLP (STORESTRUCT_LANE_P, st2_lane, 0, ALL)
   BUILTIN_VALLDIF (STORESTRUCT_LANE, st3_lane, 0, ALL)
+  BUILTIN_VSDQ_I_DI (STORESTRUCT_LANE_U, st3_lane, 0, ALL)
+  BUILTIN_VALLP (STORESTRUCT_LANE_P, st3_lane, 0, ALL)
   BUILTIN_VALLDIF (STORESTRUCT_LANE, st4_lane, 0, ALL)
+  BUILTIN_VSDQ_I_DI (STORESTRUCT_LANE_U, st4_lane, 0, ALL)
+  BUILTIN_VALLP (STORESTRUCT_LANE_P, st4_lane, 0, ALL)
 
   BUILTIN_VQW (BINOP, saddl2, 0, NONE)
-  BUILTIN_VQW (BINOP, uaddl2, 0, NONE)
+  BUILTIN_VQW (BINOPU, uaddl2, 0, NONE)
   BUILTIN_VQW (BINOP, ssubl2, 0, NONE)
-  BUILTIN_VQW (BINOP, usubl2, 0, NONE)
+  BUILTIN_VQW (BINOPU, usubl2, 0, NONE)
   BUILTIN_VQW (BINOP, saddw2, 0, NONE)
-  BUILTIN_VQW (BINOP, uaddw2, 0, NONE)
+  BUILTIN_VQW (BINOPU, uaddw2, 0, NONE)
   BUILTIN_VQW (BINOP, ssubw2, 0, NONE)
-  BUILTIN_VQW (BINOP, usubw2, 0, NONE)
+  BUILTIN_VQW (BINOPU, usubw2, 0, NONE)
   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>.  */
   BUILTIN_VD_BHSI (BINOP, saddl, 0, NONE)
-  BUILTIN_VD_BHSI (BINOP, uaddl, 0, NONE)
+  BUILTIN_VD_BHSI (BINOPU, uaddl, 0, NONE)
   BUILTIN_VD_BHSI (BINOP, ssubl, 0, NONE)
-  BUILTIN_VD_BHSI (BINOP, usubl, 0, NONE)
+  BUILTIN_VD_BHSI (BINOPU, usubl, 0, NONE)
   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>.  */
   BUILTIN_VD_BHSI (BINOP, saddw, 0, NONE)
-  BUILTIN_VD_BHSI (BINOP, uaddw, 0, NONE)
+  BUILTIN_VD_BHSI (BINOPU, uaddw, 0, NONE)
   BUILTIN_VD_BHSI (BINOP, ssubw, 0, NONE)
-  BUILTIN_VD_BHSI (BINOP, usubw, 0, NONE)
+  BUILTIN_VD_BHSI (BINOPU, usubw, 0, NONE)
   /* Implemented by aarch64_<sur>h<addsub><mode>.  */
   BUILTIN_VDQ_BHSI (BINOP, shadd, 0, NONE)
   BUILTIN_VDQ_BHSI (BINOP, shsub, 0, NONE)
-  BUILTIN_VDQ_BHSI (BINOP, uhadd, 0, NONE)
-  BUILTIN_VDQ_BHSI (BINOP, uhsub, 0, NONE)
+  BUILTIN_VDQ_BHSI (BINOPU, uhadd, 0, NONE)
+  BUILTIN_VDQ_BHSI (BINOPU, uhsub, 0, NONE)
   BUILTIN_VDQ_BHSI (BINOP, srhadd, 0, NONE)
-  BUILTIN_VDQ_BHSI (BINOP, urhadd, 0, NONE)
+  BUILTIN_VDQ_BHSI (BINOPU, urhadd, 0, NONE)
 
   /* Implemented by aarch64_<su>addlp<mode>.  */
   BUILTIN_VDQV_L (UNOP, saddlp, 0, NONE)
@@ -193,14 +223,22 @@
 
   /* Implemented by aarch64_<sur><addsub>hn<mode>.  */
   BUILTIN_VQN (BINOP, addhn, 0, NONE)
+  BUILTIN_VQN (BINOPU, addhn, 0, NONE)
   BUILTIN_VQN (BINOP, subhn, 0, NONE)
+  BUILTIN_VQN (BINOPU, subhn, 0, NONE)
   BUILTIN_VQN (BINOP, raddhn, 0, NONE)
+  BUILTIN_VQN (BINOPU, raddhn, 0, NONE)
   BUILTIN_VQN (BINOP, rsubhn, 0, NONE)
+  BUILTIN_VQN (BINOPU, rsubhn, 0, NONE)
   /* Implemented by aarch64_<sur><addsub>hn2<mode>.  */
   BUILTIN_VQN (TERNOP, addhn2, 0, NONE)
+  BUILTIN_VQN (TERNOPU, addhn2, 0, NONE)
   BUILTIN_VQN (TERNOP, subhn2, 0, NONE)
+  BUILTIN_VQN (TERNOPU, subhn2, 0, NONE)
   BUILTIN_VQN (TERNOP, raddhn2, 0, NONE)
+  BUILTIN_VQN (TERNOPU, raddhn2, 0, NONE)
   BUILTIN_VQN (TERNOP, rsubhn2, 0, NONE)
+  BUILTIN_VQN (TERNOPU, rsubhn2, 0, NONE)
 
   /* Implemented by aarch64_<us>xtl<mode>.  */
   BUILTIN_VQN (UNOP, sxtl, 0, NONE)
@@ -208,28 +246,37 @@
 
   /* Implemented by aarch64_xtn<mode>.  */
   BUILTIN_VQN (UNOP, xtn, 0, NONE)
+  BUILTIN_VQN (UNOPU, xtn, 0, NONE)
 
   /* Implemented by aarch64_mla<mode>.  */
   BUILTIN_VDQ_BHSI (TERNOP, mla, 0, NONE)
+  BUILTIN_VDQ_BHSI (TERNOPU, mla, 0, NONE)
   /* Implemented by aarch64_mla_n<mode>.  */
   BUILTIN_VDQHS (TERNOP, mla_n, 0, NONE)
+  BUILTIN_VDQHS (TERNOPU, mla_n, 0, NONE)
 
   /* Implemented by aarch64_mls<mode>.  */
   BUILTIN_VDQ_BHSI (TERNOP, mls, 0, NONE)
+  BUILTIN_VDQ_BHSI (TERNOPU, mls, 0, NONE)
   /* Implemented by aarch64_mls_n<mode>.  */
   BUILTIN_VDQHS (TERNOP, mls_n, 0, NONE)
+  BUILTIN_VDQHS (TERNOPU, mls_n, 0, NONE)
 
   /* Implemented by aarch64_shrn<mode>".  */
   BUILTIN_VQN (SHIFTIMM, shrn, 0, NONE)
+  BUILTIN_VQN (USHIFTIMM, shrn, 0, NONE)
 
   /* Implemented by aarch64_shrn2<mode>.  */
-  BUILTIN_VQN (SHIFTACC, shrn2, 0, NONE)
+  BUILTIN_VQN (SHIFT2IMM, shrn2, 0, NONE)
+  BUILTIN_VQN (USHIFT2IMM, shrn2, 0, NONE)
 
   /* Implemented by aarch64_rshrn<mode>".  */
   BUILTIN_VQN (SHIFTIMM, rshrn, 0, NONE)
+  BUILTIN_VQN (USHIFTIMM, rshrn, 0, NONE)
 
   /* Implemented by aarch64_rshrn2<mode>.  */
-  BUILTIN_VQN (SHIFTACC, rshrn2, 0, NONE)
+  BUILTIN_VQN (SHIFT2IMM, rshrn2, 0, NONE)
+  BUILTIN_VQN (USHIFT2IMM, rshrn2, 0, NONE)
 
   /* Implemented by aarch64_<su>mlsl<mode>.  */
   BUILTIN_VD_BHSI (TERNOP, smlsl, 0, NONE)
@@ -409,7 +456,7 @@
 
   BUILTIN_VDQ_I (SHIFTIMM, ashr, 3, NONE)
   VAR1 (SHIFTIMM, ashr_simd, 0, NONE, di)
-  BUILTIN_VDQ_I (SHIFTIMM, lshr, 3, NONE)
+  BUILTIN_VDQ_I (USHIFTIMM, lshr, 3, NONE)
   VAR1 (USHIFTIMM, lshr_simd, 0, NONE, di)
   /* Implemented by aarch64_<sur>shr_n<mode>.  */
   BUILTIN_VSDQ_I_DI (SHIFTIMM, srshr_n, 0, NONE)
@@ -458,7 +505,8 @@
   BUILTIN_VSDQ_I (USHIFTIMM, uqshl_n, 0, NONE)
 
   /* Implemented by aarch64_xtn2<mode>.  */
-  BUILTIN_VQN (UNOP, xtn2, 0, NONE)
+  BUILTIN_VQN (BINOP, xtn2, 0, NONE)
+  BUILTIN_VQN (BINOPU, xtn2, 0, NONE)
 
   /* Implemented by vec_unpack<su>_hi_<mode>.  */
   BUILTIN_VQW (UNOP, vec_unpacks_hi_, 10, NONE)
@@ -466,6 +514,7 @@
 
   /* Implemented by aarch64_reduc_plus_<mode>.  */
   BUILTIN_VALL (UNOP, reduc_plus_scal_, 10, NONE)
+  BUILTIN_VDQ_I (UNOPU, reduc_plus_scal_, 10, NONE)
 
   /* Implemented by reduc_<maxmin_uns>_scal_<mode> (producing scalar).  */
   BUILTIN_VDQIF_F16 (UNOP, reduc_smax_scal_, 10, NONE)
@@ -475,21 +524,19 @@
   BUILTIN_VHSDF (UNOP, reduc_smax_nan_scal_, 10, NONE)
   BUILTIN_VHSDF (UNOP, reduc_smin_nan_scal_, 10, NONE)
 
-  /* Implemented by <maxmin_uns><mode>3.
-     smax variants map to fmaxnm,
-     smax_nan variants map to fmax.  */
+  /* Implemented by <optab><mode>3.  */
   BUILTIN_VDQ_BHSI (BINOP, smax, 3, NONE)
   BUILTIN_VDQ_BHSI (BINOP, smin, 3, NONE)
   BUILTIN_VDQ_BHSI (BINOP, umax, 3, NONE)
   BUILTIN_VDQ_BHSI (BINOP, umin, 3, NONE)
-  BUILTIN_VHSDF_DF (BINOP, smax_nan, 3, NONE)
-  BUILTIN_VHSDF_DF (BINOP, smin_nan, 3, NONE)
 
-  /* Implemented by <maxmin_uns><mode>3.  */
+  /* Implemented by <fmaxmin><mode>3.  */
   BUILTIN_VHSDF_HSDF (BINOP, fmax, 3, FP)
   BUILTIN_VHSDF_HSDF (BINOP, fmin, 3, FP)
+  BUILTIN_VHSDF_DF (BINOP, fmax_nan, 3, FP)
+  BUILTIN_VHSDF_DF (BINOP, fmin_nan, 3, FP)
 
-  /* Implemented by aarch64_<maxmin_uns>p<mode>.  */
+  /* Implemented by aarch64_<optab>p<mode>.  */
   BUILTIN_VDQ_BHSI (BINOP, smaxp, 0, NONE)
   BUILTIN_VDQ_BHSI (BINOP, sminp, 0, NONE)
   BUILTIN_VDQ_BHSI (BINOP, umaxp, 0, NONE)
@@ -651,26 +698,13 @@
 
   /* Implemented by aarch64_ld1<VALL_F16:mode>.  */
   BUILTIN_VALL_F16 (LOAD1, ld1, 0, LOAD)
-  VAR1(STORE1P, ld1, 0, ALL, v2di)
+  BUILTIN_VDQ_I (LOAD1_U, ld1, 0, LOAD)
+  BUILTIN_VALLP_NO_DI (LOAD1_P, ld1, 0, LOAD)
 
   /* Implemented by aarch64_st1<VALL_F16:mode>.  */
   BUILTIN_VALL_F16 (STORE1, st1, 0, STORE)
-  VAR1 (STORE1P, st1, 0, STORE, v2di)
-
-  /* Implemented by aarch64_ld1x3<VALLDIF:mode>.  */
-  BUILTIN_VALLDIF (LOADSTRUCT, ld1x3, 0, LOAD)
-
-  /* Implemented by aarch64_ld1x4<VALLDIF:mode>.  */
-  BUILTIN_VALLDIF (LOADSTRUCT, ld1x4, 0, LOAD)
-
-  /* Implemented by aarch64_st1x2<VALLDIF:mode>.  */
-  BUILTIN_VALLDIF (STORESTRUCT, st1x2, 0, STORE)
-
-  /* Implemented by aarch64_st1x3<VALLDIF:mode>.  */
-  BUILTIN_VALLDIF (STORESTRUCT, st1x3, 0, STORE)
-
-  /* Implemented by aarch64_st1x4<VALLDIF:mode>.  */
-  BUILTIN_VALLDIF (STORESTRUCT, st1x4, 0, STORE)
+  BUILTIN_VDQ_I (STORE1_U, st1, 0, STORE)
+  BUILTIN_VALLP_NO_DI (STORE1_P, st1, 0, STORE)
 
   /* Implemented by fma<mode>4.  */
   BUILTIN_VHSDF (TERNOP, fma, 4, FP)
@@ -726,12 +760,21 @@
 
   /* Implemented by aarch64_qtbl2<mode>.  */
   VAR2 (BINOP, qtbl2, 0, NONE, v8qi, v16qi)
+  VAR2 (BINOPU, qtbl2, 0, NONE, v8qi, v16qi)
+  VAR2 (BINOP_PPU, qtbl2, 0, NONE, v8qi, v16qi)
+  VAR2 (BINOP_SSU, qtbl2, 0, NONE, v8qi, v16qi)
 
   /* Implemented by aarch64_qtbl3<mode>.  */
   VAR2 (BINOP, qtbl3, 0, NONE, v8qi, v16qi)
+  VAR2 (BINOPU, qtbl3, 0, NONE, v8qi, v16qi)
+  VAR2 (BINOP_PPU, qtbl3, 0, NONE, v8qi, v16qi)
+  VAR2 (BINOP_SSU, qtbl3, 0, NONE, v8qi, v16qi)
 
   /* Implemented by aarch64_qtbl4<mode>.  */
   VAR2 (BINOP, qtbl4, 0, NONE, v8qi, v16qi)
+  VAR2 (BINOPU, qtbl4, 0, NONE, v8qi, v16qi)
+  VAR2 (BINOP_PPU, qtbl4, 0, NONE, v8qi, v16qi)
+  VAR2 (BINOP_SSU, qtbl4, 0, NONE, v8qi, v16qi)
 
   /* Implemented by aarch64_qtbx1<mode>.  */
   VAR2 (TERNOP, qtbx1, 0, NONE, v8qi, v16qi)
@@ -741,12 +784,21 @@
 
   /* Implemented by aarch64_qtbx2<mode>.  */
   VAR2 (TERNOP, qtbx2, 0, NONE, v8qi, v16qi)
+  VAR2 (TERNOPU, qtbx2, 0, NONE, v8qi, v16qi)
+  VAR2 (TERNOP_PPPU, qtbx2, 0, NONE, v8qi, v16qi)
+  VAR2 (TERNOP_SSSU, qtbx2, 0, NONE, v8qi, v16qi)
 
   /* Implemented by aarch64_qtbx3<mode>.  */
   VAR2 (TERNOP, qtbx3, 0, NONE, v8qi, v16qi)
+  VAR2 (TERNOPU, qtbx3, 0, NONE, v8qi, v16qi)
+  VAR2 (TERNOP_PPPU, qtbx3, 0, NONE, v8qi, v16qi)
+  VAR2 (TERNOP_SSSU, qtbx3, 0, NONE, v8qi, v16qi)
 
   /* Implemented by aarch64_qtbx4<mode>.  */
   VAR2 (TERNOP, qtbx4, 0, NONE, v8qi, v16qi)
+  VAR2 (TERNOPU, qtbx4, 0, NONE, v8qi, v16qi)
+  VAR2 (TERNOP_PPPU, qtbx4, 0, NONE, v8qi, v16qi)
+  VAR2 (TERNOP_SSSU, qtbx4, 0, NONE, v8qi, v16qi)
 
   /* Builtins for ARMv8.1-A Adv.SIMD instructions.  */
 
diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md
index 29f3817..1020cd9 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -69,17 +69,19 @@
   "TARGET_SIMD"
   "@
    dup\\t%0.<Vtype>, %1.<Vetype>[0]
-   dup\\t%0.<Vtype>, %<vw>1"
+   dup\\t%0.<Vtype>, %<vwcore>1"
   [(set_attr "type" "neon_dup<q>, neon_from_gp<q>")]
 )
 
 (define_insn "aarch64_simd_dup<mode>"
-  [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
+  [(set (match_operand:VDQF_F16 0 "register_operand" "=w,w")
 	(vec_duplicate:VDQF_F16
-	  (match_operand:<VEL> 1 "register_operand" "w")))]
+	  (match_operand:<VEL> 1 "register_operand" "w,r")))]
   "TARGET_SIMD"
-  "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
-  [(set_attr "type" "neon_dup<q>")]
+  "@
+   dup\\t%0.<Vtype>, %1.<Vetype>[0]
+   dup\\t%0.<Vtype>, %<vwcore>1"
+  [(set_attr "type" "neon_dup<q>, neon_from_gp<q>")]
 )
 
 (define_insn "aarch64_dup_lane<mode>"
@@ -1551,7 +1553,7 @@
 })
 
 ;; Pairwise Integer Max/Min operations.
-(define_insn "aarch64_<maxmin_uns>p<mode>"
+(define_insn "aarch64_<optab>p<mode>"
  [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
        (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
 			 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
@@ -1562,7 +1564,7 @@
 )
 
 ;; Pairwise FP Max/Min operations.
-(define_insn "aarch64_<maxmin_uns>p<mode>"
+(define_insn "aarch64_<optab>p<mode>"
  [(set (match_operand:VHSDF 0 "register_operand" "=w")
        (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
 		      (match_operand:VHSDF 2 "register_operand" "w")]
@@ -1885,22 +1887,6 @@
   [(set_attr "type" "neon_permute<q>")]
 )
 
-(define_insn "*aarch64_topbits_shuffle<mode>_le"
-  [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
-	(vec_concat:<VNARROWQ2>
-          (unspec:<VNARROWQ> [
-              (match_operand:VQN 1 "register_operand" "w")
-	      (match_operand:VQN 2 "aarch64_simd_shift_imm_vec_exact_top")
-	     ] UNSPEC_RSHRN)
-	  (unspec:<VNARROWQ> [
-	      (match_operand:VQN 3 "register_operand" "w")
-	      (match_dup 2)
-	     ] UNSPEC_RSHRN)))]
-  "TARGET_SIMD && !BYTES_BIG_ENDIAN"
-  "uzp2\\t%0.<V2ntype>, %1.<V2ntype>, %3.<V2ntype>"
-  [(set_attr "type" "neon_permute<q>")]
-)
-
 (define_insn "*aarch64_<srn_op>topbits_shuffle<mode>_be"
   [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
 	(vec_concat:<VNARROWQ2>
@@ -1915,22 +1901,6 @@
   [(set_attr "type" "neon_permute<q>")]
 )
 
-(define_insn "*aarch64_topbits_shuffle<mode>_be"
-  [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
-	(vec_concat:<VNARROWQ2>
-	  (unspec:<VNARROWQ> [
-	      (match_operand:VQN 3 "register_operand" "w")
-	      (match_operand:VQN 2 "aarch64_simd_shift_imm_vec_exact_top")
-	     ] UNSPEC_RSHRN)
-          (unspec:<VNARROWQ> [
-              (match_operand:VQN 1 "register_operand" "w")
-	      (match_dup 2)
-	     ] UNSPEC_RSHRN)))]
-  "TARGET_SIMD && BYTES_BIG_ENDIAN"
-  "uzp2\\t%0.<V2ntype>, %1.<V2ntype>, %3.<V2ntype>"
-  [(set_attr "type" "neon_permute<q>")]
-)
-
 (define_expand "aarch64_shrn<mode>"
   [(set (match_operand:<VNARROWQ> 0 "register_operand")
 	(truncate:<VNARROWQ>
@@ -3486,7 +3456,7 @@
 ;; Vector forms for fmax, fmin, fmaxnm, fminnm.
 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
 ;; which implement the IEEE fmax ()/fmin () functions.
-(define_insn "<maxmin_uns><mode>3"
+(define_insn "<fmaxmin><mode>3"
   [(set (match_operand:VHSDF 0 "register_operand" "=w")
        (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
 		      (match_operand:VHSDF 2 "register_operand" "w")]
@@ -3620,7 +3590,7 @@
 
 ;; Template for outputting a scalar, so we can create __builtins which can be
 ;; gimple_fold'd to the IFN_REDUC_(MAX|MIN) function.  (This is FP smax/smin).
-(define_expand "reduc_<maxmin_uns>_scal_<mode>"
+(define_expand "reduc_<optab>_scal_<mode>"
   [(match_operand:<VEL> 0 "register_operand")
    (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand")]
 		  FMAXMINV)]
@@ -3628,15 +3598,15 @@
   {
     rtx elt = aarch64_endian_lane_rtx (<MODE>mode, 0);
     rtx scratch = gen_reg_rtx (<MODE>mode);
-    emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
-							      operands[1]));
+    emit_insn (gen_aarch64_reduc_<optab>_internal<mode> (scratch,
+							 operands[1]));
     emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
     DONE;
   }
 )
 
 ;; Likewise for integer cases, signed and unsigned.
-(define_expand "reduc_<maxmin_uns>_scal_<mode>"
+(define_expand "reduc_<optab>_scal_<mode>"
   [(match_operand:<VEL> 0 "register_operand")
    (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
 		    MAXMINV)]
@@ -3644,14 +3614,14 @@
   {
     rtx elt = aarch64_endian_lane_rtx (<MODE>mode, 0);
     rtx scratch = gen_reg_rtx (<MODE>mode);
-    emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
-							      operands[1]));
+    emit_insn (gen_aarch64_reduc_<optab>_internal<mode> (scratch,
+							 operands[1]));
     emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
     DONE;
   }
 )
 
-(define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
+(define_insn "aarch64_reduc_<optab>_internal<mode>"
  [(set (match_operand:VDQV_S 0 "register_operand" "=w")
        (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
 		    MAXMINV))]
@@ -3660,7 +3630,7 @@
   [(set_attr "type" "neon_reduc_minmax<q>")]
 )
 
-(define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
+(define_insn "aarch64_reduc_<optab>_internalv2si"
  [(set (match_operand:V2SI 0 "register_operand" "=w")
        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
 		    MAXMINV))]
@@ -3669,7 +3639,7 @@
   [(set_attr "type" "neon_reduc_minmax")]
 )
 
-(define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
+(define_insn "aarch64_reduc_<optab>_internal<mode>"
  [(set (match_operand:VHSDF 0 "register_operand" "=w")
        (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
 		      FMAXMINV))]
@@ -6766,162 +6736,165 @@
 
 ;; Patterns for vector struct loads and stores.
 
-(define_insn "aarch64_simd_ld2<mode>"
-  [(set (match_operand:OI 0 "register_operand" "=w")
-	(unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
-		    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD2))]
+(define_insn "aarch64_simd_ld2<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_2Q 0 "register_operand" "=w")
+	(unspec:VSTRUCT_2Q [
+	  (match_operand:VSTRUCT_2Q 1 "aarch64_simd_struct_operand" "Utv")]
+	  UNSPEC_LD2))]
   "TARGET_SIMD"
   "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
   [(set_attr "type" "neon_load2_2reg<q>")]
 )
 
-(define_insn "aarch64_simd_ld2r<mode>"
-  [(set (match_operand:OI 0 "register_operand" "=w")
-       (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-                   (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
-                  UNSPEC_LD2_DUP))]
+(define_insn "aarch64_simd_ld2r<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_2QD 0 "register_operand" "=w")
+	(unspec:VSTRUCT_2QD [
+	  (match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
+          UNSPEC_LD2_DUP))]
   "TARGET_SIMD"
   "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
   [(set_attr "type" "neon_load2_all_lanes<q>")]
 )
 
-(define_insn "aarch64_vec_load_lanesoi_lane<mode>"
-  [(set (match_operand:OI 0 "register_operand" "=w")
-	(unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-		    (match_operand:OI 2 "register_operand" "0")
-		    (match_operand:SI 3 "immediate_operand" "i")
-		    (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
-		   UNSPEC_LD2_LANE))]
+(define_insn "aarch64_vec_load_lanes<mode>_lane<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_2QD 0 "register_operand" "=w")
+	(unspec:VSTRUCT_2QD [
+		(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
+		(match_operand:VSTRUCT_2QD 2 "register_operand" "0")
+		(match_operand:SI 3 "immediate_operand" "i")]
+		UNSPEC_LD2_LANE))]
   "TARGET_SIMD"
   {
-    operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<VSTRUCT_ELT>mode,
+					   INTVAL (operands[3]));
     return "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1";
   }
   [(set_attr "type" "neon_load2_one_lane")]
 )
 
-(define_expand "vec_load_lanesoi<mode>"
-  [(set (match_operand:OI 0 "register_operand")
-	(unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand")
-		    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD2))]
+(define_expand "vec_load_lanes<mode><vstruct_elt>"
+  [(set (match_operand:VSTRUCT_2Q 0 "register_operand")
+	(unspec:VSTRUCT_2Q [
+		(match_operand:VSTRUCT_2Q 1 "aarch64_simd_struct_operand")]
+		UNSPEC_LD2))]
   "TARGET_SIMD"
 {
   if (BYTES_BIG_ENDIAN)
     {
-      rtx tmp = gen_reg_rtx (OImode);
-      rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
-      emit_insn (gen_aarch64_simd_ld2<mode> (tmp, operands[1]));
-      emit_insn (gen_aarch64_rev_reglistoi (operands[0], tmp, mask));
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
+			GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
+      emit_insn (gen_aarch64_simd_ld2<vstruct_elt> (tmp, operands[1]));
+      emit_insn (gen_aarch64_rev_reglist<mode> (operands[0], tmp, mask));
     }
   else
-    emit_insn (gen_aarch64_simd_ld2<mode> (operands[0], operands[1]));
+    emit_insn (gen_aarch64_simd_ld2<vstruct_elt> (operands[0], operands[1]));
   DONE;
 })
 
-(define_insn "aarch64_simd_st2<mode>"
-  [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:OI [(match_operand:OI 1 "register_operand" "w")
-                    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-                   UNSPEC_ST2))]
+(define_insn "aarch64_simd_st2<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_2Q 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_2Q [
+		(match_operand:VSTRUCT_2Q 1 "register_operand" "w")]
+                UNSPEC_ST2))]
   "TARGET_SIMD"
   "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
   [(set_attr "type" "neon_store2_2reg<q>")]
 )
 
 ;; RTL uses GCC vector extension indices, so flip only for assembly.
-(define_insn "aarch64_vec_store_lanesoi_lane<mode>"
+(define_insn "aarch64_vec_store_lanes<mode>_lane<vstruct_elt>"
   [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:BLK [(match_operand:OI 1 "register_operand" "w")
-		    (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
-		    (match_operand:SI 2 "immediate_operand" "i")]
-		   UNSPEC_ST2_LANE))]
+	(unspec:BLK [(match_operand:VSTRUCT_2QD 1 "register_operand" "w")
+		     (match_operand:SI 2 "immediate_operand" "i")]
+		     UNSPEC_ST2_LANE))]
   "TARGET_SIMD"
   {
-    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<VSTRUCT_ELT>mode,
+					   INTVAL (operands[2]));
     return "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0";
   }
   [(set_attr "type" "neon_store2_one_lane<q>")]
 )
 
-(define_expand "vec_store_lanesoi<mode>"
-  [(set (match_operand:OI 0 "aarch64_simd_struct_operand")
-	(unspec:OI [(match_operand:OI 1 "register_operand")
-                    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+(define_expand "vec_store_lanes<mode><vstruct_elt>"
+  [(set (match_operand:VSTRUCT_2Q 0 "aarch64_simd_struct_operand")
+	(unspec:VSTRUCT_2Q [(match_operand:VSTRUCT_2Q 1 "register_operand")]
                    UNSPEC_ST2))]
   "TARGET_SIMD"
 {
   if (BYTES_BIG_ENDIAN)
     {
-      rtx tmp = gen_reg_rtx (OImode);
-      rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
-      emit_insn (gen_aarch64_rev_reglistoi (tmp, operands[1], mask));
-      emit_insn (gen_aarch64_simd_st2<mode> (operands[0], tmp));
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
+			GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
+      emit_insn (gen_aarch64_rev_reglist<mode> (tmp, operands[1], mask));
+      emit_insn (gen_aarch64_simd_st2<vstruct_elt> (operands[0], tmp));
     }
   else
-    emit_insn (gen_aarch64_simd_st2<mode> (operands[0], operands[1]));
+    emit_insn (gen_aarch64_simd_st2<vstruct_elt> (operands[0], operands[1]));
   DONE;
 })
 
-(define_insn "aarch64_simd_ld3<mode>"
-  [(set (match_operand:CI 0 "register_operand" "=w")
-	(unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
-		    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD3))]
+(define_insn "aarch64_simd_ld3<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_3Q 0 "register_operand" "=w")
+	(unspec:VSTRUCT_3Q [
+	  (match_operand:VSTRUCT_3Q 1 "aarch64_simd_struct_operand" "Utv")]
+	  UNSPEC_LD3))]
   "TARGET_SIMD"
   "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
   [(set_attr "type" "neon_load3_3reg<q>")]
 )
 
-(define_insn "aarch64_simd_ld3r<mode>"
-  [(set (match_operand:CI 0 "register_operand" "=w")
-       (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-                   (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
-                  UNSPEC_LD3_DUP))]
+(define_insn "aarch64_simd_ld3r<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_3QD 0 "register_operand" "=w")
+	(unspec:VSTRUCT_3QD [
+	  (match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
+          UNSPEC_LD3_DUP))]
   "TARGET_SIMD"
   "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
   [(set_attr "type" "neon_load3_all_lanes<q>")]
 )
 
-(define_insn "aarch64_vec_load_lanesci_lane<mode>"
-  [(set (match_operand:CI 0 "register_operand" "=w")
-	(unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-		    (match_operand:CI 2 "register_operand" "0")
-		    (match_operand:SI 3 "immediate_operand" "i")
-		    (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD3_LANE))]
+(define_insn "aarch64_vec_load_lanes<mode>_lane<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_3QD 0 "register_operand" "=w")
+	(unspec:VSTRUCT_3QD [
+		(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
+		(match_operand:VSTRUCT_3QD 2 "register_operand" "0")
+		(match_operand:SI 3 "immediate_operand" "i")]
+		UNSPEC_LD3_LANE))]
   "TARGET_SIMD"
 {
-    operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<VSTRUCT_ELT>mode,
+					   INTVAL (operands[3]));
     return "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1";
 }
   [(set_attr "type" "neon_load3_one_lane")]
 )
 
-(define_expand "vec_load_lanesci<mode>"
-  [(set (match_operand:CI 0 "register_operand")
-	(unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand")
-		    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD3))]
+(define_expand "vec_load_lanes<mode><vstruct_elt>"
+  [(set (match_operand:VSTRUCT_3Q 0 "register_operand")
+	(unspec:VSTRUCT_3Q [
+		(match_operand:VSTRUCT_3Q 1 "aarch64_simd_struct_operand")]
+		UNSPEC_LD3))]
   "TARGET_SIMD"
 {
   if (BYTES_BIG_ENDIAN)
     {
-      rtx tmp = gen_reg_rtx (CImode);
-      rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
-      emit_insn (gen_aarch64_simd_ld3<mode> (tmp, operands[1]));
-      emit_insn (gen_aarch64_rev_reglistci (operands[0], tmp, mask));
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
+			GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
+      emit_insn (gen_aarch64_simd_ld3<vstruct_elt> (tmp, operands[1]));
+      emit_insn (gen_aarch64_rev_reglist<mode> (operands[0], tmp, mask));
     }
   else
-    emit_insn (gen_aarch64_simd_ld3<mode> (operands[0], operands[1]));
+    emit_insn (gen_aarch64_simd_ld3<vstruct_elt> (operands[0], operands[1]));
   DONE;
 })
 
-(define_insn "aarch64_simd_st3<mode>"
-  [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:CI [(match_operand:CI 1 "register_operand" "w")
-                    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+(define_insn "aarch64_simd_st3<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_3Q 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_3Q [(match_operand:VSTRUCT_3Q 1 "register_operand" "w")]
                    UNSPEC_ST3))]
   "TARGET_SIMD"
   "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
@@ -6929,141 +6902,144 @@
 )
 
 ;; RTL uses GCC vector extension indices, so flip only for assembly.
-(define_insn "aarch64_vec_store_lanesci_lane<mode>"
+(define_insn "aarch64_vec_store_lanes<mode>_lane<vstruct_elt>"
   [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:BLK [(match_operand:CI 1 "register_operand" "w")
-		     (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
+	(unspec:BLK [(match_operand:VSTRUCT_3QD 1 "register_operand" "w")
 		     (match_operand:SI 2 "immediate_operand" "i")]
-		    UNSPEC_ST3_LANE))]
+		     UNSPEC_ST3_LANE))]
   "TARGET_SIMD"
   {
-    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<VSTRUCT_ELT>mode,
+					   INTVAL (operands[2]));
     return "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0";
   }
   [(set_attr "type" "neon_store3_one_lane<q>")]
 )
 
-(define_expand "vec_store_lanesci<mode>"
-  [(set (match_operand:CI 0 "aarch64_simd_struct_operand")
-	(unspec:CI [(match_operand:CI 1 "register_operand")
-                    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-                   UNSPEC_ST3))]
+(define_expand "vec_store_lanes<mode><vstruct_elt>"
+  [(set (match_operand:VSTRUCT_3Q 0 "aarch64_simd_struct_operand")
+	(unspec:VSTRUCT_3Q [
+		(match_operand:VSTRUCT_3Q 1 "register_operand")]
+                UNSPEC_ST3))]
   "TARGET_SIMD"
 {
   if (BYTES_BIG_ENDIAN)
     {
-      rtx tmp = gen_reg_rtx (CImode);
-      rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
-      emit_insn (gen_aarch64_rev_reglistci (tmp, operands[1], mask));
-      emit_insn (gen_aarch64_simd_st3<mode> (operands[0], tmp));
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
+			GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
+      emit_insn (gen_aarch64_rev_reglist<mode> (tmp, operands[1], mask));
+      emit_insn (gen_aarch64_simd_st3<vstruct_elt> (operands[0], tmp));
     }
   else
-    emit_insn (gen_aarch64_simd_st3<mode> (operands[0], operands[1]));
+    emit_insn (gen_aarch64_simd_st3<vstruct_elt> (operands[0], operands[1]));
   DONE;
 })
 
-(define_insn "aarch64_simd_ld4<mode>"
-  [(set (match_operand:XI 0 "register_operand" "=w")
-	(unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
-		    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD4))]
+(define_insn "aarch64_simd_ld4<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_4Q 0 "register_operand" "=w")
+	(unspec:VSTRUCT_4Q [
+	  (match_operand:VSTRUCT_4Q 1 "aarch64_simd_struct_operand" "Utv")]
+	  UNSPEC_LD4))]
   "TARGET_SIMD"
   "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
   [(set_attr "type" "neon_load4_4reg<q>")]
 )
 
-(define_insn "aarch64_simd_ld4r<mode>"
-  [(set (match_operand:XI 0 "register_operand" "=w")
-       (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-                   (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
-                  UNSPEC_LD4_DUP))]
+(define_insn "aarch64_simd_ld4r<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_4QD 0 "register_operand" "=w")
+	(unspec:VSTRUCT_4QD [
+	  (match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
+          UNSPEC_LD4_DUP))]
   "TARGET_SIMD"
   "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
   [(set_attr "type" "neon_load4_all_lanes<q>")]
 )
 
-(define_insn "aarch64_vec_load_lanesxi_lane<mode>"
-  [(set (match_operand:XI 0 "register_operand" "=w")
-	(unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-		    (match_operand:XI 2 "register_operand" "0")
-		    (match_operand:SI 3 "immediate_operand" "i")
-		    (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD4_LANE))]
+(define_insn "aarch64_vec_load_lanes<mode>_lane<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_4QD 0 "register_operand" "=w")
+	(unspec:VSTRUCT_4QD [
+		(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
+		(match_operand:VSTRUCT_4QD 2 "register_operand" "0")
+		(match_operand:SI 3 "immediate_operand" "i")]
+		UNSPEC_LD4_LANE))]
   "TARGET_SIMD"
 {
-    operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
+    operands[3] = aarch64_endian_lane_rtx (<VSTRUCT_ELT>mode,
+					   INTVAL (operands[3]));
     return "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1";
 }
   [(set_attr "type" "neon_load4_one_lane")]
 )
 
-(define_expand "vec_load_lanesxi<mode>"
-  [(set (match_operand:XI 0 "register_operand")
-	(unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand")
-		    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD4))]
+(define_expand "vec_load_lanes<mode><vstruct_elt>"
+  [(set (match_operand:VSTRUCT_4Q 0 "register_operand")
+	(unspec:VSTRUCT_4Q [
+		(match_operand:VSTRUCT_4Q 1 "aarch64_simd_struct_operand")]
+		UNSPEC_LD4))]
   "TARGET_SIMD"
 {
   if (BYTES_BIG_ENDIAN)
     {
-      rtx tmp = gen_reg_rtx (XImode);
-      rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
-      emit_insn (gen_aarch64_simd_ld4<mode> (tmp, operands[1]));
-      emit_insn (gen_aarch64_rev_reglistxi (operands[0], tmp, mask));
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
+			GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
+      emit_insn (gen_aarch64_simd_ld4<vstruct_elt> (tmp, operands[1]));
+      emit_insn (gen_aarch64_rev_reglist<mode> (operands[0], tmp, mask));
     }
   else
-    emit_insn (gen_aarch64_simd_ld4<mode> (operands[0], operands[1]));
+    emit_insn (gen_aarch64_simd_ld4<vstruct_elt> (operands[0], operands[1]));
   DONE;
 })
 
-(define_insn "aarch64_simd_st4<mode>"
-  [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:XI [(match_operand:XI 1 "register_operand" "w")
-                    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-                   UNSPEC_ST4))]
+(define_insn "aarch64_simd_st4<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_4Q 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_4Q [
+		(match_operand:VSTRUCT_4Q 1 "register_operand" "w")]
+                UNSPEC_ST4))]
   "TARGET_SIMD"
   "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
   [(set_attr "type" "neon_store4_4reg<q>")]
 )
 
 ;; RTL uses GCC vector extension indices, so flip only for assembly.
-(define_insn "aarch64_vec_store_lanesxi_lane<mode>"
+(define_insn "aarch64_vec_store_lanes<mode>_lane<vstruct_elt>"
   [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:BLK [(match_operand:XI 1 "register_operand" "w")
-		     (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
+	(unspec:BLK [(match_operand:VSTRUCT_4QD 1 "register_operand" "w")
 		     (match_operand:SI 2 "immediate_operand" "i")]
-		    UNSPEC_ST4_LANE))]
+		     UNSPEC_ST4_LANE))]
   "TARGET_SIMD"
   {
-    operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
+    operands[2] = aarch64_endian_lane_rtx (<VSTRUCT_ELT>mode,
+					   INTVAL (operands[2]));
     return "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0";
   }
   [(set_attr "type" "neon_store4_one_lane<q>")]
 )
 
-(define_expand "vec_store_lanesxi<mode>"
-  [(set (match_operand:XI 0 "aarch64_simd_struct_operand")
-	(unspec:XI [(match_operand:XI 1 "register_operand")
-                    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+(define_expand "vec_store_lanes<mode><vstruct_elt>"
+  [(set (match_operand:VSTRUCT_4Q 0 "aarch64_simd_struct_operand")
+	(unspec:VSTRUCT_4Q [(match_operand:VSTRUCT_4Q 1 "register_operand")]
                    UNSPEC_ST4))]
   "TARGET_SIMD"
 {
   if (BYTES_BIG_ENDIAN)
     {
-      rtx tmp = gen_reg_rtx (XImode);
-      rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
-      emit_insn (gen_aarch64_rev_reglistxi (tmp, operands[1], mask));
-      emit_insn (gen_aarch64_simd_st4<mode> (operands[0], tmp));
+      rtx tmp = gen_reg_rtx (<MODE>mode);
+      rtx mask = aarch64_reverse_mask (<VSTRUCT_ELT>mode,
+			GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>);
+      emit_insn (gen_aarch64_rev_reglist<mode> (tmp, operands[1], mask));
+      emit_insn (gen_aarch64_simd_st4<vstruct_elt> (operands[0], tmp));
     }
   else
-    emit_insn (gen_aarch64_simd_st4<mode> (operands[0], operands[1]));
+    emit_insn (gen_aarch64_simd_st4<vstruct_elt> (operands[0], operands[1]));
   DONE;
 })
 
 (define_insn_and_split "aarch64_rev_reglist<mode>"
-[(set (match_operand:VSTRUCT 0 "register_operand" "=&w")
-	(unspec:VSTRUCT
-	           [(match_operand:VSTRUCT 1 "register_operand" "w")
+[(set (match_operand:VSTRUCT_QD 0 "register_operand" "=&w")
+	(unspec:VSTRUCT_QD
+	           [(match_operand:VSTRUCT_QD 1 "register_operand" "w")
 		    (match_operand:V16QI 2 "register_operand" "w")]
                    UNSPEC_REV_REGLIST))]
   "TARGET_SIMD"
@@ -7072,7 +7048,7 @@
   [(const_int 0)]
 {
   int i;
-  int nregs = GET_MODE_SIZE (<MODE>mode) / UNITS_PER_VREG;
+  int nregs = GET_MODE_SIZE (<MODE>mode).to_constant () / UNITS_PER_VREG;
   for (i = 0; i < nregs; i++)
     {
       rtx op0 = gen_rtx_REG (V16QImode, REGNO (operands[0]) + i);
@@ -7088,6 +7064,18 @@
 ;; Reload patterns for AdvSIMD register list operands.
 
 (define_expand "mov<mode>"
+  [(set (match_operand:VSTRUCT_QD 0 "nonimmediate_operand")
+	(match_operand:VSTRUCT_QD 1 "general_operand"))]
+  "TARGET_SIMD"
+{
+  if (can_create_pseudo_p ())
+    {
+      if (GET_CODE (operands[0]) != REG)
+	operands[1] = force_reg (<MODE>mode, operands[1]);
+    }
+})
+
+(define_expand "mov<mode>"
   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand")
 	(match_operand:VSTRUCT 1 "general_operand"))]
   "TARGET_SIMD"
@@ -7099,115 +7087,122 @@
     }
 })
 
-
-(define_expand "aarch64_ld1x3<VALLDIF:mode>"
-  [(match_operand:CI 0 "register_operand")
-   (match_operand:DI 1 "register_operand")
-   (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+(define_expand "aarch64_ld1x3<vstruct_elt>"
+  [(match_operand:VSTRUCT_3QD 0 "register_operand")
+   (match_operand:DI 1 "register_operand")]
   "TARGET_SIMD"
 {
-  rtx mem = gen_rtx_MEM (CImode, operands[1]);
-  emit_insn (gen_aarch64_ld1_x3_<VALLDIF:mode> (operands[0], mem));
+  rtx mem = gen_rtx_MEM (<MODE>mode, operands[1]);
+  emit_insn (gen_aarch64_ld1_x3_<vstruct_elt> (operands[0], mem));
   DONE;
 })
 
-(define_insn "aarch64_ld1_x3_<mode>"
-  [(set (match_operand:CI 0 "register_operand" "=w")
-        (unspec:CI
-	  [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
-	   (unspec:VALLDIF [(const_int 3)] UNSPEC_VSTRUCTDUMMY)] UNSPEC_LD1))]
+(define_insn "aarch64_ld1_x3_<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_3QD 0 "register_operand" "=w")
+        (unspec:VSTRUCT_3QD
+	  [(match_operand:VSTRUCT_3QD 1 "aarch64_simd_struct_operand" "Utv")]
+	  UNSPEC_LD1))]
   "TARGET_SIMD"
   "ld1\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
   [(set_attr "type" "neon_load1_3reg<q>")]
 )
 
-(define_expand "aarch64_ld1x4<VALLDIF:mode>"
-  [(match_operand:XI 0 "register_operand" "=w")
-   (match_operand:DI 1 "register_operand" "r")
-   (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+(define_expand "aarch64_ld1x4<vstruct_elt>"
+  [(match_operand:VSTRUCT_4QD 0 "register_operand" "=w")
+   (match_operand:DI 1 "register_operand" "r")]
   "TARGET_SIMD"
 {
-  rtx mem = gen_rtx_MEM (XImode, operands[1]);
-  emit_insn (gen_aarch64_ld1_x4_<VALLDIF:mode> (operands[0], mem));
+  rtx mem = gen_rtx_MEM (<MODE>mode, operands[1]);
+  emit_insn (gen_aarch64_ld1_x4_<vstruct_elt> (operands[0], mem));
   DONE;
 })
 
-(define_insn "aarch64_ld1_x4_<mode>"
-  [(set (match_operand:XI 0 "register_operand" "=w")
-	(unspec:XI
-	  [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
-	   (unspec:VALLDIF [(const_int 4)] UNSPEC_VSTRUCTDUMMY)]
+(define_insn "aarch64_ld1_x4_<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_4QD 0 "register_operand" "=w")
+	(unspec:VSTRUCT_4QD
+	  [(match_operand:VSTRUCT_4QD 1 "aarch64_simd_struct_operand" "Utv")]
 	UNSPEC_LD1))]
   "TARGET_SIMD"
   "ld1\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
   [(set_attr "type" "neon_load1_4reg<q>")]
 )
 
-(define_expand "aarch64_st1x2<VALLDIF:mode>"
+(define_expand "aarch64_st1x2<vstruct_elt>"
   [(match_operand:DI 0 "register_operand")
-   (match_operand:OI 1 "register_operand")
-   (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+   (match_operand:VSTRUCT_2QD 1 "register_operand")]
   "TARGET_SIMD"
 {
-  rtx mem = gen_rtx_MEM (OImode, operands[0]);
-  emit_insn (gen_aarch64_st1_x2_<VALLDIF:mode> (mem, operands[1]));
+  rtx mem = gen_rtx_MEM (<MODE>mode, operands[0]);
+  emit_insn (gen_aarch64_st1_x2_<vstruct_elt> (mem, operands[1]));
   DONE;
 })
 
-(define_insn "aarch64_st1_x2_<mode>"
-   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
-	 (unspec:OI
-	  [(match_operand:OI 1 "register_operand" "w")
-          (unspec:VALLDIF [(const_int 2)] UNSPEC_VSTRUCTDUMMY)] UNSPEC_ST1))]
+(define_insn "aarch64_st1_x2_<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_2QD 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_2QD
+		[(match_operand:VSTRUCT_2QD 1 "register_operand" "w")]
+		UNSPEC_ST1))]
   "TARGET_SIMD"
   "st1\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
   [(set_attr "type" "neon_store1_2reg<q>")]
 )
 
-(define_expand "aarch64_st1x3<VALLDIF:mode>"
+(define_expand "aarch64_st1x3<vstruct_elt>"
   [(match_operand:DI 0 "register_operand")
-   (match_operand:CI 1 "register_operand")
-   (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+   (match_operand:VSTRUCT_3QD 1 "register_operand")]
   "TARGET_SIMD"
 {
-  rtx mem = gen_rtx_MEM (CImode, operands[0]);
-  emit_insn (gen_aarch64_st1_x3_<VALLDIF:mode> (mem, operands[1]));
+  rtx mem = gen_rtx_MEM (<MODE>mode, operands[0]);
+  emit_insn (gen_aarch64_st1_x3_<vstruct_elt> (mem, operands[1]));
   DONE;
 })
 
-(define_insn "aarch64_st1_x3_<mode>"
-   [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:CI
-         [(match_operand:CI 1 "register_operand" "w")
-	  (unspec:VALLDIF [(const_int 3)] UNSPEC_VSTRUCTDUMMY)] UNSPEC_ST1))]
+(define_insn "aarch64_st1_x3_<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_3QD 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_3QD
+		[(match_operand:VSTRUCT_3QD 1 "register_operand" "w")]
+		UNSPEC_ST1))]
   "TARGET_SIMD"
   "st1\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
   [(set_attr "type" "neon_store1_3reg<q>")]
 )
 
-(define_expand "aarch64_st1x4<VALLDIF:mode>"
+(define_expand "aarch64_st1x4<vstruct_elt>"
   [(match_operand:DI 0 "register_operand" "")
-   (match_operand:XI 1 "register_operand" "")
-   (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+   (match_operand:VSTRUCT_4QD 1 "register_operand" "")]
   "TARGET_SIMD"
 {
-  rtx mem = gen_rtx_MEM (XImode, operands[0]);
-  emit_insn (gen_aarch64_st1_x4_<VALLDIF:mode> (mem, operands[1]));
+  rtx mem = gen_rtx_MEM (<MODE>mode, operands[0]);
+  emit_insn (gen_aarch64_st1_x4_<vstruct_elt> (mem, operands[1]));
   DONE;
 })
 
-(define_insn "aarch64_st1_x4_<mode>"
-  [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:XI
-	   [(match_operand:XI 1 "register_operand" "w")
-	   (unspec:VALLDIF [(const_int 4)] UNSPEC_VSTRUCTDUMMY)]
-	UNSPEC_ST1))]
+(define_insn "aarch64_st1_x4_<vstruct_elt>"
+  [(set (match_operand:VSTRUCT_4QD 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_4QD
+		[(match_operand:VSTRUCT_4QD 1 "register_operand" "w")]
+		UNSPEC_ST1))]
   "TARGET_SIMD"
   "st1\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
   [(set_attr "type" "neon_store1_4reg<q>")]
 )
 
 (define_insn "*aarch64_mov<mode>"
+  [(set (match_operand:VSTRUCT_QD 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
+	(match_operand:VSTRUCT_QD 1 "aarch64_simd_general_operand" " w,w,Utv"))]
+  "TARGET_SIMD && !BYTES_BIG_ENDIAN
+   && (register_operand (operands[0], <MODE>mode)
+       || register_operand (operands[1], <MODE>mode))"
+  "@
+   #
+   st1\\t{%S1.<Vtype> - %<Vendreg>1.<Vtype>}, %0
+   ld1\\t{%S0.<Vtype> - %<Vendreg>0.<Vtype>}, %1"
+  [(set_attr "type" "multiple,neon_store<nregs>_<nregs>reg_q,\
+		     neon_load<nregs>_<nregs>reg_q")
+   (set_attr "length" "<insn_count>,4,4")]
+)
+
+(define_insn "*aarch64_mov<mode>"
   [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
 	(match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
   "TARGET_SIMD && !BYTES_BIG_ENDIAN
@@ -7241,6 +7236,34 @@
   [(set_attr "type" "neon_store1_1reg<q>")]
 )
 
+(define_insn "*aarch64_be_mov<mode>"
+  [(set (match_operand:VSTRUCT_2D 0 "nonimmediate_operand" "=w,m,w")
+	(match_operand:VSTRUCT_2D 1 "general_operand"      " w,w,m"))]
+  "TARGET_SIMD && BYTES_BIG_ENDIAN
+   && (register_operand (operands[0], <MODE>mode)
+       || register_operand (operands[1], <MODE>mode))"
+  "@
+   #
+   stp\\t%d1, %R1, %0
+   ldp\\t%d0, %R0, %1"
+  [(set_attr "type" "multiple,neon_stp,neon_ldp")
+   (set_attr "length" "8,4,4")]
+)
+
+(define_insn "*aarch64_be_mov<mode>"
+  [(set (match_operand:VSTRUCT_2Q 0 "nonimmediate_operand" "=w,m,w")
+	(match_operand:VSTRUCT_2Q 1 "general_operand"      " w,w,m"))]
+  "TARGET_SIMD && BYTES_BIG_ENDIAN
+   && (register_operand (operands[0], <MODE>mode)
+       || register_operand (operands[1], <MODE>mode))"
+  "@
+   #
+   stp\\t%q1, %R1, %0
+   ldp\\t%q0, %R0, %1"
+  [(set_attr "type" "multiple,neon_stp_q,neon_ldp_q")
+   (set_attr "length" "8,4,4")]
+)
+
 (define_insn "*aarch64_be_movoi"
   [(set (match_operand:OI 0 "nonimmediate_operand" "=w,m,w")
 	(match_operand:OI 1 "general_operand"      " w,w,m"))]
@@ -7255,6 +7278,17 @@
    (set_attr "length" "8,4,4")]
 )
 
+(define_insn "*aarch64_be_mov<mode>"
+  [(set (match_operand:VSTRUCT_3QD 0 "nonimmediate_operand" "=w,o,w")
+	(match_operand:VSTRUCT_3QD 1 "general_operand"      " w,w,o"))]
+  "TARGET_SIMD && BYTES_BIG_ENDIAN
+   && (register_operand (operands[0], <MODE>mode)
+       || register_operand (operands[1], <MODE>mode))"
+  "#"
+  [(set_attr "type" "multiple")
+   (set_attr "length" "12,8,8")]
+)
+
 (define_insn "*aarch64_be_movci"
   [(set (match_operand:CI 0 "nonimmediate_operand" "=w,o,w")
 	(match_operand:CI 1 "general_operand"      " w,w,o"))]
@@ -7266,6 +7300,17 @@
    (set_attr "length" "12,4,4")]
 )
 
+(define_insn "*aarch64_be_mov<mode>"
+  [(set (match_operand:VSTRUCT_4QD 0 "nonimmediate_operand" "=w,o,w")
+	(match_operand:VSTRUCT_4QD 1 "general_operand"      " w,w,o"))]
+  "TARGET_SIMD && BYTES_BIG_ENDIAN
+   && (register_operand (operands[0], <MODE>mode)
+       || register_operand (operands[1], <MODE>mode))"
+  "#"
+  [(set_attr "type" "multiple")
+   (set_attr "length" "16,8,8")]
+)
+
 (define_insn "*aarch64_be_movxi"
   [(set (match_operand:XI 0 "nonimmediate_operand" "=w,o,w")
 	(match_operand:XI 1 "general_operand"      " w,w,o"))]
@@ -7278,6 +7323,16 @@
 )
 
 (define_split
+  [(set (match_operand:VSTRUCT_2QD 0 "register_operand")
+	(match_operand:VSTRUCT_2QD 1 "register_operand"))]
+  "TARGET_SIMD && reload_completed"
+  [(const_int 0)]
+{
+  aarch64_simd_emit_reg_reg_move (operands, <VSTRUCT_ELT>mode, 2);
+  DONE;
+})
+
+(define_split
   [(set (match_operand:OI 0 "register_operand")
 	(match_operand:OI 1 "register_operand"))]
   "TARGET_SIMD && reload_completed"
@@ -7288,6 +7343,42 @@
 })
 
 (define_split
+  [(set (match_operand:VSTRUCT_3QD 0 "nonimmediate_operand")
+	(match_operand:VSTRUCT_3QD 1 "general_operand"))]
+  "TARGET_SIMD && reload_completed"
+  [(const_int 0)]
+{
+  if (register_operand (operands[0], <MODE>mode)
+      && register_operand (operands[1], <MODE>mode))
+    {
+      aarch64_simd_emit_reg_reg_move (operands, <VSTRUCT_ELT>mode, 3);
+      DONE;
+    }
+  else if (BYTES_BIG_ENDIAN)
+    {
+      int elt_size = GET_MODE_SIZE (<MODE>mode).to_constant () / <nregs>;
+      machine_mode pair_mode = elt_size == 16 ? V2x16QImode : V2x8QImode;
+      emit_move_insn (simplify_gen_subreg (pair_mode, operands[0],
+					   <MODE>mode, 0),
+		      simplify_gen_subreg (pair_mode, operands[1],
+					   <MODE>mode, 0));
+      emit_move_insn (gen_lowpart (<VSTRUCT_ELT>mode,
+				   simplify_gen_subreg (<VSTRUCT_ELT>mode,
+							operands[0],
+							<MODE>mode,
+							2 * elt_size)),
+		      gen_lowpart (<VSTRUCT_ELT>mode,
+				   simplify_gen_subreg (<VSTRUCT_ELT>mode,
+							operands[1],
+							<MODE>mode,
+							2 * elt_size)));
+      DONE;
+    }
+  else
+    FAIL;
+})
+
+(define_split
   [(set (match_operand:CI 0 "nonimmediate_operand")
 	(match_operand:CI 1 "general_operand"))]
   "TARGET_SIMD && reload_completed"
@@ -7316,6 +7407,36 @@
 })
 
 (define_split
+  [(set (match_operand:VSTRUCT_4QD 0 "nonimmediate_operand")
+	(match_operand:VSTRUCT_4QD 1 "general_operand"))]
+  "TARGET_SIMD && reload_completed"
+  [(const_int 0)]
+{
+  if (register_operand (operands[0], <MODE>mode)
+      && register_operand (operands[1], <MODE>mode))
+    {
+      aarch64_simd_emit_reg_reg_move (operands, <VSTRUCT_ELT>mode, 4);
+      DONE;
+    }
+  else if (BYTES_BIG_ENDIAN)
+    {
+      int elt_size = GET_MODE_SIZE (<MODE>mode).to_constant () / <nregs>;
+      machine_mode pair_mode = elt_size == 16 ? V2x16QImode : V2x8QImode;
+      emit_move_insn (simplify_gen_subreg (pair_mode, operands[0],
+					   <MODE>mode, 0),
+		      simplify_gen_subreg (pair_mode, operands[1],
+					   <MODE>mode, 0));
+      emit_move_insn (simplify_gen_subreg (pair_mode, operands[0],
+					   <MODE>mode, 2 * elt_size),
+		      simplify_gen_subreg (pair_mode, operands[1],
+					   <MODE>mode, 2 * elt_size));
+      DONE;
+    }
+  else
+    FAIL;
+})
+
+(define_split
   [(set (match_operand:XI 0 "nonimmediate_operand")
 	(match_operand:XI 1 "general_operand"))]
   "TARGET_SIMD && reload_completed"
@@ -7339,91 +7460,85 @@
     FAIL;
 })
 
-(define_expand "aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>"
-  [(match_operand:VSTRUCT 0 "register_operand")
-   (match_operand:DI 1 "register_operand")
-   (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+(define_expand "aarch64_ld<nregs>r<vstruct_elt>"
+  [(match_operand:VSTRUCT_QD 0 "register_operand")
+   (match_operand:DI 1 "register_operand")]
   "TARGET_SIMD"
 {
   rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
-  set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
-		     * <VSTRUCT:nregs>);
+  set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) * <nregs>);
 
-  emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs>r<VALLDIF:mode> (operands[0],
-								mem));
+  emit_insn (gen_aarch64_simd_ld<nregs>r<vstruct_elt> (operands[0], mem));
   DONE;
 })
 
-(define_insn "aarch64_ld2<mode>_dreg"
-  [(set (match_operand:OI 0 "register_operand" "=w")
-	(unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-		    (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD2_DREG))]
+(define_insn "aarch64_ld2<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_2DNX 0 "register_operand" "=w")
+	(unspec:VSTRUCT_2DNX [
+	  (match_operand:VSTRUCT_2DNX 1 "aarch64_simd_struct_operand" "Utv")]
+	  UNSPEC_LD2_DREG))]
   "TARGET_SIMD"
   "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
   [(set_attr "type" "neon_load2_2reg<q>")]
 )
 
-(define_insn "aarch64_ld2<mode>_dreg"
-  [(set (match_operand:OI 0 "register_operand" "=w")
-	(unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-		    (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD2_DREG))]
+(define_insn "aarch64_ld2<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_2DX 0 "register_operand" "=w")
+	(unspec:VSTRUCT_2DX [
+	  (match_operand:VSTRUCT_2DX 1 "aarch64_simd_struct_operand" "Utv")]
+	  UNSPEC_LD2_DREG))]
   "TARGET_SIMD"
   "ld1\\t{%S0.1d - %T0.1d}, %1"
   [(set_attr "type" "neon_load1_2reg<q>")]
 )
 
-(define_insn "aarch64_ld3<mode>_dreg"
-  [(set (match_operand:CI 0 "register_operand" "=w")
-	(unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-		    (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD3_DREG))]
+(define_insn "aarch64_ld3<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_3DNX 0 "register_operand" "=w")
+	(unspec:VSTRUCT_3DNX [
+	  (match_operand:VSTRUCT_3DNX 1 "aarch64_simd_struct_operand" "Utv")]
+	  UNSPEC_LD3_DREG))]
   "TARGET_SIMD"
   "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
   [(set_attr "type" "neon_load3_3reg<q>")]
 )
 
-(define_insn "aarch64_ld3<mode>_dreg"
-  [(set (match_operand:CI 0 "register_operand" "=w")
-	(unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-		    (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD3_DREG))]
+(define_insn "aarch64_ld3<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_3DX 0 "register_operand" "=w")
+	(unspec:VSTRUCT_3DX [
+	  (match_operand:VSTRUCT_3DX 1 "aarch64_simd_struct_operand" "Utv")]
+	  UNSPEC_LD3_DREG))]
   "TARGET_SIMD"
   "ld1\\t{%S0.1d - %U0.1d}, %1"
   [(set_attr "type" "neon_load1_3reg<q>")]
 )
 
-(define_insn "aarch64_ld4<mode>_dreg"
-  [(set (match_operand:XI 0 "register_operand" "=w")
-	(unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-		    (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD4_DREG))]
+(define_insn "aarch64_ld4<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_4DNX 0 "register_operand" "=w")
+	(unspec:VSTRUCT_4DNX [
+	  (match_operand:VSTRUCT_4DNX 1 "aarch64_simd_struct_operand" "Utv")]
+	  UNSPEC_LD4_DREG))]
   "TARGET_SIMD"
   "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
   [(set_attr "type" "neon_load4_4reg<q>")]
 )
 
-(define_insn "aarch64_ld4<mode>_dreg"
-  [(set (match_operand:XI 0 "register_operand" "=w")
-	(unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
-		    (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD4_DREG))]
+(define_insn "aarch64_ld4<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_4DX 0 "register_operand" "=w")
+	(unspec:VSTRUCT_4DX [
+	  (match_operand:VSTRUCT_4DX 1 "aarch64_simd_struct_operand" "Utv")]
+	  UNSPEC_LD4_DREG))]
   "TARGET_SIMD"
   "ld1\\t{%S0.1d - %V0.1d}, %1"
   [(set_attr "type" "neon_load1_4reg<q>")]
 )
 
-(define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
- [(match_operand:VSTRUCT 0 "register_operand")
-  (match_operand:DI 1 "register_operand")
-  (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+(define_expand "aarch64_ld<nregs><vstruct_elt>"
+ [(match_operand:VSTRUCT_D 0 "register_operand")
+  (match_operand:DI 1 "register_operand")]
   "TARGET_SIMD"
 {
-  rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
-  set_mem_size (mem, <VSTRUCT:nregs> * 8);
-
-  emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
+  rtx mem = gen_rtx_MEM (<MODE>mode, operands[1]);
+  emit_insn (gen_aarch64_ld<nregs><vstruct_elt>_dreg (operands[0], mem));
   DONE;
 })
 
@@ -7442,97 +7557,42 @@
   DONE;
 })
 
-(define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
- [(match_operand:VSTRUCT 0 "register_operand")
-  (match_operand:DI 1 "register_operand")
-  (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-  "TARGET_SIMD"
-{
-  machine_mode mode = <VSTRUCT:MODE>mode;
-  rtx mem = gen_rtx_MEM (mode, operands[1]);
-
-  emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs><VQ:mode> (operands[0], mem));
-  DONE;
-})
-
-(define_expand "aarch64_ld1x2<VQ:mode>"
- [(match_operand:OI 0 "register_operand")
-  (match_operand:DI 1 "register_operand")
-  (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+(define_expand "aarch64_ld<nregs><vstruct_elt>"
+ [(match_operand:VSTRUCT_Q 0 "register_operand")
+  (match_operand:DI 1 "register_operand")]
   "TARGET_SIMD"
 {
-  machine_mode mode = OImode;
-  rtx mem = gen_rtx_MEM (mode, operands[1]);
-
-  emit_insn (gen_aarch64_simd_ld1<VQ:mode>_x2 (operands[0], mem));
+  rtx mem = gen_rtx_MEM (<MODE>mode, operands[1]);
+  emit_insn (gen_aarch64_simd_ld<nregs><vstruct_elt> (operands[0], mem));
   DONE;
 })
 
-(define_expand "aarch64_ld1x2<VDC:mode>"
- [(match_operand:OI 0 "register_operand")
-  (match_operand:DI 1 "register_operand")
-  (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+(define_expand "aarch64_ld1x2<vstruct_elt>"
+ [(match_operand:VSTRUCT_2QD 0 "register_operand")
+  (match_operand:DI 1 "register_operand")]
   "TARGET_SIMD"
 {
-  machine_mode mode = OImode;
+  machine_mode mode = <MODE>mode;
   rtx mem = gen_rtx_MEM (mode, operands[1]);
 
-  emit_insn (gen_aarch64_simd_ld1<VDC:mode>_x2 (operands[0], mem));
+  emit_insn (gen_aarch64_simd_ld1<vstruct_elt>_x2 (operands[0], mem));
   DONE;
 })
 
-
-(define_expand "aarch64_ld<VSTRUCT:nregs>_lane<VALLDIF:mode>"
-  [(match_operand:VSTRUCT 0 "register_operand")
+(define_expand "aarch64_ld<nregs>_lane<vstruct_elt>"
+  [(match_operand:VSTRUCT_QD 0 "register_operand")
 	(match_operand:DI 1 "register_operand")
-	(match_operand:VSTRUCT 2 "register_operand")
-	(match_operand:SI 3 "immediate_operand")
-	(unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+	(match_operand:VSTRUCT_QD 2 "register_operand")
+	(match_operand:SI 3 "immediate_operand")]
   "TARGET_SIMD"
 {
   rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
-  set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
-		     * <VSTRUCT:nregs>);
+  set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) * <nregs>);
 
-  aarch64_simd_lane_bounds (operands[3], 0, <VALLDIF:nunits>, NULL);
-  emit_insn (gen_aarch64_vec_load_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
-	operands[0], mem, operands[2], operands[3]));
-  DONE;
-})
-
-;; Expanders for builtins to extract vector registers from large
-;; opaque integer modes.
-
-;; D-register list.
-
-(define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
- [(match_operand:VDC 0 "register_operand")
-  (match_operand:VSTRUCT 1 "register_operand")
-  (match_operand:SI 2 "immediate_operand")]
-  "TARGET_SIMD"
-{
-  int part = INTVAL (operands[2]);
-  rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
-  int offset = part * 16;
-
-  emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
-  emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
-  DONE;
-})
-
-;; Q-register list.
-
-(define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
- [(match_operand:VQ 0 "register_operand")
-  (match_operand:VSTRUCT 1 "register_operand")
-  (match_operand:SI 2 "immediate_operand")]
-  "TARGET_SIMD"
-{
-  int part = INTVAL (operands[2]);
-  int offset = part * 16;
-
-  emit_move_insn (operands[0],
-		  gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
+  aarch64_simd_lane_bounds (operands[3], 0,
+		GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>, NULL);
+  emit_insn (gen_aarch64_vec_load_lanes<mode>_lane<vstruct_elt> (operands[0],
+				mem, operands[2], operands[3]));
   DONE;
 })
 
@@ -7579,7 +7639,7 @@
 
 (define_insn "aarch64_qtbl2<mode>"
   [(set (match_operand:VB 0 "register_operand" "=w")
-	(unspec:VB [(match_operand:OI 1 "register_operand" "w")
+	(unspec:VB [(match_operand:V2x16QI 1 "register_operand" "w")
 		      (match_operand:VB 2 "register_operand" "w")]
 		      UNSPEC_TBL))]
   "TARGET_SIMD"
@@ -7590,7 +7650,7 @@
 (define_insn "aarch64_qtbx2<mode>"
   [(set (match_operand:VB 0 "register_operand" "=w")
 	(unspec:VB [(match_operand:VB 1 "register_operand" "0")
-		      (match_operand:OI 2 "register_operand" "w")
+		      (match_operand:V2x16QI 2 "register_operand" "w")
 		      (match_operand:VB 3 "register_operand" "w")]
 		      UNSPEC_TBX))]
   "TARGET_SIMD"
@@ -7602,7 +7662,7 @@
 
 (define_insn "aarch64_qtbl3<mode>"
   [(set (match_operand:VB 0 "register_operand" "=w")
-	(unspec:VB [(match_operand:CI 1 "register_operand" "w")
+	(unspec:VB [(match_operand:V3x16QI 1 "register_operand" "w")
 		      (match_operand:VB 2 "register_operand" "w")]
 		      UNSPEC_TBL))]
   "TARGET_SIMD"
@@ -7613,7 +7673,7 @@
 (define_insn "aarch64_qtbx3<mode>"
   [(set (match_operand:VB 0 "register_operand" "=w")
 	(unspec:VB [(match_operand:VB 1 "register_operand" "0")
-		      (match_operand:CI 2 "register_operand" "w")
+		      (match_operand:V3x16QI 2 "register_operand" "w")
 		      (match_operand:VB 3 "register_operand" "w")]
 		      UNSPEC_TBX))]
   "TARGET_SIMD"
@@ -7625,7 +7685,7 @@
 
 (define_insn "aarch64_qtbl4<mode>"
   [(set (match_operand:VB 0 "register_operand" "=w")
-	(unspec:VB [(match_operand:XI 1 "register_operand" "w")
+	(unspec:VB [(match_operand:V4x16QI 1 "register_operand" "w")
 		      (match_operand:VB 2 "register_operand" "w")]
 		      UNSPEC_TBL))]
   "TARGET_SIMD"
@@ -7636,7 +7696,7 @@
 (define_insn "aarch64_qtbx4<mode>"
   [(set (match_operand:VB 0 "register_operand" "=w")
 	(unspec:VB [(match_operand:VB 1 "register_operand" "0")
-		      (match_operand:XI 2 "register_operand" "w")
+		      (match_operand:V4x16QI 2 "register_operand" "w")
 		      (match_operand:VB 3 "register_operand" "w")]
 		      UNSPEC_TBX))]
   "TARGET_SIMD"
@@ -7645,10 +7705,10 @@
 )
 
 (define_insn_and_split "aarch64_combinev16qi"
-  [(set (match_operand:OI 0 "register_operand" "=w")
-	(unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
-		    (match_operand:V16QI 2 "register_operand" "w")]
-		   UNSPEC_CONCAT))]
+  [(set (match_operand:V2x16QI 0 "register_operand" "=w")
+	(unspec:V2x16QI [(match_operand:V16QI 1 "register_operand" "w")
+			 (match_operand:V16QI 2 "register_operand" "w")]
+			UNSPEC_CONCAT))]
   "TARGET_SIMD"
   "#"
   "&& reload_completed"
@@ -7704,105 +7764,99 @@
   [(set_attr "type" "neon_rev<q>")]
 )
 
-(define_insn "aarch64_st2<mode>_dreg"
-  [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:BLK [(match_operand:OI 1 "register_operand" "w")
-                    (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-                   UNSPEC_ST2))]
+(define_insn "aarch64_st2<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_2DNX 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_2DNX [
+		(match_operand:VSTRUCT_2DNX 1 "register_operand" "w")]
+		UNSPEC_ST2))]
   "TARGET_SIMD"
   "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
   [(set_attr "type" "neon_store2_2reg")]
 )
 
-(define_insn "aarch64_st2<mode>_dreg"
-  [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:BLK [(match_operand:OI 1 "register_operand" "w")
-                    (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-                   UNSPEC_ST2))]
+(define_insn "aarch64_st2<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_2DX 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_2DX [
+		(match_operand:VSTRUCT_2DX 1 "register_operand" "w")]
+		UNSPEC_ST2))]
   "TARGET_SIMD"
   "st1\\t{%S1.1d - %T1.1d}, %0"
   [(set_attr "type" "neon_store1_2reg")]
 )
 
-(define_insn "aarch64_st3<mode>_dreg"
-  [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:BLK [(match_operand:CI 1 "register_operand" "w")
-                    (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-                   UNSPEC_ST3))]
+(define_insn "aarch64_st3<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_3DNX 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_3DNX [
+		(match_operand:VSTRUCT_3DNX 1 "register_operand" "w")]
+		UNSPEC_ST3))]
   "TARGET_SIMD"
   "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
   [(set_attr "type" "neon_store3_3reg")]
 )
 
-(define_insn "aarch64_st3<mode>_dreg"
-  [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:BLK [(match_operand:CI 1 "register_operand" "w")
-                    (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-                   UNSPEC_ST3))]
+(define_insn "aarch64_st3<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_3DX 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_3DX [
+		(match_operand:VSTRUCT_3DX 1 "register_operand" "w")]
+		UNSPEC_ST3))]
   "TARGET_SIMD"
   "st1\\t{%S1.1d - %U1.1d}, %0"
   [(set_attr "type" "neon_store1_3reg")]
 )
 
-(define_insn "aarch64_st4<mode>_dreg"
-  [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:BLK [(match_operand:XI 1 "register_operand" "w")
-                    (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-                   UNSPEC_ST4))]
+(define_insn "aarch64_st4<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_4DNX 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_4DNX [
+		(match_operand:VSTRUCT_4DNX 1 "register_operand" "w")]
+		UNSPEC_ST4))]
   "TARGET_SIMD"
   "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
   [(set_attr "type" "neon_store4_4reg")]
 )
 
-(define_insn "aarch64_st4<mode>_dreg"
-  [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
-	(unspec:BLK [(match_operand:XI 1 "register_operand" "w")
-                    (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-                   UNSPEC_ST4))]
+(define_insn "aarch64_st4<vstruct_elt>_dreg"
+  [(set (match_operand:VSTRUCT_4DX 0 "aarch64_simd_struct_operand" "=Utv")
+	(unspec:VSTRUCT_4DX [
+		(match_operand:VSTRUCT_4DX 1 "register_operand" "w")]
+		UNSPEC_ST4))]
   "TARGET_SIMD"
   "st1\\t{%S1.1d - %V1.1d}, %0"
   [(set_attr "type" "neon_store1_4reg")]
 )
 
-(define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
+(define_expand "aarch64_st<nregs><vstruct_elt>"
  [(match_operand:DI 0 "register_operand")
-  (match_operand:VSTRUCT 1 "register_operand")
-  (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+  (match_operand:VSTRUCT_D 1 "register_operand")]
   "TARGET_SIMD"
 {
-  rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
-  set_mem_size (mem, <VSTRUCT:nregs> * 8);
-
-  emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
+  rtx mem = gen_rtx_MEM (<MODE>mode, operands[0]);
+  emit_insn (gen_aarch64_st<nregs><vstruct_elt>_dreg (mem, operands[1]));
   DONE;
 })
 
-(define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
+(define_expand "aarch64_st<nregs><vstruct_elt>"
  [(match_operand:DI 0 "register_operand")
-  (match_operand:VSTRUCT 1 "register_operand")
-  (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
+  (match_operand:VSTRUCT_Q 1 "register_operand")]
   "TARGET_SIMD"
 {
-  machine_mode mode = <VSTRUCT:MODE>mode;
-  rtx mem = gen_rtx_MEM (mode, operands[0]);
-
-  emit_insn (gen_aarch64_simd_st<VSTRUCT:nregs><VQ:mode> (mem, operands[1]));
+  rtx mem = gen_rtx_MEM (<MODE>mode, operands[0]);
+  emit_insn (gen_aarch64_simd_st<nregs><vstruct_elt> (mem, operands[1]));
   DONE;
 })
 
-(define_expand "aarch64_st<VSTRUCT:nregs>_lane<VALLDIF:mode>"
+(define_expand "aarch64_st<nregs>_lane<vstruct_elt>"
  [(match_operand:DI 0 "register_operand")
-  (match_operand:VSTRUCT 1 "register_operand")
-  (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
+  (match_operand:VSTRUCT_QD 1 "register_operand")
   (match_operand:SI 2 "immediate_operand")]
   "TARGET_SIMD"
 {
   rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
-  set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
-		     * <VSTRUCT:nregs>);
+  set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) * <nregs>);
 
-  emit_insn (gen_aarch64_vec_store_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
-		mem, operands[1], operands[2]));
+  aarch64_simd_lane_bounds (operands[2], 0,
+		GET_MODE_NUNITS (<MODE>mode).to_constant () / <nregs>, NULL);
+  emit_insn (gen_aarch64_vec_store_lanes<mode>_lane<vstruct_elt> (mem,
+					operands[1], operands[2]));
   DONE;
 })
 
@@ -7821,28 +7875,6 @@
   DONE;
 })
 
-;; Expander for builtins to insert vector registers into large
-;; opaque integer modes.
-
-;; Q-register list.  We don't need a D-reg inserter as we zero
-;; extend them in arm_neon.h and insert the resulting Q-regs.
-
-(define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
- [(match_operand:VSTRUCT 0 "register_operand")
-  (match_operand:VSTRUCT 1 "register_operand")
-  (match_operand:VQ 2 "register_operand")
-  (match_operand:SI 3 "immediate_operand")]
-  "TARGET_SIMD"
-{
-  int part = INTVAL (operands[3]);
-  int offset = part * 16;
-
-  emit_move_insn (operands[0], operands[1]);
-  emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
-		  operands[2]);
-  DONE;
-})
-
 ;; Standard pattern name vec_init<mode><Vel>.
 
 (define_expand "vec_init<mode><Vel>"
@@ -7872,21 +7904,11 @@
   [(set_attr "type" "neon_load1_all_lanes")]
 )
 
-(define_insn "aarch64_simd_ld1<mode>_x2"
-  [(set (match_operand:OI 0 "register_operand" "=w")
-	(unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
-		    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD1))]
-  "TARGET_SIMD"
-  "ld1\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
-  [(set_attr "type" "neon_load1_2reg<q>")]
-)
-
-(define_insn "aarch64_simd_ld1<mode>_x2"
-  [(set (match_operand:OI 0 "register_operand" "=w")
-	(unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
-		    (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
-		   UNSPEC_LD1))]
+(define_insn "aarch64_simd_ld1<vstruct_elt>_x2"
+  [(set (match_operand:VSTRUCT_2QD 0 "register_operand" "=w")
+	(unspec:VSTRUCT_2QD [
+	    (match_operand:VSTRUCT_2QD 1 "aarch64_simd_struct_operand" "Utv")]
+	    UNSPEC_LD1))]
   "TARGET_SIMD"
   "ld1\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
   [(set_attr "type" "neon_load1_2reg<q>")]
diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md
index 8fe4c72..5de479e 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -6288,7 +6288,7 @@
 
 ;; Unpredicated fmax/fmin (the libm functions).  The optabs for the
 ;; smin/smax rtx codes are handled in the generic section above.
-(define_expand "<maxmin_uns><mode>3"
+(define_expand "<fmaxmin><mode>3"
   [(set (match_operand:SVE_FULL_F 0 "register_operand")
 	(unspec:SVE_FULL_F
 	  [(match_dup 3)
diff --git a/gcc/config/aarch64/aarch64-tune.md b/gcc/config/aarch64/aarch64-tune.md
index 12be913..3eed700 100644
--- a/gcc/config/aarch64/aarch64-tune.md
+++ b/gcc/config/aarch64/aarch64-tune.md
@@ -1,5 +1,5 @@
 ;; -*- buffer-read-only: t -*-
 ;; Generated automatically by gentune.sh from aarch64-cores.def
 (define_attr "tune"
-	"cortexa34,cortexa35,cortexa53,cortexa57,cortexa72,cortexa73,thunderx,thunderxt88p1,thunderxt88,octeontx,octeontxt81,octeontxt83,thunderxt81,thunderxt83,emag,xgene1,falkor,qdf24xx,exynosm1,phecda,thunderx2t99p1,vulcan,thunderx2t99,cortexa55,cortexa75,cortexa76,cortexa76ae,cortexa77,cortexa78,cortexa78ae,cortexa78c,cortexa65,cortexa65ae,cortexx1,ares,neoversen1,neoversee1,octeontx2,octeontx2t98,octeontx2t96,octeontx2t93,octeontx2f95,octeontx2f95n,octeontx2f95mm,a64fx,tsv110,thunderx3t110,zeus,neoversev1,neoverse512tvb,saphira,neoversen2,cortexa57cortexa53,cortexa72cortexa53,cortexa73cortexa35,cortexa73cortexa53,cortexa75cortexa55,cortexa76cortexa55,cortexr82,cortexa510,cortexa710,cortexx2"
+	"cortexa34,cortexa35,cortexa53,cortexa57,cortexa72,cortexa73,thunderx,thunderxt88p1,thunderxt88,octeontx,octeontxt81,octeontxt83,thunderxt81,thunderxt83,ampere1,emag,xgene1,falkor,qdf24xx,exynosm1,phecda,thunderx2t99p1,vulcan,thunderx2t99,cortexa55,cortexa75,cortexa76,cortexa76ae,cortexa77,cortexa78,cortexa78ae,cortexa78c,cortexa65,cortexa65ae,cortexx1,ares,neoversen1,neoversee1,octeontx2,octeontx2t98,octeontx2t96,octeontx2t93,octeontx2f95,octeontx2f95n,octeontx2f95mm,a64fx,tsv110,thunderx3t110,zeus,neoversev1,neoverse512tvb,saphira,neoversen2,cortexa57cortexa53,cortexa72cortexa53,cortexa73cortexa35,cortexa73cortexa53,cortexa75cortexa55,cortexa76cortexa55,cortexr82,cortexa510,cortexa710,cortexx2"
 	(const (symbol_ref "((enum attr_tune) aarch64_tune)")))
diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c
index 699c105..d8410fc 100644
--- a/gcc/config/aarch64/aarch64.c
+++ b/gcc/config/aarch64/aarch64.c
@@ -79,6 +79,7 @@
 #include "tree-ssa-loop-niter.h"
 #include "fractional-cost.h"
 #include "rtlanal.h"
+#include "tree-dfa.h"
 
 /* This file should be included last.  */
 #include "target-def.h"
@@ -1067,6 +1068,43 @@ static const struct cpu_vector_cost thunderx3t110_vector_cost =
   nullptr /* issue_info  */
 };
 
+static const advsimd_vec_cost ampere1_advsimd_vector_cost =
+{
+  3, /* int_stmt_cost  */
+  3, /* fp_stmt_cost  */
+  0, /* ld2_st2_permute_cost  */
+  0, /* ld3_st3_permute_cost  */
+  0, /* ld4_st4_permute_cost  */
+  2, /* permute_cost  */
+  12, /* reduc_i8_cost  */
+  9, /* reduc_i16_cost  */
+  6, /* reduc_i32_cost  */
+  5, /* reduc_i64_cost  */
+  9, /* reduc_f16_cost  */
+  6, /* reduc_f32_cost  */
+  5, /* reduc_f64_cost  */
+  8, /* store_elt_extra_cost  */
+  6, /* vec_to_scalar_cost  */
+  7, /* scalar_to_vec_cost  */
+  5, /* align_load_cost  */
+  5, /* unalign_load_cost  */
+  2, /* unalign_store_cost  */
+  2  /* store_cost  */
+};
+
+/* Ampere-1 costs for vector insn classes.  */
+static const struct cpu_vector_cost ampere1_vector_cost =
+{
+  1, /* scalar_int_stmt_cost  */
+  1, /* scalar_fp_stmt_cost  */
+  4, /* scalar_load_cost  */
+  1, /* scalar_store_cost  */
+  1, /* cond_taken_branch_cost  */
+  1, /* cond_not_taken_branch_cost  */
+  &ampere1_advsimd_vector_cost, /* advsimd  */
+  nullptr, /* sve  */
+  nullptr  /* issue_info  */
+};
 
 /* Generic costs for branch instructions.  */
 static const struct cpu_branch_cost generic_branch_cost =
@@ -1210,6 +1248,17 @@ static const cpu_prefetch_tune a64fx_prefetch_tune =
   -1			/* default_opt_level  */
 };
 
+static const cpu_prefetch_tune ampere1_prefetch_tune =
+{
+  0,			/* num_slots  */
+  64,			/* l1_cache_size  */
+  64,			/* l1_cache_line_size  */
+  2048,			/* l2_cache_size  */
+  true,			/* prefetch_dynamic_strides */
+  -1,			/* minimum_stride */
+  -1			/* default_opt_level  */
+};
+
 static const struct tune_params generic_tunings =
 {
   &cortexa57_extra_costs,
@@ -1670,6 +1719,36 @@ static const struct tune_params neoversen1_tunings =
   &generic_prefetch_tune
 };
 
+static const struct tune_params ampere1_tunings =
+{
+  &ampere1_extra_costs,
+  &generic_addrcost_table,
+  &generic_regmove_cost,
+  &ampere1_vector_cost,
+  &generic_branch_cost,
+  &generic_approx_modes,
+  SVE_NOT_IMPLEMENTED, /* sve_width  */
+  4, /* memmov_cost  */
+  4, /* issue_rate  */
+  (AARCH64_FUSE_ADRP_ADD | AARCH64_FUSE_AES_AESMC |
+   AARCH64_FUSE_MOV_MOVK | AARCH64_FUSE_MOVK_MOVK |
+   AARCH64_FUSE_ALU_BRANCH /* adds, ands, bics, ccmp, ccmn */ |
+   AARCH64_FUSE_CMP_BRANCH),
+  /* fusible_ops  */
+  "32",		/* function_align.  */
+  "4",		/* jump_align.  */
+  "32:16",	/* loop_align.  */
+  2,	/* int_reassoc_width.  */
+  4,	/* fp_reassoc_width.  */
+  2,	/* vec_reassoc_width.  */
+  2,	/* min_div_recip_mul_sf.  */
+  2,	/* min_div_recip_mul_df.  */
+  0,	/* max_case_values.  */
+  tune_params::AUTOPREFETCHER_WEAK,	/* autoprefetcher_model.  */
+  (AARCH64_EXTRA_TUNE_NONE),		/* tune_flags.  */
+  &ampere1_prefetch_tune
+};
+
 static const advsimd_vec_cost neoversev1_advsimd_vector_cost =
 {
   2, /* int_stmt_cost  */
@@ -2785,14 +2864,6 @@ aarch64_estimated_sve_vq ()
   return estimated_poly_value (BITS_PER_SVE_VECTOR) / 128;
 }
 
-/* Return true if MODE is any of the Advanced SIMD structure modes.  */
-static bool
-aarch64_advsimd_struct_mode_p (machine_mode mode)
-{
-  return (TARGET_SIMD
-	  && (mode == OImode || mode == CImode || mode == XImode));
-}
-
 /* Return true if MODE is an SVE predicate mode.  */
 static bool
 aarch64_sve_pred_mode_p (machine_mode mode)
@@ -2823,9 +2894,6 @@ const unsigned int VEC_ANY_DATA = VEC_ADVSIMD | VEC_SVE_DATA;
 static unsigned int
 aarch64_classify_vector_mode (machine_mode mode)
 {
-  if (aarch64_advsimd_struct_mode_p (mode))
-    return VEC_ADVSIMD | VEC_STRUCT;
-
   if (aarch64_sve_pred_mode_p (mode))
     return VEC_SVE_PRED;
 
@@ -2892,6 +2960,65 @@ aarch64_classify_vector_mode (machine_mode mode)
     case E_VNx8DFmode:
       return TARGET_SVE ? VEC_SVE_DATA | VEC_STRUCT : 0;
 
+    case E_OImode:
+    case E_CImode:
+    case E_XImode:
+      return TARGET_SIMD ? VEC_ADVSIMD | VEC_STRUCT : 0;
+
+    /* Structures of 64-bit Advanced SIMD vectors.  */
+    case E_V2x8QImode:
+    case E_V2x4HImode:
+    case E_V2x2SImode:
+    case E_V2x1DImode:
+    case E_V2x4BFmode:
+    case E_V2x4HFmode:
+    case E_V2x2SFmode:
+    case E_V2x1DFmode:
+    case E_V3x8QImode:
+    case E_V3x4HImode:
+    case E_V3x2SImode:
+    case E_V3x1DImode:
+    case E_V3x4BFmode:
+    case E_V3x4HFmode:
+    case E_V3x2SFmode:
+    case E_V3x1DFmode:
+    case E_V4x8QImode:
+    case E_V4x4HImode:
+    case E_V4x2SImode:
+    case E_V4x1DImode:
+    case E_V4x4BFmode:
+    case E_V4x4HFmode:
+    case E_V4x2SFmode:
+    case E_V4x1DFmode:
+      return TARGET_SIMD ? VEC_ADVSIMD | VEC_STRUCT | VEC_PARTIAL : 0;
+
+    /* Structures of 128-bit Advanced SIMD vectors.  */
+    case E_V2x16QImode:
+    case E_V2x8HImode:
+    case E_V2x4SImode:
+    case E_V2x2DImode:
+    case E_V2x8BFmode:
+    case E_V2x8HFmode:
+    case E_V2x4SFmode:
+    case E_V2x2DFmode:
+    case E_V3x16QImode:
+    case E_V3x8HImode:
+    case E_V3x4SImode:
+    case E_V3x2DImode:
+    case E_V3x8BFmode:
+    case E_V3x8HFmode:
+    case E_V3x4SFmode:
+    case E_V3x2DFmode:
+    case E_V4x16QImode:
+    case E_V4x8HImode:
+    case E_V4x4SImode:
+    case E_V4x2DImode:
+    case E_V4x8BFmode:
+    case E_V4x8HFmode:
+    case E_V4x4SFmode:
+    case E_V4x2DFmode:
+      return TARGET_SIMD ? VEC_ADVSIMD | VEC_STRUCT : 0;
+
     /* 64-bit Advanced SIMD vectors.  */
     case E_V8QImode:
     case E_V4HImode:
@@ -2917,6 +3044,29 @@ aarch64_classify_vector_mode (machine_mode mode)
     }
 }
 
+/* Return true if MODE is any of the Advanced SIMD structure modes.  */
+bool
+aarch64_advsimd_struct_mode_p (machine_mode mode)
+{
+  unsigned int vec_flags = aarch64_classify_vector_mode (mode);
+  return (vec_flags & VEC_ADVSIMD) && (vec_flags & VEC_STRUCT);
+}
+
+/* Return true if MODE is an Advanced SIMD D-register structure mode.  */
+static bool
+aarch64_advsimd_partial_struct_mode_p (machine_mode mode)
+{
+  return (aarch64_classify_vector_mode (mode)
+	  == (VEC_ADVSIMD | VEC_STRUCT | VEC_PARTIAL));
+}
+
+/* Return true if MODE is an Advanced SIMD Q-register structure mode.  */
+static bool
+aarch64_advsimd_full_struct_mode_p (machine_mode mode)
+{
+  return (aarch64_classify_vector_mode (mode) == (VEC_ADVSIMD | VEC_STRUCT));
+}
+
 /* Return true if MODE is any of the data vector modes, including
    structure modes.  */
 static bool
@@ -2959,14 +3109,70 @@ aarch64_vl_bytes (machine_mode mode, unsigned int vec_flags)
   return BYTES_PER_SVE_PRED;
 }
 
+/* If MODE holds an array of vectors, return the number of vectors
+   in the array, otherwise return 1.  */
+
+static unsigned int
+aarch64_ldn_stn_vectors (machine_mode mode)
+{
+  unsigned int vec_flags = aarch64_classify_vector_mode (mode);
+  if (vec_flags == (VEC_ADVSIMD | VEC_PARTIAL | VEC_STRUCT))
+    return exact_div (GET_MODE_SIZE (mode), 8).to_constant ();
+  if (vec_flags == (VEC_ADVSIMD | VEC_STRUCT))
+    return exact_div (GET_MODE_SIZE (mode), 16).to_constant ();
+  if (vec_flags == (VEC_SVE_DATA | VEC_STRUCT))
+    return exact_div (GET_MODE_SIZE (mode),
+		      BYTES_PER_SVE_VECTOR).to_constant ();
+  return 1;
+}
+
+/* Given an Advanced SIMD vector mode MODE and a tuple size NELEMS, return the
+   corresponding vector structure mode.  */
+static opt_machine_mode
+aarch64_advsimd_vector_array_mode (machine_mode mode,
+				   unsigned HOST_WIDE_INT nelems)
+{
+  unsigned int flags = VEC_ADVSIMD | VEC_STRUCT;
+  if (known_eq (GET_MODE_SIZE (mode), 8))
+    flags |= VEC_PARTIAL;
+
+  machine_mode struct_mode;
+  FOR_EACH_MODE_IN_CLASS (struct_mode, GET_MODE_CLASS (mode))
+    if (aarch64_classify_vector_mode (struct_mode) == flags
+	&& GET_MODE_INNER (struct_mode) == GET_MODE_INNER (mode)
+	&& known_eq (GET_MODE_NUNITS (struct_mode),
+	     GET_MODE_NUNITS (mode) * nelems))
+      return struct_mode;
+  return opt_machine_mode ();
+}
+
+/* Return the SVE vector mode that has NUNITS elements of mode INNER_MODE.  */
+
+opt_machine_mode
+aarch64_sve_data_mode (scalar_mode inner_mode, poly_uint64 nunits)
+{
+  enum mode_class mclass = (is_a <scalar_float_mode> (inner_mode)
+			    ? MODE_VECTOR_FLOAT : MODE_VECTOR_INT);
+  machine_mode mode;
+  FOR_EACH_MODE_IN_CLASS (mode, mclass)
+    if (inner_mode == GET_MODE_INNER (mode)
+	&& known_eq (nunits, GET_MODE_NUNITS (mode))
+	&& aarch64_sve_data_mode_p (mode))
+      return mode;
+  return opt_machine_mode ();
+}
+
 /* Implement target hook TARGET_ARRAY_MODE.  */
 static opt_machine_mode
 aarch64_array_mode (machine_mode mode, unsigned HOST_WIDE_INT nelems)
 {
   if (aarch64_classify_vector_mode (mode) == VEC_SVE_DATA
       && IN_RANGE (nelems, 2, 4))
-    return mode_for_vector (GET_MODE_INNER (mode),
-			    GET_MODE_NUNITS (mode) * nelems);
+    return aarch64_sve_data_mode (GET_MODE_INNER (mode),
+				  GET_MODE_NUNITS (mode) * nelems);
+  if (aarch64_classify_vector_mode (mode) == VEC_ADVSIMD
+      && IN_RANGE (nelems, 2, 4))
+    return aarch64_advsimd_vector_array_mode (mode, nelems);
 
   return opt_machine_mode ();
 }
@@ -3043,22 +3249,6 @@ aarch64_get_mask_mode (machine_mode mode)
   return default_get_mask_mode (mode);
 }
 
-/* Return the SVE vector mode that has NUNITS elements of mode INNER_MODE.  */
-
-opt_machine_mode
-aarch64_sve_data_mode (scalar_mode inner_mode, poly_uint64 nunits)
-{
-  enum mode_class mclass = (is_a <scalar_float_mode> (inner_mode)
-			    ? MODE_VECTOR_FLOAT : MODE_VECTOR_INT);
-  machine_mode mode;
-  FOR_EACH_MODE_IN_CLASS (mode, mclass)
-    if (inner_mode == GET_MODE_INNER (mode)
-	&& known_eq (nunits, GET_MODE_NUNITS (mode))
-	&& aarch64_sve_data_mode_p (mode))
-      return mode;
-  return opt_machine_mode ();
-}
-
 /* Return the integer element mode associated with SVE mode MODE.  */
 
 static scalar_int_mode
@@ -3183,6 +3373,8 @@ aarch64_hard_regno_nregs (unsigned regno, machine_mode mode)
 	if (vec_flags & VEC_SVE_DATA)
 	  return exact_div (GET_MODE_SIZE (mode),
 			    aarch64_vl_bytes (mode, vec_flags)).to_constant ();
+	if (vec_flags == (VEC_ADVSIMD | VEC_STRUCT | VEC_PARTIAL))
+	  return GET_MODE_SIZE (mode).to_constant () / 8;
 	return CEIL (lowest_size, UNITS_PER_VREG);
       }
     case PR_REGS:
@@ -3754,47 +3946,8 @@ aarch64_load_symref_appropriately (rtx dest, rtx imm,
       }
 
     case SYMBOL_SMALL_GOT_4G:
-      {
-	/* In ILP32, the mode of dest can be either SImode or DImode,
-	   while the got entry is always of SImode size.  The mode of
-	   dest depends on how dest is used: if dest is assigned to a
-	   pointer (e.g. in the memory), it has SImode; it may have
-	   DImode if dest is dereferenced to access the memeory.
-	   This is why we have to handle three different ldr_got_small
-	   patterns here (two patterns for ILP32).  */
-
-	rtx insn;
-	rtx mem;
-	rtx tmp_reg = dest;
-	machine_mode mode = GET_MODE (dest);
-
-	if (can_create_pseudo_p ())
-	  tmp_reg = gen_reg_rtx (mode);
-
-	emit_move_insn (tmp_reg, gen_rtx_HIGH (mode, imm));
-	if (mode == ptr_mode)
-	  {
-	    if (mode == DImode)
-	      insn = gen_ldr_got_small_di (dest, tmp_reg, imm);
-	    else
-	      insn = gen_ldr_got_small_si (dest, tmp_reg, imm);
-
-	    mem = XVECEXP (SET_SRC (insn), 0, 0);
-	  }
-	else
-	  {
-	    gcc_assert (mode == Pmode);
-
-	    insn = gen_ldr_got_small_sidi (dest, tmp_reg, imm);
-	    mem = XVECEXP (XEXP (SET_SRC (insn), 0), 0, 0);
-	  }
-
-	gcc_assert (MEM_P (mem));
-	MEM_READONLY_P (mem) = 1;
-	MEM_NOTRAP_P (mem) = 1;
-	emit_insn (insn);
-	return;
-      }
+      emit_insn (gen_rtx_SET (dest, imm));
+      return;
 
     case SYMBOL_SMALL_TLSGD:
       {
@@ -6338,6 +6491,12 @@ aarch64_function_value (const_tree type, const_tree func,
 	  gcc_assert (count == 1 && mode == ag_mode);
 	  return gen_rtx_REG (mode, V0_REGNUM);
 	}
+      else if (aarch64_advsimd_full_struct_mode_p (mode)
+	       && known_eq (GET_MODE_SIZE (ag_mode), 16))
+	return gen_rtx_REG (mode, V0_REGNUM);
+      else if (aarch64_advsimd_partial_struct_mode_p (mode)
+	       && known_eq (GET_MODE_SIZE (ag_mode), 8))
+	return gen_rtx_REG (mode, V0_REGNUM);
       else
 	{
 	  int i;
@@ -6633,6 +6792,12 @@ aarch64_layout_arg (cumulative_args_t pcum_v, const function_arg_info &arg)
 	      gcc_assert (nregs == 1);
 	      pcum->aapcs_reg = gen_rtx_REG (mode, V0_REGNUM + nvrn);
 	    }
+	  else if (aarch64_advsimd_full_struct_mode_p (mode)
+		   && known_eq (GET_MODE_SIZE (pcum->aapcs_vfp_rmode), 16))
+	    pcum->aapcs_reg = gen_rtx_REG (mode, V0_REGNUM + nvrn);
+	  else if (aarch64_advsimd_partial_struct_mode_p (mode)
+		   && known_eq (GET_MODE_SIZE (pcum->aapcs_vfp_rmode), 8))
+	    pcum->aapcs_reg = gen_rtx_REG (mode, V0_REGNUM + nvrn);
 	  else
 	    {
 	      rtx par;
@@ -9076,13 +9241,9 @@ aarch64_expand_epilogue (bool for_sibcall)
 	2) The RETAA instruction is not available before ARMv8.3-A, so if we are
 	   generating code for !TARGET_ARMV8_3 we can't use it and must
 	   explicitly authenticate.
-
-	3) On an eh_return path we make extra stack adjustments to update the
-	   canonical frame address to be the exception handler's CFA.  We want
-	   to authenticate using the CFA of the function which calls eh_return.
     */
   if (aarch64_return_address_signing_enabled ()
-      && (for_sibcall || !TARGET_ARMV8_3 || crtl->calls_eh_return))
+      && (for_sibcall || !TARGET_ARMV8_3))
     {
       switch (aarch64_ra_sign_key)
 	{
@@ -9855,21 +10016,39 @@ aarch64_classify_address (struct aarch64_address_info *info,
 	     instruction (only big endian will get here).
 	     For ldp/stp instructions, the offset is scaled for the size of a
 	     single element of the pair.  */
-	  if (mode == OImode)
+	  if (aarch64_advsimd_partial_struct_mode_p (mode)
+	      && known_eq (GET_MODE_SIZE (mode), 16))
+	    return aarch64_offset_7bit_signed_scaled_p (DImode, offset);
+	  if (aarch64_advsimd_full_struct_mode_p (mode)
+	      && known_eq (GET_MODE_SIZE (mode), 32))
 	    return aarch64_offset_7bit_signed_scaled_p (TImode, offset);
 
 	  /* Three 9/12 bit offsets checks because CImode will emit three
 	     ldr/str instructions (only big endian will get here).  */
-	  if (mode == CImode)
+	  if (aarch64_advsimd_partial_struct_mode_p (mode)
+	      && known_eq (GET_MODE_SIZE (mode), 24))
+	    return (aarch64_offset_7bit_signed_scaled_p (DImode, offset)
+		    && (aarch64_offset_9bit_signed_unscaled_p (DImode,
+							       offset + 16)
+			|| offset_12bit_unsigned_scaled_p (DImode,
+							   offset + 16)));
+	  if (aarch64_advsimd_full_struct_mode_p (mode)
+	      && known_eq (GET_MODE_SIZE (mode), 48))
 	    return (aarch64_offset_7bit_signed_scaled_p (TImode, offset)
-		    && (aarch64_offset_9bit_signed_unscaled_p (V16QImode,
+		    && (aarch64_offset_9bit_signed_unscaled_p (TImode,
 							       offset + 32)
-			|| offset_12bit_unsigned_scaled_p (V16QImode,
+			|| offset_12bit_unsigned_scaled_p (TImode,
 							   offset + 32)));
 
 	  /* Two 7bit offsets checks because XImode will emit two ldp/stp
 	     instructions (only big endian will get here).  */
-	  if (mode == XImode)
+	  if (aarch64_advsimd_partial_struct_mode_p (mode)
+	      && known_eq (GET_MODE_SIZE (mode), 32))
+	    return (aarch64_offset_7bit_signed_scaled_p (DImode, offset)
+		    && aarch64_offset_7bit_signed_scaled_p (DImode,
+							    offset + 16));
+	  if (aarch64_advsimd_full_struct_mode_p (mode)
+	      && known_eq (GET_MODE_SIZE (mode), 64))
 	    return (aarch64_offset_7bit_signed_scaled_p (TImode, offset)
 		    && aarch64_offset_7bit_signed_scaled_p (TImode,
 							    offset + 32));
@@ -10956,7 +11135,10 @@ aarch64_print_operand (FILE *f, rtx x, int code)
       break;
 
     case 'R':
-      if (REG_P (x) && FP_REGNUM_P (REGNO (x)))
+      if (REG_P (x) && FP_REGNUM_P (REGNO (x))
+	  && (aarch64_advsimd_partial_struct_mode_p (GET_MODE (x))))
+	asm_fprintf (f, "d%d", REGNO (x) - V0_REGNUM + 1);
+      else if (REG_P (x) && FP_REGNUM_P (REGNO (x)))
 	asm_fprintf (f, "q%d", REGNO (x) - V0_REGNUM + 1);
       else if (REG_P (x) && GP_REGNUM_P (REGNO (x)))
 	asm_fprintf (f, "x%d", REGNO (x) - R0_REGNUM + 1);
@@ -11160,7 +11342,7 @@ aarch64_print_operand (FILE *f, rtx x, int code)
       switch (aarch64_classify_symbolic_expression (x))
 	{
 	case SYMBOL_SMALL_GOT_4G:
-	  asm_fprintf (asm_out_file, ":lo12:");
+	  asm_fprintf (asm_out_file, ":got_lo12:");
 	  break;
 
 	case SYMBOL_SMALL_TLSGD:
@@ -12347,9 +12529,10 @@ aarch64_address_cost (rtx x,
 	  cost += addr_cost->pre_modify;
 	else if (c == POST_INC || c == POST_DEC || c == POST_MODIFY)
 	  {
-	    if (mode == CImode)
+	    unsigned int nvectors = aarch64_ldn_stn_vectors (mode);
+	    if (nvectors == 3)
 	      cost += addr_cost->post_modify_ld3_st3;
-	    else if (mode == XImode)
+	    else if (nvectors == 4)
 	      cost += addr_cost->post_modify_ld4_st4;
 	    else
 	      cost += addr_cost->post_modify;
@@ -12705,7 +12888,7 @@ aarch64_rtx_costs (rtx x, machine_mode mode, int outer ATTRIBUTE_UNUSED,
   rtx op0, op1, op2;
   const struct cpu_cost_table *extra_cost
     = aarch64_tune_params.insn_extra_cost;
-  int code = GET_CODE (x);
+  rtx_code code = GET_CODE (x);
   scalar_int_mode int_mode;
 
   /* By default, assume that everything has equivalent cost to the
@@ -13466,8 +13649,7 @@ cost_plus:
 
 	 we must cost the explicit register move.  */
       if (mode == DImode
-	  && GET_MODE (op0) == SImode
-	  && outer == SET)
+	  && GET_MODE (op0) == SImode)
 	{
 	  int op_cost = rtx_cost (op0, VOIDmode, ZERO_EXTEND, 0, speed);
 
@@ -14006,8 +14188,39 @@ cost_plus:
 			     mode, MULT, 1, speed);
           return true;
         }
+	break;
+    case CONST_VECTOR:
+	{
+	  /* Load using MOVI/MVNI.  */
+	  if (aarch64_simd_valid_immediate (x, NULL))
+	    *cost = extra_cost->vect.movi;
+	  else /* Load using constant pool.  */
+	    *cost = extra_cost->ldst.load;
+	  break;
+	}
+    case VEC_CONCAT:
+	/* depending on the operation, either DUP or INS.
+	   For now, keep default costing.  */
+	break;
+    case VEC_DUPLICATE:
+	/* Load using a DUP.  */
+	*cost = extra_cost->vect.dup;
+	return false;
+    case VEC_SELECT:
+	{
+	  rtx op0 = XEXP (x, 0);
+	  *cost = rtx_cost (op0, GET_MODE (op0), VEC_SELECT, 0, speed);
 
-      /* Fall through.  */
+	  /* cost subreg of 0 as free, otherwise as DUP */
+	  rtx op1 = XEXP (x, 1);
+	  if (vec_series_lowpart_p (mode, GET_MODE (op1), op1))
+	    ;
+	  else if (vec_series_highpart_p (mode, GET_MODE (op1), op1))
+	    *cost = extra_cost->vect.dup;
+	  else
+	    *cost = extra_cost->vect.extract;
+	  return true;
+	}
     default:
       break;
     }
@@ -14494,8 +14707,25 @@ aarch64_first_cycle_multipass_dfa_lookahead_guard (rtx_insn *insn,
 /* Information about how the CPU would issue the scalar, Advanced SIMD
    or SVE version of a vector loop, using the scheme defined by the
    aarch64_base_vec_issue_info hierarchy of structures.  */
-struct aarch64_vec_op_count
+class aarch64_vec_op_count
 {
+public:
+  aarch64_vec_op_count () = default;
+  aarch64_vec_op_count (const aarch64_vec_issue_info *, unsigned int,
+			unsigned int = 1);
+
+  unsigned int vec_flags () const { return m_vec_flags; }
+  unsigned int vf_factor () const { return m_vf_factor; }
+
+  const aarch64_base_vec_issue_info *base_issue_info () const;
+  const aarch64_simd_vec_issue_info *simd_issue_info () const;
+  const aarch64_sve_vec_issue_info *sve_issue_info () const;
+
+  fractional_cost rename_cycles_per_iter () const;
+  fractional_cost min_nonpred_cycles_per_iter () const;
+  fractional_cost min_pred_cycles_per_iter () const;
+  fractional_cost min_cycles_per_iter () const;
+
   void dump () const;
 
   /* The number of individual "general" operations.  See the comments
@@ -14511,45 +14741,197 @@ struct aarch64_vec_op_count
      operations, which in the vector code become associated with
      reductions.  */
   unsigned int reduction_latency = 0;
-};
-
-/* Extends aarch64_vec_op_count with SVE-specific information.  */
-struct aarch64_sve_op_count : aarch64_vec_op_count
-{
-  void dump () const;
 
   /* The number of individual predicate operations.  See the comments
      in aarch64_sve_vec_issue_info for details.  */
   unsigned int pred_ops = 0;
+
+private:
+  /* The issue information for the core.  */
+  const aarch64_vec_issue_info *m_issue_info = nullptr;
+
+  /* - If M_VEC_FLAGS is zero then this structure describes scalar code
+     - If M_VEC_FLAGS & VEC_ADVSIMD is nonzero then this structure describes
+       Advanced SIMD code.
+     - If M_VEC_FLAGS & VEC_ANY_SVE is nonzero then this structure describes
+       SVE code.  */
+  unsigned int m_vec_flags = 0;
+
+  /* Assume that, when the code is executing on the core described
+     by M_ISSUE_INFO, one iteration of the loop will handle M_VF_FACTOR
+     times more data than the vectorizer anticipates.
+
+     This is only ever different from 1 for SVE.  It allows us to consider
+     what would happen on a 256-bit SVE target even when the -mtune
+     parameters say that the “likely” SVE length is 128 bits.  */
+  unsigned int m_vf_factor = 1;
 };
 
-/* Information about vector code that we're in the process of costing.  */
-struct aarch64_vector_costs
+aarch64_vec_op_count::
+aarch64_vec_op_count (const aarch64_vec_issue_info *issue_info,
+		      unsigned int vec_flags, unsigned int vf_factor)
+  : m_issue_info (issue_info),
+    m_vec_flags (vec_flags),
+    m_vf_factor (vf_factor)
+{
+}
+
+/* Return the base issue information (i.e. the parts that make sense
+   for both scalar and vector code).  Return null if we have no issue
+   information.  */
+const aarch64_base_vec_issue_info *
+aarch64_vec_op_count::base_issue_info () const
+{
+  if (auto *ret = simd_issue_info ())
+    return ret;
+  return m_issue_info->scalar;
+}
+
+/* If the structure describes vector code and we have associated issue
+   information, return that issue information, otherwise return null.  */
+const aarch64_simd_vec_issue_info *
+aarch64_vec_op_count::simd_issue_info () const
+{
+  if (auto *ret = sve_issue_info ())
+    return ret;
+  if (m_vec_flags)
+    return m_issue_info->advsimd;
+  return nullptr;
+}
+
+/* If the structure describes SVE code and we have associated issue
+   information, return that issue information, otherwise return null.  */
+const aarch64_sve_vec_issue_info *
+aarch64_vec_op_count::sve_issue_info () const
+{
+  if (m_vec_flags & VEC_ANY_SVE)
+    return m_issue_info->sve;
+  return nullptr;
+}
+
+/* Estimate the minimum number of cycles per iteration needed to rename
+   the instructions.
+
+   ??? For now this is done inline rather than via cost tables, since it
+   isn't clear how it should be parameterized for the general case.  */
+fractional_cost
+aarch64_vec_op_count::rename_cycles_per_iter () const
+{
+  if (sve_issue_info () == &neoverse512tvb_sve_issue_info)
+    /* + 1 for an addition.  We've already counted a general op for each
+       store, so we don't need to account for stores separately.  The branch
+       reads no registers and so does not need to be counted either.
+
+       ??? This value is very much on the pessimistic side, but seems to work
+       pretty well in practice.  */
+    return { general_ops + loads + pred_ops + 1, 5 };
+
+  return 0;
+}
+
+/* Like min_cycles_per_iter, but excluding predicate operations.  */
+fractional_cost
+aarch64_vec_op_count::min_nonpred_cycles_per_iter () const
 {
-  /* The normal latency-based costs for each region (prologue, body and
-     epilogue), indexed by vect_cost_model_location.  */
-  unsigned int region[3] = {};
+  auto *issue_info = base_issue_info ();
 
-  /* True if we have performed one-time initialization based on the vec_info.
+  fractional_cost cycles = MAX (reduction_latency, 1);
+  cycles = std::max (cycles, { stores, issue_info->stores_per_cycle });
+  cycles = std::max (cycles, { loads + stores,
+			       issue_info->loads_stores_per_cycle });
+  cycles = std::max (cycles, { general_ops,
+			       issue_info->general_ops_per_cycle });
+  cycles = std::max (cycles, rename_cycles_per_iter ());
+  return cycles;
+}
 
-     This variable exists because the vec_info is not passed to the
-     init_cost hook.  We therefore have to defer initialization based on
-     it till later.  */
-  bool analyzed_vinfo = false;
+/* Like min_cycles_per_iter, but including only the predicate operations.  */
+fractional_cost
+aarch64_vec_op_count::min_pred_cycles_per_iter () const
+{
+  if (auto *issue_info = sve_issue_info ())
+    return { pred_ops, issue_info->pred_ops_per_cycle };
+  return 0;
+}
 
-  /* True if we're costing a vector loop, false if we're costing block-level
-     vectorization.  */
-  bool is_loop = false;
+/* Estimate the minimum number of cycles needed to issue the operations.
+   This is a very simplistic model!  */
+fractional_cost
+aarch64_vec_op_count::min_cycles_per_iter () const
+{
+  return std::max (min_nonpred_cycles_per_iter (),
+		   min_pred_cycles_per_iter ());
+}
 
-  /* True if we've seen an SVE operation that we cannot currently vectorize
-     using Advanced SIMD.  */
-  bool saw_sve_only_op = false;
+/* Dump information about the structure.  */
+void
+aarch64_vec_op_count::dump () const
+{
+  dump_printf_loc (MSG_NOTE, vect_location,
+		   "  load operations = %d\n", loads);
+  dump_printf_loc (MSG_NOTE, vect_location,
+		   "  store operations = %d\n", stores);
+  dump_printf_loc (MSG_NOTE, vect_location,
+		   "  general operations = %d\n", general_ops);
+  if (sve_issue_info ())
+    dump_printf_loc (MSG_NOTE, vect_location,
+		     "  predicate operations = %d\n", pred_ops);
+  dump_printf_loc (MSG_NOTE, vect_location,
+		   "  reduction latency = %d\n", reduction_latency);
+  if (auto rcpi = rename_cycles_per_iter ())
+    dump_printf_loc (MSG_NOTE, vect_location,
+		     "  estimated cycles per iteration to rename = %f\n",
+		     rcpi.as_double ());
+  if (auto pred_cpi = min_pred_cycles_per_iter ())
+    {
+      dump_printf_loc (MSG_NOTE, vect_location,
+		       "  estimated min cycles per iteration"
+		       " without predication = %f\n",
+		       min_nonpred_cycles_per_iter ().as_double ());
+      dump_printf_loc (MSG_NOTE, vect_location,
+		       "  estimated min cycles per iteration"
+		       " for predication = %f\n", pred_cpi.as_double ());
+    }
+  if (auto cpi = min_cycles_per_iter ())
+    dump_printf_loc (MSG_NOTE, vect_location,
+		     "  estimated min cycles per iteration = %f\n",
+		     cpi.as_double ());
+}
 
-  /* - If VEC_FLAGS is zero then we're costing the original scalar code.
-     - If VEC_FLAGS & VEC_ADVSIMD is nonzero then we're costing Advanced
+/* Information about vector code that we're in the process of costing.  */
+class aarch64_vector_costs : public vector_costs
+{
+public:
+  aarch64_vector_costs (vec_info *, bool);
+
+  unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
+			      stmt_vec_info stmt_info, tree vectype,
+			      int misalign,
+			      vect_cost_model_location where) override;
+  void finish_cost (const vector_costs *) override;
+  bool better_main_loop_than_p (const vector_costs *other) const override;
+
+private:
+  void record_potential_advsimd_unrolling (loop_vec_info);
+  void analyze_loop_vinfo (loop_vec_info);
+  void count_ops (unsigned int, vect_cost_for_stmt, stmt_vec_info,
+		  aarch64_vec_op_count *);
+  fractional_cost adjust_body_cost_sve (const aarch64_vec_op_count *,
+					fractional_cost, unsigned int,
+					unsigned int *, bool *);
+  unsigned int adjust_body_cost (loop_vec_info, const aarch64_vector_costs *,
+				 unsigned int);
+  bool prefer_unrolled_loop () const;
+
+  /* True if we have performed one-time initialization based on the
+     vec_info.  */
+  bool m_analyzed_vinfo = false;
+
+  /* - If M_VEC_FLAGS is zero then we're costing the original scalar code.
+     - If M_VEC_FLAGS & VEC_ADVSIMD is nonzero then we're costing Advanced
        SIMD code.
-     - If VEC_FLAGS & VEC_ANY_SVE is nonzero then we're costing SVE code.  */
-  unsigned int vec_flags = 0;
+     - If M_VEC_FLAGS & VEC_ANY_SVE is nonzero then we're costing SVE code.  */
+  unsigned int m_vec_flags = 0;
 
   /* On some CPUs, SVE and Advanced SIMD provide the same theoretical vector
      throughput, such as 4x128 Advanced SIMD vs. 2x256 SVE.  In those
@@ -14559,46 +14941,49 @@ struct aarch64_vector_costs
      than length-agnostic SVE, since the SVE loop would execute an unknown
      number of times and so could not be completely unrolled in the same way.
 
-     If we're applying this heuristic, UNROLLED_ADVSIMD_NITERS is the
+     If we're applying this heuristic, M_UNROLLED_ADVSIMD_NITERS is the
      number of Advanced SIMD loop iterations that would be unrolled and
-     UNROLLED_ADVSIMD_STMTS estimates the total number of statements
+     M_UNROLLED_ADVSIMD_STMTS estimates the total number of statements
      in the unrolled loop.  Both values are zero if we're not applying
      the heuristic.  */
-  unsigned HOST_WIDE_INT unrolled_advsimd_niters = 0;
-  unsigned HOST_WIDE_INT unrolled_advsimd_stmts = 0;
+  unsigned HOST_WIDE_INT m_unrolled_advsimd_niters = 0;
+  unsigned HOST_WIDE_INT m_unrolled_advsimd_stmts = 0;
 
   /* If we're vectorizing a loop that executes a constant number of times,
      this variable gives the number of times that the vector loop would
      iterate, otherwise it is zero.  */
-  uint64_t num_vector_iterations = 0;
-
-  /* Used only when vectorizing loops.  Estimates the number and kind of scalar
-     operations that would be needed to perform the same work as one iteration
-     of the vector loop.  */
-  aarch64_vec_op_count scalar_ops;
-
-  /* Used only when vectorizing loops.  If VEC_FLAGS & VEC_ADVSIMD,
-     this structure estimates the number and kind of operations that the
-     vector loop would contain.  If VEC_FLAGS & VEC_SVE, the structure
-     estimates what the equivalent Advanced SIMD-only code would need in
-     order to perform the same work as one iteration of the SVE loop.  */
-  aarch64_vec_op_count advsimd_ops;
-
-  /* Used only when vectorizing loops with SVE.  It estimates the number and
-     kind of operations that the SVE loop would contain.  */
-  aarch64_sve_op_count sve_ops;
-
-  /* Used to detect cases in which we end up costing the same load twice,
-     once to account for results that are actually used and once to account
-     for unused results.  */
-  hash_map<nofree_ptr_hash<_stmt_vec_info>, unsigned int> seen_loads;
+  uint64_t m_num_vector_iterations = 0;
+
+  /* Used only when vectorizing loops.  Estimates the number and kind of
+     operations that would be needed by one iteration of the scalar
+     or vector loop.  There is one entry for each tuning option of
+     interest.  */
+  auto_vec<aarch64_vec_op_count, 2> m_ops;
 };
 
-/* Implement TARGET_VECTORIZE_INIT_COST.  */
-void *
-aarch64_init_cost (class loop *, bool)
+aarch64_vector_costs::aarch64_vector_costs (vec_info *vinfo,
+					    bool costing_for_scalar)
+  : vector_costs (vinfo, costing_for_scalar),
+    m_vec_flags (costing_for_scalar ? 0
+		 : aarch64_classify_vector_mode (vinfo->vector_mode))
+{
+  if (auto *issue_info = aarch64_tune_params.vec_costs->issue_info)
+    {
+      m_ops.quick_push ({ issue_info, m_vec_flags });
+      if (aarch64_tune_params.vec_costs == &neoverse512tvb_vector_cost)
+	{
+	  unsigned int vf_factor = (m_vec_flags & VEC_ANY_SVE) ? 2 : 1;
+	  m_ops.quick_push ({ &neoversev1_vec_issue_info, m_vec_flags,
+			      vf_factor });
+	}
+    }
+}
+
+/* Implement TARGET_VECTORIZE_CREATE_COSTS.  */
+vector_costs *
+aarch64_vectorize_create_costs (vec_info *vinfo, bool costing_for_scalar)
 {
-  return new aarch64_vector_costs;
+  return new aarch64_vector_costs (vinfo, costing_for_scalar);
 }
 
 /* Return true if the current CPU should use the new costs defined
@@ -14633,13 +15018,22 @@ aarch64_simd_vec_costs_for_flags (unsigned int flags)
   return costs->advsimd;
 }
 
+/* If STMT_INFO is a memory reference, return the scalar memory type,
+   otherwise return null.  */
+static tree
+aarch64_dr_type (stmt_vec_info stmt_info)
+{
+  if (auto dr = STMT_VINFO_DATA_REF (stmt_info))
+    return TREE_TYPE (DR_REF (dr));
+  return NULL_TREE;
+}
+
 /* Decide whether to use the unrolling heuristic described above
-   aarch64_vector_costs::unrolled_advsimd_niters, updating that
-   field if so.  LOOP_VINFO describes the loop that we're vectorizing
-   and COSTS are the costs that we're calculating for it.  */
-static void
-aarch64_record_potential_advsimd_unrolling (loop_vec_info loop_vinfo,
-					    aarch64_vector_costs *costs)
+   m_unrolled_advsimd_niters, updating that field if so.  LOOP_VINFO
+   describes the loop that we're vectorizing.  */
+void
+aarch64_vector_costs::
+record_potential_advsimd_unrolling (loop_vec_info loop_vinfo)
 {
   /* The heuristic only makes sense on targets that have the same
      vector throughput for SVE and Advanced SIMD.  */
@@ -14649,7 +15043,7 @@ aarch64_record_potential_advsimd_unrolling (loop_vec_info loop_vinfo,
 
   /* We only want to apply the heuristic if LOOP_VINFO is being
      vectorized for SVE.  */
-  if (!(costs->vec_flags & VEC_ANY_SVE))
+  if (!(m_vec_flags & VEC_ANY_SVE))
     return;
 
   /* Check whether it is possible in principle to use Advanced SIMD
@@ -14682,17 +15076,14 @@ aarch64_record_potential_advsimd_unrolling (loop_vec_info loop_vinfo,
 
   /* Record that we're applying the heuristic and should try to estimate
      the number of statements in the Advanced SIMD loop.  */
-  costs->unrolled_advsimd_niters = unrolled_advsimd_niters;
+  m_unrolled_advsimd_niters = unrolled_advsimd_niters;
 }
 
-/* Do one-time initialization of COSTS given that we're costing the loop
-   vectorization described by LOOP_VINFO.  */
-static void
-aarch64_analyze_loop_vinfo (loop_vec_info loop_vinfo,
-			    aarch64_vector_costs *costs)
+/* Do one-time initialization of the aarch64_vector_costs given that we're
+   costing the loop vectorization described by LOOP_VINFO.  */
+void
+aarch64_vector_costs::analyze_loop_vinfo (loop_vec_info loop_vinfo)
 {
-  costs->is_loop = true;
-
   /* Record the number of times that the vector loop would execute,
      if known.  */
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
@@ -14701,33 +15092,18 @@ aarch64_analyze_loop_vinfo (loop_vec_info loop_vinfo,
     {
       unsigned int vf = vect_vf_for_cost (loop_vinfo);
       if (LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
-	costs->num_vector_iterations = scalar_niters / vf;
+	m_num_vector_iterations = scalar_niters / vf;
       else
-	costs->num_vector_iterations = CEIL (scalar_niters, vf);
+	m_num_vector_iterations = CEIL (scalar_niters, vf);
     }
 
-  /* Detect whether we're costing the scalar code or the vector code.
-     This is a bit hacky: it would be better if the vectorizer told
-     us directly.
-
-     If we're costing the vector code, record whether we're vectorizing
-     for Advanced SIMD or SVE.  */
-  if (costs == LOOP_VINFO_TARGET_COST_DATA (loop_vinfo))
-    costs->vec_flags = aarch64_classify_vector_mode (loop_vinfo->vector_mode);
-  else
-    costs->vec_flags = 0;
-
-  /* Detect whether we're vectorizing for SVE and should
-     apply the unrolling heuristic described above
-     aarch64_vector_costs::unrolled_advsimd_niters.  */
-  aarch64_record_potential_advsimd_unrolling (loop_vinfo, costs);
+  /* Detect whether we're vectorizing for SVE and should apply the unrolling
+     heuristic described above m_unrolled_advsimd_niters.  */
+  record_potential_advsimd_unrolling (loop_vinfo);
 
   /* Record the issue information for any SVE WHILE instructions that the
      loop needs.  */
-  auto *issue_info = aarch64_tune_params.vec_costs->issue_info;
-  if (issue_info
-      && issue_info->sve
-      && !LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
+  if (!m_ops.is_empty () && !LOOP_VINFO_MASKS (loop_vinfo).is_empty ())
     {
       unsigned int num_masks = 0;
       rgroup_controls *rgm;
@@ -14735,21 +15111,12 @@ aarch64_analyze_loop_vinfo (loop_vec_info loop_vinfo,
       FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo), num_vectors_m1, rgm)
 	if (rgm->type)
 	  num_masks += num_vectors_m1 + 1;
-      costs->sve_ops.pred_ops += num_masks * issue_info->sve->while_pred_ops;
+      for (auto &ops : m_ops)
+	if (auto *issue = ops.sve_issue_info ())
+	  ops.pred_ops += num_masks * issue->while_pred_ops;
     }
 }
 
-/* Do one-time initialization of COSTS given that we're costing the block
-   vectorization described by BB_VINFO.  */
-static void
-aarch64_analyze_bb_vinfo (bb_vec_info bb_vinfo, aarch64_vector_costs *costs)
-{
-  /* Unfortunately, there's no easy way of telling whether we're costing
-     the vector code or the scalar code, so just assume that we're costing
-     the vector code.  */
-  costs->vec_flags = aarch64_classify_vector_mode (bb_vinfo->vector_mode);
-}
-
 /* Implement targetm.vectorize.builtin_vectorization_cost.  */
 static int
 aarch64_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
@@ -14924,50 +15291,13 @@ aarch64_multiply_add_p (vec_info *vinfo, stmt_vec_info stmt_info,
   return false;
 }
 
-/* Return true if the vectorized form of STMT_INFO is something that is only
-   possible when using SVE instead of Advanced SIMD.  VECTYPE is the type of
-   the vector that STMT_INFO is operating on.  */
-static bool
-aarch64_sve_only_stmt_p (stmt_vec_info stmt_info, tree vectype)
-{
-  if (!aarch64_sve_mode_p (TYPE_MODE (vectype)))
-    return false;
-
-  if (STMT_VINFO_DATA_REF (stmt_info))
-    {
-      /* Check for true gathers and scatters (rather than just strided accesses
-	 that we've chosen to implement using gathers and scatters).  Although
-	 in principle we could use elementwise accesses for Advanced SIMD,
-	 the vectorizer doesn't yet support that.  */
-      if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-	return true;
-
-      /* Check for masked loads and stores.  */
-      if (auto *call = dyn_cast<gcall *> (stmt_info->stmt))
-	if (gimple_call_internal_p (call)
-	    && internal_fn_mask_index (gimple_call_internal_fn (call)) >= 0)
-	  return true;
-    }
-
-  /* Check for 64-bit integer multiplications.  */
-  auto *assign = dyn_cast<gassign *> (stmt_info->stmt);
-  if (assign
-      && gimple_assign_rhs_code (assign) == MULT_EXPR
-      && GET_MODE_INNER (TYPE_MODE (vectype)) == DImode
-      && !integer_pow2p (gimple_assign_rhs2 (assign)))
-    return true;
-
-  return false;
-}
-
-/* We are considering implementing STMT_INFO using SVE vector type VECTYPE.
-   If STMT_INFO is an in-loop reduction that SVE supports directly, return
-   its latency in cycles, otherwise return zero.  SVE_COSTS specifies the
-   latencies of the relevant instructions.  */
+/* We are considering implementing STMT_INFO using SVE.  If STMT_INFO is an
+   in-loop reduction that SVE supports directly, return its latency in cycles,
+   otherwise return zero.  SVE_COSTS specifies the latencies of the relevant
+   instructions.  */
 static unsigned int
 aarch64_sve_in_loop_reduction_latency (vec_info *vinfo,
 				       stmt_vec_info stmt_info,
-				       tree vectype,
 				       const sve_vec_cost *sve_costs)
 {
   switch (vect_reduc_type (vinfo, stmt_info))
@@ -14976,7 +15306,7 @@ aarch64_sve_in_loop_reduction_latency (vec_info *vinfo,
       return sve_costs->clast_cost;
 
     case FOLD_LEFT_REDUCTION:
-      switch (GET_MODE_INNER (TYPE_MODE (vectype)))
+      switch (TYPE_MODE (TREE_TYPE (gimple_get_lhs (stmt_info->stmt))))
 	{
 	case E_HFmode:
 	case E_BFmode:
@@ -15008,14 +15338,10 @@ aarch64_sve_in_loop_reduction_latency (vec_info *vinfo,
      Advanced SIMD implementation.
 
    - If VEC_FLAGS & VEC_ANY_SVE, return the loop carry latency of the
-     SVE implementation.
-
-   VECTYPE is the type of vector that the vectorizer is considering using
-   for STMT_INFO, which might be different from the type of vector described
-   by VEC_FLAGS.  */
+     SVE implementation.  */
 static unsigned int
 aarch64_in_loop_reduction_latency (vec_info *vinfo, stmt_vec_info stmt_info,
-				   tree vectype, unsigned int vec_flags)
+				   unsigned int vec_flags)
 {
   const cpu_vector_cost *vec_costs = aarch64_tune_params.vec_costs;
   const sve_vec_cost *sve_costs = nullptr;
@@ -15027,16 +15353,16 @@ aarch64_in_loop_reduction_latency (vec_info *vinfo, stmt_vec_info stmt_info,
   if (sve_costs)
     {
       unsigned int latency
-	= aarch64_sve_in_loop_reduction_latency (vinfo, stmt_info, vectype,
-						 sve_costs);
+	= aarch64_sve_in_loop_reduction_latency (vinfo, stmt_info, sve_costs);
       if (latency)
 	return latency;
     }
 
   /* Handle scalar costs.  */
+  bool is_float = FLOAT_TYPE_P (TREE_TYPE (gimple_get_lhs (stmt_info->stmt)));
   if (vec_flags == 0)
     {
-      if (FLOAT_TYPE_P (vectype))
+      if (is_float)
 	return vec_costs->scalar_fp_stmt_cost;
       return vec_costs->scalar_int_stmt_cost;
     }
@@ -15045,7 +15371,7 @@ aarch64_in_loop_reduction_latency (vec_info *vinfo, stmt_vec_info stmt_info,
      with a vector reduction outside the loop.  */
   const simd_vec_cost *simd_costs
     = aarch64_simd_vec_costs_for_flags (vec_flags);
-  if (FLOAT_TYPE_P (vectype))
+  if (is_float)
     return simd_costs->fp_stmt_cost;
   return simd_costs->int_stmt_cost;
 }
@@ -15122,8 +15448,7 @@ aarch64_detect_vector_stmt_subtype (vec_info *vinfo, vect_cost_for_stmt kind,
       && sve_costs)
     {
       unsigned int latency
-	= aarch64_sve_in_loop_reduction_latency (vinfo, stmt_info, vectype,
-						 sve_costs);
+	= aarch64_sve_in_loop_reduction_latency (vinfo, stmt_info, sve_costs);
       if (latency)
 	return latency;
     }
@@ -15283,89 +15608,37 @@ aarch64_adjust_stmt_cost (vect_cost_for_stmt kind, stmt_vec_info stmt_info,
   return stmt_cost;
 }
 
-/* VINFO, COSTS, COUNT, KIND, STMT_INFO and VECTYPE are the same as for
-   TARGET_VECTORIZE_ADD_STMT_COST and they describe an operation in the
-   body of a vector loop.  Record issue information relating to the vector
-   operation in OPS, where OPS is one of COSTS->scalar_ops, COSTS->advsimd_ops
-   or COSTS->sve_ops; see the comments above those variables for details.
-   In addition:
-
-   - VEC_FLAGS is zero if OPS is COSTS->scalar_ops.
-
-   - VEC_FLAGS & VEC_ADVSIMD is nonzero if OPS is COSTS->advsimd_ops.
-
-   - VEC_FLAGS & VEC_ANY_SVE is nonzero if OPS is COSTS->sve_ops.
-
-   ISSUE_INFO provides the scalar, Advanced SIMD or SVE issue information
-   associated with OPS and VEC_FLAGS.  FACTOR says how many iterations of
-   the loop described by VEC_FLAGS would be needed to match one iteration
-   of the vector loop in VINFO.  */
-static void
-aarch64_count_ops (class vec_info *vinfo, aarch64_vector_costs *costs,
-		   unsigned int count, enum vect_cost_for_stmt kind,
-		   _stmt_vec_info *stmt_info, tree vectype,
-		   unsigned int vec_flags, aarch64_vec_op_count *ops,
-		   const aarch64_base_vec_issue_info *issue_info,
-		   unsigned int factor)
-{
-  if (!issue_info)
+/* COUNT, KIND and STMT_INFO are the same as for vector_costs::add_stmt_cost
+   and they describe an operation in the body of a vector loop.  Record issue
+   information relating to the vector operation in OPS.  */
+void
+aarch64_vector_costs::count_ops (unsigned int count, vect_cost_for_stmt kind,
+				 stmt_vec_info stmt_info,
+				 aarch64_vec_op_count *ops)
+{
+  const aarch64_base_vec_issue_info *base_issue = ops->base_issue_info ();
+  if (!base_issue)
     return;
-
-  const aarch64_simd_vec_issue_info *simd_issue = nullptr;
-  if (vec_flags)
-    simd_issue = static_cast<const aarch64_simd_vec_issue_info *> (issue_info);
-
-  const aarch64_sve_vec_issue_info *sve_issue = nullptr;
-  if (vec_flags & VEC_ANY_SVE)
-    sve_issue = static_cast<const aarch64_sve_vec_issue_info *> (issue_info);
+  const aarch64_simd_vec_issue_info *simd_issue = ops->simd_issue_info ();
+  const aarch64_sve_vec_issue_info *sve_issue = ops->sve_issue_info ();
 
   /* Calculate the minimum cycles per iteration imposed by a reduction
      operation.  */
-  if ((kind == vector_stmt || kind == vec_to_scalar)
+  if ((kind == scalar_stmt || kind == vector_stmt || kind == vec_to_scalar)
       && vect_is_reduction (stmt_info))
     {
       unsigned int base
-	= aarch64_in_loop_reduction_latency (vinfo, stmt_info, vectype,
-					     vec_flags);
-      if (vect_reduc_type (vinfo, stmt_info) == FOLD_LEFT_REDUCTION)
-	{
-	  if (aarch64_sve_mode_p (TYPE_MODE (vectype)))
-	    {
-	      /* When costing an SVE FADDA, the vectorizer treats vec_to_scalar
-		 as a single operation, whereas for Advanced SIMD it is a
-		 per-element one.  Increase the factor accordingly, both for
-		 the reduction_latency calculation and for the op couting.  */
-	      if (vec_flags & VEC_ADVSIMD)
-		factor = vect_nunits_for_cost (vectype);
-	    }
-	  else
-	    /* An Advanced SIMD fold-left reduction is the same as a
-	       scalar one and the vectorizer therefore treats vec_to_scalar
-	       as a per-element cost.  There is no extra factor to apply for
-	       scalar code, either for reduction_latency or for the op
-	       counting below.  */
-	    factor = 1;
-	}
+	= aarch64_in_loop_reduction_latency (m_vinfo, stmt_info, m_vec_flags);
 
-      /* ??? Ideally for vector code we'd do COUNT * FACTOR reductions in
-	 parallel, but unfortunately that's not yet the case.  */
-      ops->reduction_latency = MAX (ops->reduction_latency,
-				    base * count * factor);
+      /* ??? Ideally we'd do COUNT reductions in parallel, but unfortunately
+	 that's not yet the case.  */
+      ops->reduction_latency = MAX (ops->reduction_latency, base * count);
     }
 
   /* Assume that multiply-adds will become a single operation.  */
-  if (stmt_info && aarch64_multiply_add_p (vinfo, stmt_info, vec_flags))
+  if (stmt_info && aarch64_multiply_add_p (m_vinfo, stmt_info, m_vec_flags))
     return;
 
-  /* When costing scalar statements in vector code, the count already
-     includes the number of scalar elements in the vector, so we don't
-     need to apply the factor as well.  */
-  if (kind == scalar_load || kind == scalar_store || kind == scalar_stmt)
-    factor = 1;
-
-  /* This can go negative with the load handling below.  */
-  int num_copies = count * factor;
-
   /* Count the basic operation cost associated with KIND.  */
   switch (kind)
     {
@@ -15381,65 +15654,38 @@ aarch64_count_ops (class vec_info *vinfo, aarch64_vector_costs *costs,
     case vec_construct:
     case vec_to_scalar:
     case scalar_to_vec:
-      /* Assume that these operations have no overhead in the original
-	 scalar code.  */
-      if (!vec_flags)
-	break;
-      /* Fallthrough.  */
     case vector_stmt:
     case scalar_stmt:
-      ops->general_ops += num_copies;
+      ops->general_ops += count;
       break;
 
     case scalar_load:
     case vector_load:
     case unaligned_load:
-      /* When costing scalars, detect cases in which we are called twice for
-	 the same load.  This happens for LD[234] operations if only some of
-	 the results are used.  The first time represents the cost of loading
-	 the unused vectors, while the second time represents the cost of
-	 loading the useful parts.  Only the latter should count towards the
-	 scalar costs.  */
-      if (stmt_info && !vec_flags)
-	{
-	  bool existed = false;
-	  unsigned int &prev_count
-	    = costs->seen_loads.get_or_insert (stmt_info, &existed);
-	  if (existed)
-	    num_copies -= prev_count;
-	  else
-	    prev_count = num_copies;
-	}
-      ops->loads += num_copies;
-      if (vec_flags || FLOAT_TYPE_P (vectype))
-	ops->general_ops += issue_info->fp_simd_load_general_ops * num_copies;
+      ops->loads += count;
+      if (m_vec_flags || FLOAT_TYPE_P (aarch64_dr_type (stmt_info)))
+	ops->general_ops += base_issue->fp_simd_load_general_ops * count;
       break;
 
     case vector_store:
     case unaligned_store:
     case scalar_store:
-      ops->stores += num_copies;
-      if (vec_flags || FLOAT_TYPE_P (vectype))
-	ops->general_ops += issue_info->fp_simd_store_general_ops * num_copies;
+      ops->stores += count;
+      if (m_vec_flags || FLOAT_TYPE_P (aarch64_dr_type (stmt_info)))
+	ops->general_ops += base_issue->fp_simd_store_general_ops * count;
       break;
     }
 
   /* Add any embedded comparison operations.  */
   if ((kind == scalar_stmt || kind == vector_stmt || kind == vec_to_scalar)
       && vect_embedded_comparison_type (stmt_info))
-    ops->general_ops += num_copies;
+    ops->general_ops += count;
 
-  /* Detect COND_REDUCTIONs and things that would need to become
-     COND_REDUCTIONs if they were implemented using Advanced SIMD.
-     There are then two sets of VEC_COND_EXPRs, whereas so far we
+  /* COND_REDUCTIONS need two sets of VEC_COND_EXPRs, whereas so far we
      have only accounted for one.  */
-  if (vec_flags && (kind == vector_stmt || kind == vec_to_scalar))
-    {
-      int reduc_type = vect_reduc_type (vinfo, stmt_info);
-      if ((reduc_type == EXTRACT_LAST_REDUCTION && (vec_flags & VEC_ADVSIMD))
-	  || reduc_type == COND_REDUCTION)
-	ops->general_ops += num_copies;
-    }
+  if ((kind == vector_stmt || kind == vec_to_scalar)
+      && vect_reduc_type (m_vinfo, stmt_info) == COND_REDUCTION)
+    ops->general_ops += count;
 
   /* Count the predicate operations needed by an SVE comparison.  */
   if (sve_issue && (kind == vector_stmt || kind == vec_to_scalar))
@@ -15448,7 +15694,7 @@ aarch64_count_ops (class vec_info *vinfo, aarch64_vector_costs *costs,
 	unsigned int base = (FLOAT_TYPE_P (type)
 			     ? sve_issue->fp_cmp_pred_ops
 			     : sve_issue->int_cmp_pred_ops);
-	costs->sve_ops.pred_ops += base * num_copies;
+	ops->pred_ops += base * count;
       }
 
   /* Add any extra overhead associated with LD[234] and ST[234] operations.  */
@@ -15456,15 +15702,15 @@ aarch64_count_ops (class vec_info *vinfo, aarch64_vector_costs *costs,
     switch (aarch64_ld234_st234_vectors (kind, stmt_info))
       {
       case 2:
-	ops->general_ops += simd_issue->ld2_st2_general_ops * num_copies;
+	ops->general_ops += simd_issue->ld2_st2_general_ops * count;
 	break;
 
       case 3:
-	ops->general_ops += simd_issue->ld3_st3_general_ops * num_copies;
+	ops->general_ops += simd_issue->ld3_st3_general_ops * count;
 	break;
 
       case 4:
-	ops->general_ops += simd_issue->ld4_st4_general_ops * num_copies;
+	ops->general_ops += simd_issue->ld4_st4_general_ops * count;
 	break;
       }
 
@@ -15474,65 +15720,56 @@ aarch64_count_ops (class vec_info *vinfo, aarch64_vector_costs *costs,
       && STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_GATHER_SCATTER)
     {
       unsigned int pairs = CEIL (count, 2);
-      costs->sve_ops.pred_ops
-	+= sve_issue->gather_scatter_pair_pred_ops * pairs;
+      ops->pred_ops += sve_issue->gather_scatter_pair_pred_ops * pairs;
       ops->general_ops += sve_issue->gather_scatter_pair_general_ops * pairs;
     }
 }
 
-/* Implement targetm.vectorize.add_stmt_cost.  */
-static unsigned
-aarch64_add_stmt_cost (class vec_info *vinfo, void *data, int count,
-		       enum vect_cost_for_stmt kind,
-		       struct _stmt_vec_info *stmt_info, tree vectype,
-		       int misalign, enum vect_cost_model_location where)
+unsigned
+aarch64_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
+				     stmt_vec_info stmt_info, tree vectype,
+				     int misalign,
+				     vect_cost_model_location where)
 {
-  auto *costs = static_cast<aarch64_vector_costs *> (data);
-
   fractional_cost stmt_cost
     = aarch64_builtin_vectorization_cost (kind, vectype, misalign);
 
   bool in_inner_loop_p = (where == vect_body
 			  && stmt_info
-			  && stmt_in_inner_loop_p (vinfo, stmt_info));
+			  && stmt_in_inner_loop_p (m_vinfo, stmt_info));
 
   /* Do one-time initialization based on the vinfo.  */
-  loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
-  bb_vec_info bb_vinfo = dyn_cast<bb_vec_info> (vinfo);
-  if (!costs->analyzed_vinfo && aarch64_use_new_vector_costs_p ())
+  loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
+  if (!m_analyzed_vinfo && aarch64_use_new_vector_costs_p ())
     {
       if (loop_vinfo)
-	aarch64_analyze_loop_vinfo (loop_vinfo, costs);
-      else
-	aarch64_analyze_bb_vinfo (bb_vinfo, costs);
-      costs->analyzed_vinfo = true;
+	analyze_loop_vinfo (loop_vinfo);
+
+      m_analyzed_vinfo = true;
     }
 
   /* Try to get a more accurate cost by looking at STMT_INFO instead
      of just looking at KIND.  */
   if (stmt_info && aarch64_use_new_vector_costs_p ())
     {
-      if (vectype && aarch64_sve_only_stmt_p (stmt_info, vectype))
-	costs->saw_sve_only_op = true;
-
       /* If we scalarize a strided store, the vectorizer costs one
 	 vec_to_scalar for each element.  However, we can store the first
 	 element using an FP store without a separate extract step.  */
       if (vect_is_store_elt_extraction (kind, stmt_info))
 	count -= 1;
 
-      stmt_cost = aarch64_detect_scalar_stmt_subtype
-	(vinfo, kind, stmt_info, stmt_cost);
+      stmt_cost = aarch64_detect_scalar_stmt_subtype (m_vinfo, kind,
+						      stmt_info, stmt_cost);
 
-      if (vectype && costs->vec_flags)
-	stmt_cost = aarch64_detect_vector_stmt_subtype (vinfo, kind,
+      if (vectype && m_vec_flags)
+	stmt_cost = aarch64_detect_vector_stmt_subtype (m_vinfo, kind,
 							stmt_info, vectype,
 							where, stmt_cost);
     }
 
   /* Do any SVE-specific adjustments to the cost.  */
   if (stmt_info && vectype && aarch64_sve_mode_p (TYPE_MODE (vectype)))
-    stmt_cost = aarch64_sve_adjust_stmt_cost (vinfo, kind, stmt_info,
+    stmt_cost = aarch64_sve_adjust_stmt_cost (m_vinfo, kind, stmt_info,
 					      vectype, stmt_cost);
 
   if (stmt_info && aarch64_use_new_vector_costs_p ())
@@ -15545,182 +15782,71 @@ aarch64_add_stmt_cost (class vec_info *vinfo, void *data, int count,
       /* If we're recording a nonzero vector loop body cost for the
 	 innermost loop, also estimate the operations that would need
 	 to be issued by all relevant implementations of the loop.  */
-      auto *issue_info = aarch64_tune_params.vec_costs->issue_info;
       if (loop_vinfo
-	  && issue_info
-	  && costs->vec_flags
-	  && where == vect_body
+	  && (m_costing_for_scalar || where == vect_body)
 	  && (!LOOP_VINFO_LOOP (loop_vinfo)->inner || in_inner_loop_p)
-	  && vectype
 	  && stmt_cost != 0)
-	{
-	  /* Record estimates for the scalar code.  */
-	  aarch64_count_ops (vinfo, costs, count, kind, stmt_info, vectype,
-			     0, &costs->scalar_ops, issue_info->scalar,
-			     vect_nunits_for_cost (vectype));
-
-	  if (aarch64_sve_mode_p (vinfo->vector_mode) && issue_info->sve)
-	    {
-	      /* Record estimates for a possible Advanced SIMD version
-		 of the SVE code.  */
-	      aarch64_count_ops (vinfo, costs, count, kind, stmt_info,
-				 vectype, VEC_ADVSIMD, &costs->advsimd_ops,
-				 issue_info->advsimd,
-				 aarch64_estimated_sve_vq ());
-
-	      /* Record estimates for the SVE code itself.  */
-	      aarch64_count_ops (vinfo, costs, count, kind, stmt_info,
-				 vectype, VEC_ANY_SVE, &costs->sve_ops,
-				 issue_info->sve, 1);
-	    }
-	  else
-	    /* Record estimates for the Advanced SIMD code.  Treat SVE like
-	       Advanced SIMD if the CPU has no specific SVE costs.  */
-	    aarch64_count_ops (vinfo, costs, count, kind, stmt_info,
-			       vectype, VEC_ADVSIMD, &costs->advsimd_ops,
-			       issue_info->advsimd, 1);
-	}
+	for (auto &ops : m_ops)
+	  count_ops (count, kind, stmt_info, &ops);
 
       /* If we're applying the SVE vs. Advanced SIMD unrolling heuristic,
 	 estimate the number of statements in the unrolled Advanced SIMD
 	 loop.  For simplicitly, we assume that one iteration of the
 	 Advanced SIMD loop would need the same number of statements
 	 as one iteration of the SVE loop.  */
-      if (where == vect_body && costs->unrolled_advsimd_niters)
-	costs->unrolled_advsimd_stmts
-	  += count * costs->unrolled_advsimd_niters;
+      if (where == vect_body && m_unrolled_advsimd_niters)
+	m_unrolled_advsimd_stmts += count * m_unrolled_advsimd_niters;
     }
-
-  /* Statements in an inner loop relative to the loop being
-     vectorized are weighted more heavily.  The value here is
-     arbitrary and could potentially be improved with analysis.  */
-  if (in_inner_loop_p)
-    {
-      gcc_assert (loop_vinfo);
-      count *= LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo); /*  FIXME  */
-    }
-
-  unsigned retval = (count * stmt_cost).ceil ();
-  costs->region[where] += retval;
-
-  return retval;
-}
-
-/* Dump information about the structure.  */
-void
-aarch64_vec_op_count::dump () const
-{
-  dump_printf_loc (MSG_NOTE, vect_location,
-		   "  load operations = %d\n", loads);
-  dump_printf_loc (MSG_NOTE, vect_location,
-		   "  store operations = %d\n", stores);
-  dump_printf_loc (MSG_NOTE, vect_location,
-		   "  general operations = %d\n", general_ops);
-  dump_printf_loc (MSG_NOTE, vect_location,
-		   "  reduction latency = %d\n", reduction_latency);
+  return record_stmt_cost (stmt_info, where, (count * stmt_cost).ceil ());
 }
 
-/* Dump information about the structure.  */
-void
-aarch64_sve_op_count::dump () const
+/* Return true if (a) we're applying the Advanced SIMD vs. SVE unrolling
+   heuristic described above m_unrolled_advsimd_niters and (b) the heuristic
+   says that we should prefer the Advanced SIMD loop.  */
+bool
+aarch64_vector_costs::prefer_unrolled_loop () const
 {
-  aarch64_vec_op_count::dump ();
-  dump_printf_loc (MSG_NOTE, vect_location,
-		   "  predicate operations = %d\n", pred_ops);
-}
+  if (!m_unrolled_advsimd_stmts)
+    return false;
 
-/* Use ISSUE_INFO to estimate the minimum number of cycles needed to issue
-   the operations described by OPS.  This is a very simplistic model!  */
-static fractional_cost
-aarch64_estimate_min_cycles_per_iter
-  (const aarch64_vec_op_count *ops,
-   const aarch64_base_vec_issue_info *issue_info)
+  if (dump_enabled_p ())
+    dump_printf_loc (MSG_NOTE, vect_location, "Number of insns in"
+		     " unrolled Advanced SIMD loop = %d\n",
+		     m_unrolled_advsimd_stmts);
+
+  /* The balance here is tricky.  On the one hand, we can't be sure whether
+     the code is vectorizable with Advanced SIMD or not.  However, even if
+     it isn't vectorizable with Advanced SIMD, there's a possibility that
+     the scalar code could also be unrolled.  Some of the code might then
+     benefit from SLP, or from using LDP and STP.  We therefore apply
+     the heuristic regardless of can_use_advsimd_p.  */
+  return (m_unrolled_advsimd_stmts
+	  && (m_unrolled_advsimd_stmts
+	      <= (unsigned int) param_max_completely_peeled_insns));
+}
+
+/* Subroutine of adjust_body_cost for handling SVE.  Use ISSUE_INFO to work out
+   how fast the SVE code can be issued and compare it to the equivalent value
+   for scalar code (SCALAR_CYCLES_PER_ITER).  If COULD_USE_ADVSIMD is true,
+   also compare it to the issue rate of Advanced SIMD code
+   (ADVSIMD_CYCLES_PER_ITER).
+
+   ORIG_BODY_COST is the cost originally passed to adjust_body_cost and
+   *BODY_COST is the current value of the adjusted cost.  *SHOULD_DISPARAGE
+   is true if we think the loop body is too expensive.  */
+
+fractional_cost
+aarch64_vector_costs::
+adjust_body_cost_sve (const aarch64_vec_op_count *ops,
+		      fractional_cost scalar_cycles_per_iter,
+		      unsigned int orig_body_cost, unsigned int *body_cost,
+		      bool *should_disparage)
 {
-  fractional_cost cycles = MAX (ops->reduction_latency, 1);
-  cycles = std::max (cycles, { ops->stores, issue_info->stores_per_cycle });
-  cycles = std::max (cycles, { ops->loads + ops->stores,
-			       issue_info->loads_stores_per_cycle });
-  cycles = std::max (cycles, { ops->general_ops,
-			       issue_info->general_ops_per_cycle });
-  return cycles;
-}
-
-/* Subroutine of aarch64_adjust_body_cost for handling SVE.
-   Use ISSUE_INFO to work out how fast the SVE code can be issued and compare
-   it to the equivalent value for scalar code (SCALAR_CYCLES_PER_ITER).
-   If COULD_USE_ADVSIMD is true, also compare it to the issue rate of
-   Advanced SIMD code (ADVSIMD_CYCLES_PER_ITER).
-
-   COSTS is as for aarch64_adjust_body_cost.  ORIG_BODY_COST is the cost
-   originally passed to aarch64_adjust_body_cost and *BODY_COST is the current
-   value of the adjusted cost.  *SHOULD_DISPARAGE is true if we think the loop
-   body is too expensive.  */
-
-static fractional_cost
-aarch64_adjust_body_cost_sve (const aarch64_vector_costs *costs,
-			      const aarch64_vec_issue_info *issue_info,
-			      fractional_cost scalar_cycles_per_iter,
-			      fractional_cost advsimd_cycles_per_iter,
-			      bool could_use_advsimd,
-			      unsigned int orig_body_cost,
-			      unsigned int *body_cost,
-			      bool *should_disparage)
-{
-  /* Estimate the minimum number of cycles per iteration needed to issue
-     non-predicate operations.  */
-  fractional_cost sve_nonpred_issue_cycles_per_iter
-    = aarch64_estimate_min_cycles_per_iter (&costs->sve_ops,
-					    issue_info->sve);
-
-  /* Estimate the minimum number of cycles per iteration needed to rename
-     SVE instructions.
-
-     ??? For now this is done inline rather than via cost tables, since it
-     isn't clear how it should be parameterized for the general case.  */
-  fractional_cost sve_rename_cycles_per_iter = 0;
-  if (issue_info == &neoverse512tvb_vec_issue_info)
-    /* + 1 for an addition.  We've already counted a general op for each
-       store, so we don't need to account for stores separately.  The branch
-       reads no registers and so does not need to be counted either.
-
-       ??? This value is very much on the pessimistic side, but seems to work
-       pretty well in practice.  */
-    sve_rename_cycles_per_iter
-      = { costs->sve_ops.general_ops
-	  + costs->sve_ops.loads
-	  + costs->sve_ops.pred_ops + 1, 5 };
-
-  /* Combine the rename and non-predicate issue limits into a single value.  */
-  fractional_cost sve_nonpred_cycles_per_iter
-    = std::max (sve_nonpred_issue_cycles_per_iter, sve_rename_cycles_per_iter);
-
-  /* Separately estimate the minimum number of cycles per iteration needed
-     to issue the predicate operations.  */
-  fractional_cost sve_pred_issue_cycles_per_iter
-    = { costs->sve_ops.pred_ops, issue_info->sve->pred_ops_per_cycle };
-
-  /* Calculate the overall limit on the number of cycles per iteration.  */
-  fractional_cost sve_cycles_per_iter
-    = std::max (sve_nonpred_cycles_per_iter, sve_pred_issue_cycles_per_iter);
-
   if (dump_enabled_p ())
-    {
-      costs->sve_ops.dump ();
-      dump_printf_loc (MSG_NOTE, vect_location,
-		       "  estimated cycles per iteration = %f\n",
-		       sve_cycles_per_iter.as_double ());
-      if (costs->sve_ops.pred_ops)
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "    predicate issue = %f\n",
-			 sve_pred_issue_cycles_per_iter.as_double ());
-      if (costs->sve_ops.pred_ops || sve_rename_cycles_per_iter)
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "    non-predicate issue = %f\n",
-			 sve_nonpred_issue_cycles_per_iter.as_double ());
-      if (sve_rename_cycles_per_iter)
-	dump_printf_loc (MSG_NOTE, vect_location, "    rename = %f\n",
-			 sve_rename_cycles_per_iter.as_double ());
-    }
+    ops->dump ();
+
+  fractional_cost sve_pred_cycles_per_iter = ops->min_pred_cycles_per_iter ();
+  fractional_cost sve_cycles_per_iter = ops->min_cycles_per_iter ();
 
   /* If the scalar version of the loop could issue at least as
      quickly as the predicate parts of the SVE loop, make the SVE loop
@@ -15732,7 +15858,7 @@ aarch64_adjust_body_cost_sve (const aarch64_vector_costs *costs,
      costs would not model.  Adding this kind of cliffedge would be
      too drastic for scalar_cycles_per_iter vs. sve_cycles_per_iter;
      code in the caller handles that case in a more conservative way.  */
-  fractional_cost sve_estimate = sve_pred_issue_cycles_per_iter + 1;
+  fractional_cost sve_estimate = sve_pred_cycles_per_iter + 1;
   if (scalar_cycles_per_iter < sve_estimate)
     {
       unsigned int min_cost
@@ -15750,51 +15876,23 @@ aarch64_adjust_body_cost_sve (const aarch64_vector_costs *costs,
 	}
     }
 
-  /* If it appears that the Advanced SIMD version of a loop could issue
-     more quickly than the SVE one, increase the SVE cost in proportion
-     to the difference.  The intention is to make Advanced SIMD preferable
-     in cases where an Advanced SIMD version exists, without increasing
-     the costs so much that SVE won't be used at all.
-
-     The reasoning is similar to the scalar vs. predicate comparison above:
-     if the issue rate of the SVE code is limited by predicate operations
-     (i.e. if sve_pred_issue_cycles_per_iter > sve_nonpred_cycles_per_iter),
-     and if the Advanced SIMD code could issue within the limit imposed
-     by the predicate operations, the predicate operations are adding an
-     overhead that the original code didn't have and so we should prefer
-     the Advanced SIMD version.  However, if the predicate operations
-     do not dominate in this way, we should only increase the cost of
-     the SVE code if sve_cycles_per_iter is strictly greater than
-     advsimd_cycles_per_iter.  Given rounding effects, this should mean
-     that Advanced SIMD is either better or at least no worse.  */
-  if (sve_nonpred_cycles_per_iter >= sve_pred_issue_cycles_per_iter)
-    sve_estimate = sve_cycles_per_iter;
-  if (could_use_advsimd && advsimd_cycles_per_iter < sve_estimate)
-    {
-      /* This ensures that min_cost > orig_body_cost * 2.  */
-      unsigned int factor = fractional_cost::scale (1, sve_estimate,
-						    advsimd_cycles_per_iter);
-      unsigned int min_cost = orig_body_cost * factor + 1;
-      if (*body_cost < min_cost)
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_NOTE, vect_location,
-			     "Increasing body cost to %d because Advanced"
-			     " SIMD code could issue as quickly\n",
-			     min_cost);
-	  *body_cost = min_cost;
-	  *should_disparage = true;
-	}
-    }
-
   return sve_cycles_per_iter;
 }
 
-/* BODY_COST is the cost of a vector loop body recorded in COSTS.
-   Adjust the cost as necessary and return the new cost.  */
-static unsigned int
-aarch64_adjust_body_cost (aarch64_vector_costs *costs, unsigned int body_cost)
+/* BODY_COST is the cost of a vector loop body.  Adjust the cost as necessary
+   and return the new cost.  */
+unsigned int
+aarch64_vector_costs::
+adjust_body_cost (loop_vec_info loop_vinfo,
+		  const aarch64_vector_costs *scalar_costs,
+		  unsigned int body_cost)
 {
+  if (scalar_costs->m_ops.is_empty () || m_ops.is_empty ())
+    return body_cost;
+
+  const auto &scalar_ops = scalar_costs->m_ops[0];
+  const auto &vector_ops = m_ops[0];
+  unsigned int estimated_vf = vect_vf_for_cost (loop_vinfo);
   unsigned int orig_body_cost = body_cost;
   bool should_disparage = false;
 
@@ -15802,98 +15900,32 @@ aarch64_adjust_body_cost (aarch64_vector_costs *costs, unsigned int body_cost)
     dump_printf_loc (MSG_NOTE, vect_location,
 		     "Original vector body cost = %d\n", body_cost);
 
-  if (costs->unrolled_advsimd_stmts)
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location, "Number of insns in"
-			 " unrolled Advanced SIMD loop = %d\n",
-			 costs->unrolled_advsimd_stmts);
-
-      /* Apply the Advanced SIMD vs. SVE unrolling heuristic described above
-	 aarch64_vector_costs::unrolled_advsimd_niters.
-
-	 The balance here is tricky.  On the one hand, we can't be sure whether
-	 the code is vectorizable with Advanced SIMD or not.  However, even if
-	 it isn't vectorizable with Advanced SIMD, there's a possibility that
-	 the scalar code could also be unrolled.  Some of the code might then
-	 benefit from SLP, or from using LDP and STP.  We therefore apply
-	 the heuristic regardless of can_use_advsimd_p.  */
-      if (costs->unrolled_advsimd_stmts
-	  && (costs->unrolled_advsimd_stmts
-	      <= (unsigned int) param_max_completely_peeled_insns))
-	{
-	  unsigned int estimated_vq = aarch64_estimated_sve_vq ();
-	  unsigned int min_cost = (orig_body_cost * estimated_vq) + 1;
-	  if (body_cost < min_cost)
-	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_NOTE, vect_location,
-				 "Increasing body cost to %d to account for"
-				 " unrolling\n", min_cost);
-	      body_cost = min_cost;
-	      should_disparage = true;
-	    }
-	}
-    }
-
-  auto *issue_info = aarch64_tune_params.vec_costs->issue_info;
-  if (!issue_info)
-    return body_cost;
-
   fractional_cost scalar_cycles_per_iter
-    = aarch64_estimate_min_cycles_per_iter (&costs->scalar_ops,
-					    issue_info->scalar);
+    = scalar_ops.min_cycles_per_iter () * estimated_vf;
 
-  fractional_cost advsimd_cycles_per_iter
-    = aarch64_estimate_min_cycles_per_iter (&costs->advsimd_ops,
-					    issue_info->advsimd);
-
-  bool could_use_advsimd
-    = ((costs->vec_flags & VEC_ADVSIMD)
-       || (aarch64_autovec_preference != 2
-	   && (aarch64_tune_params.extra_tuning_flags
-	       & AARCH64_EXTRA_TUNE_MATCHED_VECTOR_THROUGHPUT)
-	   && !costs->saw_sve_only_op));
+  fractional_cost vector_cycles_per_iter = vector_ops.min_cycles_per_iter ();
 
   if (dump_enabled_p ())
     {
-      if (IN_RANGE (costs->num_vector_iterations, 0, 65536))
+      if (IN_RANGE (m_num_vector_iterations, 0, 65536))
 	dump_printf_loc (MSG_NOTE, vect_location,
 			 "Vector loop iterates at most %wd times\n",
-			 costs->num_vector_iterations);
+			 m_num_vector_iterations);
       dump_printf_loc (MSG_NOTE, vect_location, "Scalar issue estimate:\n");
-      costs->scalar_ops.dump ();
+      scalar_ops.dump ();
       dump_printf_loc (MSG_NOTE, vect_location,
-		       "  estimated cycles per iteration = %f\n",
-		       scalar_cycles_per_iter.as_double ());
-      if (could_use_advsimd)
-	{
-	  dump_printf_loc (MSG_NOTE, vect_location,
-			   "Advanced SIMD issue estimate:\n");
-	  costs->advsimd_ops.dump ();
-	  dump_printf_loc (MSG_NOTE, vect_location,
-			   "  estimated cycles per iteration = %f\n",
-			   advsimd_cycles_per_iter.as_double ());
-	}
-      else
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "Loop could not use Advanced SIMD\n");
+		       "  estimated cycles per vector iteration"
+		       " (for VF %d) = %f\n",
+		       estimated_vf, scalar_cycles_per_iter.as_double ());
     }
 
-  fractional_cost vector_cycles_per_iter = advsimd_cycles_per_iter;
-  unsigned int vector_reduction_latency = costs->advsimd_ops.reduction_latency;
-
-  if ((costs->vec_flags & VEC_ANY_SVE) && issue_info->sve)
+  if (vector_ops.sve_issue_info ())
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_NOTE, vect_location, "SVE issue estimate:\n");
-      vector_reduction_latency = costs->sve_ops.reduction_latency;
       vector_cycles_per_iter
-	= aarch64_adjust_body_cost_sve (costs, issue_info,
-					scalar_cycles_per_iter,
-					advsimd_cycles_per_iter,
-					could_use_advsimd, orig_body_cost,
-					&body_cost, &should_disparage);
+	= adjust_body_cost_sve (&vector_ops, scalar_cycles_per_iter,
+				orig_body_cost, &body_cost, &should_disparage);
 
       if (aarch64_tune_params.vec_costs == &neoverse512tvb_vector_cost)
 	{
@@ -15903,22 +15935,29 @@ aarch64_adjust_body_cost (aarch64_vector_costs *costs, unsigned int body_cost)
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location,
 			     "Neoverse V1 estimate:\n");
-	  aarch64_adjust_body_cost_sve (costs, &neoversev1_vec_issue_info,
-					scalar_cycles_per_iter * 2,
-					advsimd_cycles_per_iter * 2,
-					could_use_advsimd, orig_body_cost,
-					&body_cost, &should_disparage);
+	  auto vf_factor = m_ops[1].vf_factor ();
+	  adjust_body_cost_sve (&m_ops[1], scalar_cycles_per_iter * vf_factor,
+				orig_body_cost, &body_cost, &should_disparage);
+	}
+    }
+  else
+    {
+      if (dump_enabled_p ())
+	{
+	  dump_printf_loc (MSG_NOTE, vect_location,
+			   "Vector issue estimate:\n");
+	  vector_ops.dump ();
 	}
     }
 
   /* Decide whether to stick to latency-based costs or whether to try to
      take issue rates into account.  */
   unsigned int threshold = aarch64_loop_vect_issue_rate_niters;
-  if (costs->vec_flags & VEC_ANY_SVE)
+  if (m_vec_flags & VEC_ANY_SVE)
     threshold = CEIL (threshold, aarch64_estimated_sve_vq ());
 
-  if (costs->num_vector_iterations >= 1
-      && costs->num_vector_iterations < threshold)
+  if (m_num_vector_iterations >= 1
+      && m_num_vector_iterations < threshold)
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_NOTE, vect_location,
@@ -15951,8 +15990,8 @@ aarch64_adjust_body_cost (aarch64_vector_costs *costs, unsigned int body_cost)
      vector code is an improvement, even if adding the other (non-loop-carried)
      latencies tends to hide this saving.  We therefore reduce the cost of the
      vector loop body in proportion to the saving.  */
-  else if (costs->scalar_ops.reduction_latency > vector_reduction_latency
-	   && costs->scalar_ops.reduction_latency == scalar_cycles_per_iter
+  else if (scalar_ops.reduction_latency > vector_ops.reduction_latency
+	   && scalar_ops.reduction_latency == scalar_cycles_per_iter
 	   && scalar_cycles_per_iter > vector_cycles_per_iter
 	   && !should_disparage)
     {
@@ -15967,27 +16006,131 @@ aarch64_adjust_body_cost (aarch64_vector_costs *costs, unsigned int body_cost)
   return body_cost;
 }
 
-/* Implement TARGET_VECTORIZE_FINISH_COST.  */
-static void
-aarch64_finish_cost (void *data, unsigned *prologue_cost,
-		     unsigned *body_cost, unsigned *epilogue_cost)
+void
+aarch64_vector_costs::finish_cost (const vector_costs *uncast_scalar_costs)
 {
-  auto *costs = static_cast<aarch64_vector_costs *> (data);
-  *prologue_cost = costs->region[vect_prologue];
-  *body_cost     = costs->region[vect_body];
-  *epilogue_cost = costs->region[vect_epilogue];
-
-  if (costs->is_loop
-      && costs->vec_flags
+  auto *scalar_costs
+    = static_cast<const aarch64_vector_costs *> (uncast_scalar_costs);
+  loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
+  if (loop_vinfo
+      && m_vec_flags
       && aarch64_use_new_vector_costs_p ())
-    *body_cost = aarch64_adjust_body_cost (costs, *body_cost);
+    m_costs[vect_body] = adjust_body_cost (loop_vinfo, scalar_costs,
+					   m_costs[vect_body]);
+
+  vector_costs::finish_cost (scalar_costs);
 }
 
-/* Implement TARGET_VECTORIZE_DESTROY_COST_DATA.  */
-static void
-aarch64_destroy_cost_data (void *data)
+bool
+aarch64_vector_costs::
+better_main_loop_than_p (const vector_costs *uncast_other) const
 {
-  delete static_cast<aarch64_vector_costs *> (data);
+  auto other = static_cast<const aarch64_vector_costs *> (uncast_other);
+
+  auto this_loop_vinfo = as_a<loop_vec_info> (this->m_vinfo);
+  auto other_loop_vinfo = as_a<loop_vec_info> (other->m_vinfo);
+
+  if (dump_enabled_p ())
+    dump_printf_loc (MSG_NOTE, vect_location,
+		     "Comparing two main loops (%s at VF %d vs %s at VF %d)\n",
+		     GET_MODE_NAME (this_loop_vinfo->vector_mode),
+		     vect_vf_for_cost (this_loop_vinfo),
+		     GET_MODE_NAME (other_loop_vinfo->vector_mode),
+		     vect_vf_for_cost (other_loop_vinfo));
+
+  /* Apply the unrolling heuristic described above
+     m_unrolled_advsimd_niters.  */
+  if (bool (m_unrolled_advsimd_stmts)
+      != bool (other->m_unrolled_advsimd_stmts))
+    {
+      bool this_prefer_unrolled = this->prefer_unrolled_loop ();
+      bool other_prefer_unrolled = other->prefer_unrolled_loop ();
+      if (this_prefer_unrolled != other_prefer_unrolled)
+	{
+	  if (dump_enabled_p ())
+	    dump_printf_loc (MSG_NOTE, vect_location,
+			     "Preferring Advanced SIMD loop because"
+			     " it can be unrolled\n");
+	  return other_prefer_unrolled;
+	}
+    }
+
+  for (unsigned int i = 0; i < m_ops.length (); ++i)
+    {
+      if (dump_enabled_p ())
+	{
+	  if (i)
+	    dump_printf_loc (MSG_NOTE, vect_location,
+			     "Reconsidering with subtuning %d\n", i);
+	  dump_printf_loc (MSG_NOTE, vect_location,
+			   "Issue info for %s loop:\n",
+			   GET_MODE_NAME (this_loop_vinfo->vector_mode));
+	  this->m_ops[i].dump ();
+	  dump_printf_loc (MSG_NOTE, vect_location,
+			   "Issue info for %s loop:\n",
+			   GET_MODE_NAME (other_loop_vinfo->vector_mode));
+	  other->m_ops[i].dump ();
+	}
+
+      auto this_estimated_vf = (vect_vf_for_cost (this_loop_vinfo)
+				* this->m_ops[i].vf_factor ());
+      auto other_estimated_vf = (vect_vf_for_cost (other_loop_vinfo)
+				 * other->m_ops[i].vf_factor ());
+
+      /* If it appears that one loop could process the same amount of data
+	 in fewer cycles, prefer that loop over the other one.  */
+      fractional_cost this_cost
+	= this->m_ops[i].min_cycles_per_iter () * other_estimated_vf;
+      fractional_cost other_cost
+	= other->m_ops[i].min_cycles_per_iter () * this_estimated_vf;
+      if (dump_enabled_p ())
+	{
+	  dump_printf_loc (MSG_NOTE, vect_location,
+			   "Weighted cycles per iteration of %s loop ~= %f\n",
+			   GET_MODE_NAME (this_loop_vinfo->vector_mode),
+			   this_cost.as_double ());
+	  dump_printf_loc (MSG_NOTE, vect_location,
+			   "Weighted cycles per iteration of %s loop ~= %f\n",
+			   GET_MODE_NAME (other_loop_vinfo->vector_mode),
+			   other_cost.as_double ());
+	}
+      if (this_cost != other_cost)
+	{
+	  if (dump_enabled_p ())
+	    dump_printf_loc (MSG_NOTE, vect_location,
+			     "Preferring loop with lower cycles"
+			     " per iteration\n");
+	  return this_cost < other_cost;
+	}
+
+      /* If the issue rate of SVE code is limited by predicate operations
+	 (i.e. if sve_pred_cycles_per_iter > sve_nonpred_cycles_per_iter),
+	 and if Advanced SIMD code could issue within the limit imposed
+	 by the predicate operations, the predicate operations are adding an
+	 overhead that the original code didn't have and so we should prefer
+	 the Advanced SIMD version.  */
+      auto better_pred_limit_p = [](const aarch64_vec_op_count &a,
+				    const aarch64_vec_op_count &b) -> bool
+	{
+	  if (a.pred_ops == 0
+	      && (b.min_pred_cycles_per_iter ()
+		  > b.min_nonpred_cycles_per_iter ()))
+	    {
+	      if (dump_enabled_p ())
+		dump_printf_loc (MSG_NOTE, vect_location,
+				 "Preferring Advanced SIMD loop since"
+				 " SVE loop is predicate-limited\n");
+	      return true;
+	    }
+	  return false;
+	};
+      if (better_pred_limit_p (this->m_ops[i], other->m_ops[i]))
+	return true;
+      if (better_pred_limit_p (other->m_ops[i], this->m_ops[i]))
+	return false;
+    }
+
+  return vector_costs::better_main_loop_than_p (other);
 }
 
 static void initialize_aarch64_code_model (struct gcc_options *);
@@ -17818,7 +17961,18 @@ aarch64_process_target_attr (tree args)
       num_attrs++;
       if (!aarch64_process_one_target_attr (token))
 	{
-	  error ("pragma or attribute %<target(\"%s\")%> is not valid", token);
+	  /* Check if token is possibly an arch extension without
+	     leading '+'.  */
+	  uint64_t isa_temp = 0;
+	  auto with_plus = std::string ("+") + token;
+	  enum aarch64_parse_opt_result ext_res
+	    = aarch64_parse_extension (with_plus.c_str (), &isa_temp, nullptr);
+
+	  if (ext_res == AARCH64_PARSE_OK)
+	    error ("arch extension %<%s%> should be prefixed by %<+%>",
+		   token);
+	  else
+	    error ("pragma or attribute %<target(\"%s\")%> is not valid", token);
 	  return false;
 	}
 
@@ -20238,7 +20392,14 @@ aarch64_mov_operand_p (rtx x, machine_mode mode)
       return aarch64_simd_valid_immediate (x, NULL);
     }
 
+  /* Remove UNSPEC_SALT_ADDR before checking symbol reference.  */
   x = strip_salt (x);
+
+  /* GOT accesses are valid moves.  */
+  if (SYMBOL_REF_P (x)
+      && aarch64_classify_symbolic_expression (x) == SYMBOL_SMALL_GOT_4G)
+    return true;
+
   if (SYMBOL_REF_P (x) && mode == DImode && CONSTANT_ADDRESS_P (x))
     return true;
 
@@ -22326,7 +22487,7 @@ aarch64_expand_vec_perm_1 (rtx target, rtx op0, rtx op1, rtx sel)
 	}
       else
 	{
-	  pair = gen_reg_rtx (OImode);
+	  pair = gen_reg_rtx (V2x16QImode);
 	  emit_insn (gen_aarch64_combinev16qi (pair, op0, op1));
 	  emit_insn (gen_aarch64_qtbl2v16qi (target, pair, sel));
 	}
@@ -22936,11 +23097,10 @@ aarch64_evpc_ins (struct expand_vec_perm_d *d)
     }
   gcc_assert (extractindex < nelt);
 
-  emit_move_insn (d->target, insv);
   insn_code icode = code_for_aarch64_simd_vec_copy_lane (mode);
   expand_operand ops[5];
   create_output_operand (&ops[0], d->target, mode);
-  create_input_operand (&ops[1], d->target, mode);
+  create_input_operand (&ops[1], insv, mode);
   create_integer_operand (&ops[2], 1 << idx);
   create_input_operand (&ops[3], extractv, mode);
   create_integer_operand (&ops[4], extractindex);
@@ -23303,6 +23463,12 @@ aarch64_expand_sve_vcond (machine_mode data_mode, machine_mode cmp_mode,
 static bool
 aarch64_modes_tieable_p (machine_mode mode1, machine_mode mode2)
 {
+  if ((aarch64_advsimd_partial_struct_mode_p (mode1)
+       != aarch64_advsimd_partial_struct_mode_p (mode2))
+      && maybe_gt (GET_MODE_SIZE (mode1), 8)
+      && maybe_gt (GET_MODE_SIZE (mode2), 8))
+    return false;
+
   if (GET_MODE_CLASS (mode1) == GET_MODE_CLASS (mode2))
     return true;
 
@@ -24387,6 +24553,97 @@ aarch64_sched_adjust_priority (rtx_insn *insn, int priority)
   return priority;
 }
 
+/* Check if *MEM1 and *MEM2 are consecutive memory references and,
+   if they are, try to make them use constant offsets from the same base
+   register.  Return true on success.  When returning true, set *REVERSED
+   to true if *MEM1 comes after *MEM2, false if *MEM1 comes before *MEM2.  */
+static bool
+aarch64_check_consecutive_mems (rtx *mem1, rtx *mem2, bool *reversed)
+{
+  *reversed = false;
+  if (GET_RTX_CLASS (GET_CODE (XEXP (*mem1, 0))) == RTX_AUTOINC
+      || GET_RTX_CLASS (GET_CODE (XEXP (*mem2, 0))) == RTX_AUTOINC)
+    return false;
+
+  if (!MEM_SIZE_KNOWN_P (*mem1) || !MEM_SIZE_KNOWN_P (*mem2))
+    return false;
+
+  auto size1 = MEM_SIZE (*mem1);
+  auto size2 = MEM_SIZE (*mem2);
+
+  rtx base1, base2, offset1, offset2;
+  extract_base_offset_in_addr (*mem1, &base1, &offset1);
+  extract_base_offset_in_addr (*mem2, &base2, &offset2);
+
+  /* Make sure at least one memory is in base+offset form.  */
+  if (!(base1 && offset1) && !(base2 && offset2))
+    return false;
+
+  /* If both mems already use the same base register, just check the
+     offsets.  */
+  if (base1 && base2 && rtx_equal_p (base1, base2))
+    {
+      if (!offset1 || !offset2)
+	return false;
+
+      if (known_eq (UINTVAL (offset1) + size1, UINTVAL (offset2)))
+	return true;
+
+      if (known_eq (UINTVAL (offset2) + size2, UINTVAL (offset1)))
+	{
+	  *reversed = true;
+	  return true;
+	}
+
+      return false;
+    }
+
+  /* Otherwise, check whether the MEM_EXPRs and MEM_OFFSETs together
+     guarantee that the values are consecutive.  */
+  if (MEM_EXPR (*mem1)
+      && MEM_EXPR (*mem2)
+      && MEM_OFFSET_KNOWN_P (*mem1)
+      && MEM_OFFSET_KNOWN_P (*mem2))
+    {
+      poly_int64 expr_offset1;
+      poly_int64 expr_offset2;
+      tree expr_base1 = get_addr_base_and_unit_offset (MEM_EXPR (*mem1),
+						       &expr_offset1);
+      tree expr_base2 = get_addr_base_and_unit_offset (MEM_EXPR (*mem2),
+						       &expr_offset2);
+      if (!expr_base1
+	  || !expr_base2
+	  || !operand_equal_p (expr_base1, expr_base2, OEP_ADDRESS_OF))
+	return false;
+
+      expr_offset1 += MEM_OFFSET (*mem1);
+      expr_offset2 += MEM_OFFSET (*mem2);
+
+      if (known_eq (expr_offset1 + size1, expr_offset2))
+	;
+      else if (known_eq (expr_offset2 + size2, expr_offset1))
+	*reversed = true;
+      else
+	return false;
+
+      if (base2)
+	{
+	  rtx addr1 = plus_constant (Pmode, XEXP (*mem2, 0),
+				     expr_offset1 - expr_offset2);
+	  *mem1 = replace_equiv_address_nv (*mem1, addr1);
+	}
+      else
+	{
+	  rtx addr2 = plus_constant (Pmode, XEXP (*mem1, 0),
+				     expr_offset2 - expr_offset1);
+	  *mem2 = replace_equiv_address_nv (*mem2, addr2);
+	}
+      return true;
+    }
+
+  return false;
+}
+
 /* Given OPERANDS of consecutive load/store, check if we can merge
    them into ldp/stp.  LOAD is true if they are load instructions.
    MODE is the mode of memory operands.  */
@@ -24395,9 +24652,8 @@ bool
 aarch64_operands_ok_for_ldpstp (rtx *operands, bool load,
 				machine_mode mode)
 {
-  HOST_WIDE_INT offval_1, offval_2, msize;
   enum reg_class rclass_1, rclass_2;
-  rtx mem_1, mem_2, reg_1, reg_2, base_1, base_2, offset_1, offset_2;
+  rtx mem_1, mem_2, reg_1, reg_2;
 
   if (load)
     {
@@ -24408,6 +24664,8 @@ aarch64_operands_ok_for_ldpstp (rtx *operands, bool load,
       gcc_assert (REG_P (reg_1) && REG_P (reg_2));
       if (REGNO (reg_1) == REGNO (reg_2))
 	return false;
+      if (reg_overlap_mentioned_p (reg_1, mem_2))
+	return false;
     }
   else
     {
@@ -24431,40 +24689,13 @@ aarch64_operands_ok_for_ldpstp (rtx *operands, bool load,
     return false;
 
   /* Check if the addresses are in the form of [base+offset].  */
-  extract_base_offset_in_addr (mem_1, &base_1, &offset_1);
-  if (base_1 == NULL_RTX || offset_1 == NULL_RTX)
-    return false;
-  extract_base_offset_in_addr (mem_2, &base_2, &offset_2);
-  if (base_2 == NULL_RTX || offset_2 == NULL_RTX)
-    return false;
-
-  /* Check if the bases are same.  */
-  if (!rtx_equal_p (base_1, base_2))
+  bool reversed = false;
+  if (!aarch64_check_consecutive_mems (&mem_1, &mem_2, &reversed))
     return false;
 
   /* The operands must be of the same size.  */
   gcc_assert (known_eq (GET_MODE_SIZE (GET_MODE (mem_1)),
-			 GET_MODE_SIZE (GET_MODE (mem_2))));
-
-  offval_1 = INTVAL (offset_1);
-  offval_2 = INTVAL (offset_2);
-  /* We should only be trying this for fixed-sized modes.  There is no
-     SVE LDP/STP instruction.  */
-  msize = GET_MODE_SIZE (mode).to_constant ();
-  /* Check if the offsets are consecutive.  */
-  if (offval_1 != (offval_2 + msize) && offval_2 != (offval_1 + msize))
-    return false;
-
-  /* Check if the addresses are clobbered by load.  */
-  if (load)
-    {
-      if (reg_mentioned_p (reg_1, mem_1))
-	return false;
-
-      /* In increasing order, the last load can clobber the address.  */
-      if (offval_1 > offval_2 && reg_mentioned_p (reg_2, mem_2))
-	return false;
-    }
+			GET_MODE_SIZE (GET_MODE (mem_2))));
 
   /* One of the memory accesses must be a mempair operand.
      If it is not the first one, they need to be swapped by the
@@ -24495,27 +24726,13 @@ aarch64_operands_ok_for_ldpstp (rtx *operands, bool load,
 void
 aarch64_swap_ldrstr_operands (rtx* operands, bool load)
 {
-  rtx mem_1, mem_2, base_1, base_2, offset_1, offset_2;
-  HOST_WIDE_INT offval_1, offval_2;
-
-  if (load)
-    {
-      mem_1 = operands[1];
-      mem_2 = operands[3];
-    }
-  else
-    {
-      mem_1 = operands[0];
-      mem_2 = operands[2];
-    }
-
-  extract_base_offset_in_addr (mem_1, &base_1, &offset_1);
-  extract_base_offset_in_addr (mem_2, &base_2, &offset_2);
-
-  offval_1 = INTVAL (offset_1);
-  offval_2 = INTVAL (offset_2);
+  int mem_op = load ? 1 : 0;
+  bool reversed = false;
+  if (!aarch64_check_consecutive_mems (operands + mem_op,
+				       operands + mem_op + 2, &reversed))
+    gcc_unreachable ();
 
-  if (offval_1 > offval_2)
+  if (reversed)
     {
       /* Irrespective of whether this is a load or a store,
 	 we do the same swap.  */
@@ -25158,6 +25375,10 @@ aarch64_can_change_mode_class (machine_mode from,
   bool from_pred_p = (from_flags & VEC_SVE_PRED);
   bool to_pred_p = (to_flags & VEC_SVE_PRED);
 
+  bool from_full_advsimd_struct_p = (from_flags == (VEC_ADVSIMD | VEC_STRUCT));
+  bool to_partial_advsimd_struct_p = (to_flags == (VEC_ADVSIMD | VEC_STRUCT
+						   | VEC_PARTIAL));
+
   /* Don't allow changes between predicate modes and other modes.
      Only predicate registers can hold predicate modes and only
      non-predicate registers can hold non-predicate modes, so any
@@ -25178,6 +25399,11 @@ aarch64_can_change_mode_class (machine_mode from,
 	  || GET_MODE_UNIT_SIZE (from) != GET_MODE_UNIT_SIZE (to)))
     return false;
 
+  /* Don't allow changes between partial and full Advanced SIMD structure
+     modes.  */
+  if (from_full_advsimd_struct_p && to_partial_advsimd_struct_p)
+    return false;
+
   if (maybe_ne (BITS_PER_SVE_VECTOR, 128u))
     {
       /* Don't allow changes between SVE modes and other modes that might
@@ -26286,17 +26512,8 @@ aarch64_libgcc_floating_mode_supported_p
 #undef TARGET_ARRAY_MODE_SUPPORTED_P
 #define TARGET_ARRAY_MODE_SUPPORTED_P aarch64_array_mode_supported_p
 
-#undef TARGET_VECTORIZE_INIT_COST
-#define TARGET_VECTORIZE_INIT_COST aarch64_init_cost
-
-#undef TARGET_VECTORIZE_ADD_STMT_COST
-#define TARGET_VECTORIZE_ADD_STMT_COST aarch64_add_stmt_cost
-
-#undef TARGET_VECTORIZE_FINISH_COST
-#define TARGET_VECTORIZE_FINISH_COST aarch64_finish_cost
-
-#undef TARGET_VECTORIZE_DESTROY_COST_DATA
-#define TARGET_VECTORIZE_DESTROY_COST_DATA aarch64_destroy_cost_data
+#undef TARGET_VECTORIZE_CREATE_COSTS
+#define TARGET_VECTORIZE_CREATE_COSTS aarch64_vectorize_create_costs
 
 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md
index 1a39470..5297b2d 100644
--- a/gcc/config/aarch64/aarch64.md
+++ b/gcc/config/aarch64/aarch64.md
@@ -879,8 +879,7 @@
   {
     const char *ret = NULL;
     if (aarch64_return_address_signing_enabled ()
-	&& (TARGET_PAUTH)
-	&& !crtl->calls_eh_return)
+	&& (TARGET_PAUTH))
       {
 	if (aarch64_ra_sign_key == AARCH64_KEY_B)
 	  ret = "retab";
@@ -1263,8 +1262,8 @@
 )
 
 (define_insn_and_split "*movsi_aarch64"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r, w,r,w, w")
-	(match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
+  [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r,  r, w,r,w, w")
+	(match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usw,Usa,Ush,rZ,w,w,Ds"))]
   "(register_operand (operands[0], SImode)
     || aarch64_reg_or_zero (operands[1], SImode))"
   "@
@@ -1278,6 +1277,7 @@
    ldr\\t%s0, %1
    str\\t%w1, %0
    str\\t%s1, %0
+   adrp\\t%x0, %A1\;ldr\\t%w0, [%x0, %L1]
    adr\\t%x0, %c1
    adrp\\t%x0, %A1
    fmov\\t%s0, %w1
@@ -1293,13 +1293,15 @@
     }"
   ;; The "mov_imm" type for CNT is just a placeholder.
   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
-		    load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
-   (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
+		    load_4,store_4,store_4,load_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
+   (set_attr "arch"   "*,*,*,*,*,sve,*,fp,*,fp,*,*,*,fp,fp,fp,simd")
+   (set_attr "length" "4,4,4,4,*,  4,4, 4,4, 4,8,4,4, 4, 4, 4,   4")
+]
 )
 
 (define_insn_and_split "*movdi_aarch64"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,  r,  r, w,r,w, w")
-	(match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,   r,  r,  r, w,r,w, w")
+	(match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usw,Usa,Ush,rZ,w,w,Dd"))]
   "(register_operand (operands[0], DImode)
     || aarch64_reg_or_zero (operands[1], DImode))"
   "@
@@ -1314,13 +1316,14 @@
    ldr\\t%d0, %1
    str\\t%x1, %0
    str\\t%d1, %0
+   * return TARGET_ILP32 ? \"adrp\\t%0, %A1\;ldr\\t%w0, [%0, %L1]\" : \"adrp\\t%0, %A1\;ldr\\t%0, [%0, %L1]\";
    adr\\t%x0, %c1
    adrp\\t%x0, %A1
    fmov\\t%d0, %x1
    fmov\\t%x0, %d1
    fmov\\t%d0, %d1
    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
-   "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
+   "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode)
     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
    [(const_int 0)]
    "{
@@ -1329,9 +1332,10 @@
     }"
   ;; The "mov_imm" type for CNTD is just a placeholder.
   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
-		     load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
-		     neon_move")
-   (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
+		     load_8,load_8,store_8,store_8,load_8,adr,adr,f_mcr,f_mrc,
+		     fmov,neon_move")
+   (set_attr "arch"   "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,*,fp,fp,fp,simd")
+   (set_attr "length" "4,4,4,4,4,*,  4,4, 4,4, 4,8,4,4, 4, 4, 4,   4")]
 )
 
 (define_insn "insv_imm<mode>"
@@ -6471,7 +6475,7 @@
 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
 ;; which implement the IEEE fmax ()/fmin () functions.
-(define_insn "<maxmin_uns><mode>3"
+(define_insn "<fmaxmin><mode>3"
   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
 	(unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
 		     (match_operand:GPF_F16 2 "register_operand" "w")]
@@ -6707,29 +6711,6 @@
   [(set_attr "type" "alu_imm")]
 )
 
-(define_insn "ldr_got_small_<mode>"
-  [(set (match_operand:PTR 0 "register_operand" "=r")
-	(unspec:PTR [(mem:PTR (lo_sum:PTR
-			      (match_operand:PTR 1 "register_operand" "r")
-			      (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
-		    UNSPEC_GOTSMALLPIC))]
-  ""
-  "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
-  [(set_attr "type" "load_<ldst_sz>")]
-)
-
-(define_insn "ldr_got_small_sidi"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-	(zero_extend:DI
-	 (unspec:SI [(mem:SI (lo_sum:DI
-			     (match_operand:DI 1 "register_operand" "r")
-			     (match_operand:DI 2 "aarch64_valid_symref" "S")))]
-		    UNSPEC_GOTSMALLPIC)))]
-  "TARGET_ILP32"
-  "ldr\\t%w0, [%1, #:got_lo12:%c2]"
-  [(set_attr "type" "load_4")]
-)
-
 (define_insn "ldr_got_small_28k_<mode>"
   [(set (match_operand:PTR 0 "register_operand" "=r")
 	(unspec:PTR [(mem:PTR (lo_sum:PTR
diff --git a/gcc/config/aarch64/arm_neon.h b/gcc/config/aarch64/arm_neon.h
index 1768c6b..dd6877b 100644
--- a/gcc/config/aarch64/arm_neon.h
+++ b/gcc/config/aarch64/arm_neon.h
@@ -30,6 +30,8 @@
 #pragma GCC push_options
 #pragma GCC target ("+nothing+simd")
 
+#pragma GCC aarch64 "arm_neon.h"
+
 #include <stdint.h>
 
 #define __AARCH64_UINT64_C(__C) ((uint64_t) __C)
@@ -76,456 +78,6 @@ typedef double float64_t;
 typedef __Bfloat16x4_t bfloat16x4_t;
 typedef __Bfloat16x8_t bfloat16x8_t;
 
-typedef struct bfloat16x4x2_t
-{
-  bfloat16x4_t val[2];
-} bfloat16x4x2_t;
-
-typedef struct bfloat16x8x2_t
-{
-  bfloat16x8_t val[2];
-} bfloat16x8x2_t;
-
-typedef struct bfloat16x4x3_t
-{
-  bfloat16x4_t val[3];
-} bfloat16x4x3_t;
-
-typedef struct bfloat16x8x3_t
-{
-  bfloat16x8_t val[3];
-} bfloat16x8x3_t;
-
-typedef struct bfloat16x4x4_t
-{
-  bfloat16x4_t val[4];
-} bfloat16x4x4_t;
-
-typedef struct bfloat16x8x4_t
-{
-  bfloat16x8_t val[4];
-} bfloat16x8x4_t;
-
-typedef struct int8x8x2_t
-{
-  int8x8_t val[2];
-} int8x8x2_t;
-
-typedef struct int8x16x2_t
-{
-  int8x16_t val[2];
-} int8x16x2_t;
-
-typedef struct int16x4x2_t
-{
-  int16x4_t val[2];
-} int16x4x2_t;
-
-typedef struct int16x8x2_t
-{
-  int16x8_t val[2];
-} int16x8x2_t;
-
-typedef struct int32x2x2_t
-{
-  int32x2_t val[2];
-} int32x2x2_t;
-
-typedef struct int32x4x2_t
-{
-  int32x4_t val[2];
-} int32x4x2_t;
-
-typedef struct int64x1x2_t
-{
-  int64x1_t val[2];
-} int64x1x2_t;
-
-typedef struct int64x2x2_t
-{
-  int64x2_t val[2];
-} int64x2x2_t;
-
-typedef struct uint8x8x2_t
-{
-  uint8x8_t val[2];
-} uint8x8x2_t;
-
-typedef struct uint8x16x2_t
-{
-  uint8x16_t val[2];
-} uint8x16x2_t;
-
-typedef struct uint16x4x2_t
-{
-  uint16x4_t val[2];
-} uint16x4x2_t;
-
-typedef struct uint16x8x2_t
-{
-  uint16x8_t val[2];
-} uint16x8x2_t;
-
-typedef struct uint32x2x2_t
-{
-  uint32x2_t val[2];
-} uint32x2x2_t;
-
-typedef struct uint32x4x2_t
-{
-  uint32x4_t val[2];
-} uint32x4x2_t;
-
-typedef struct uint64x1x2_t
-{
-  uint64x1_t val[2];
-} uint64x1x2_t;
-
-typedef struct uint64x2x2_t
-{
-  uint64x2_t val[2];
-} uint64x2x2_t;
-
-typedef struct float16x4x2_t
-{
-  float16x4_t val[2];
-} float16x4x2_t;
-
-typedef struct float16x8x2_t
-{
-  float16x8_t val[2];
-} float16x8x2_t;
-
-typedef struct float32x2x2_t
-{
-  float32x2_t val[2];
-} float32x2x2_t;
-
-typedef struct float32x4x2_t
-{
-  float32x4_t val[2];
-} float32x4x2_t;
-
-typedef struct float64x2x2_t
-{
-  float64x2_t val[2];
-} float64x2x2_t;
-
-typedef struct float64x1x2_t
-{
-  float64x1_t val[2];
-} float64x1x2_t;
-
-typedef struct poly8x8x2_t
-{
-  poly8x8_t val[2];
-} poly8x8x2_t;
-
-typedef struct poly8x16x2_t
-{
-  poly8x16_t val[2];
-} poly8x16x2_t;
-
-typedef struct poly16x4x2_t
-{
-  poly16x4_t val[2];
-} poly16x4x2_t;
-
-typedef struct poly16x8x2_t
-{
-  poly16x8_t val[2];
-} poly16x8x2_t;
-
-typedef struct poly64x1x2_t
-{
-  poly64x1_t val[2];
-} poly64x1x2_t;
-
-typedef struct poly64x1x3_t
-{
-  poly64x1_t val[3];
-} poly64x1x3_t;
-
-typedef struct poly64x1x4_t
-{
-  poly64x1_t val[4];
-} poly64x1x4_t;
-
-typedef struct poly64x2x2_t
-{
-  poly64x2_t val[2];
-} poly64x2x2_t;
-
-typedef struct poly64x2x3_t
-{
-  poly64x2_t val[3];
-} poly64x2x3_t;
-
-typedef struct poly64x2x4_t
-{
-  poly64x2_t val[4];
-} poly64x2x4_t;
-
-typedef struct int8x8x3_t
-{
-  int8x8_t val[3];
-} int8x8x3_t;
-
-typedef struct int8x16x3_t
-{
-  int8x16_t val[3];
-} int8x16x3_t;
-
-typedef struct int16x4x3_t
-{
-  int16x4_t val[3];
-} int16x4x3_t;
-
-typedef struct int16x8x3_t
-{
-  int16x8_t val[3];
-} int16x8x3_t;
-
-typedef struct int32x2x3_t
-{
-  int32x2_t val[3];
-} int32x2x3_t;
-
-typedef struct int32x4x3_t
-{
-  int32x4_t val[3];
-} int32x4x3_t;
-
-typedef struct int64x1x3_t
-{
-  int64x1_t val[3];
-} int64x1x3_t;
-
-typedef struct int64x2x3_t
-{
-  int64x2_t val[3];
-} int64x2x3_t;
-
-typedef struct uint8x8x3_t
-{
-  uint8x8_t val[3];
-} uint8x8x3_t;
-
-typedef struct uint8x16x3_t
-{
-  uint8x16_t val[3];
-} uint8x16x3_t;
-
-typedef struct uint16x4x3_t
-{
-  uint16x4_t val[3];
-} uint16x4x3_t;
-
-typedef struct uint16x8x3_t
-{
-  uint16x8_t val[3];
-} uint16x8x3_t;
-
-typedef struct uint32x2x3_t
-{
-  uint32x2_t val[3];
-} uint32x2x3_t;
-
-typedef struct uint32x4x3_t
-{
-  uint32x4_t val[3];
-} uint32x4x3_t;
-
-typedef struct uint64x1x3_t
-{
-  uint64x1_t val[3];
-} uint64x1x3_t;
-
-typedef struct uint64x2x3_t
-{
-  uint64x2_t val[3];
-} uint64x2x3_t;
-
-typedef struct float16x4x3_t
-{
-  float16x4_t val[3];
-} float16x4x3_t;
-
-typedef struct float16x8x3_t
-{
-  float16x8_t val[3];
-} float16x8x3_t;
-
-typedef struct float32x2x3_t
-{
-  float32x2_t val[3];
-} float32x2x3_t;
-
-typedef struct float32x4x3_t
-{
-  float32x4_t val[3];
-} float32x4x3_t;
-
-typedef struct float64x2x3_t
-{
-  float64x2_t val[3];
-} float64x2x3_t;
-
-typedef struct float64x1x3_t
-{
-  float64x1_t val[3];
-} float64x1x3_t;
-
-typedef struct poly8x8x3_t
-{
-  poly8x8_t val[3];
-} poly8x8x3_t;
-
-typedef struct poly8x16x3_t
-{
-  poly8x16_t val[3];
-} poly8x16x3_t;
-
-typedef struct poly16x4x3_t
-{
-  poly16x4_t val[3];
-} poly16x4x3_t;
-
-typedef struct poly16x8x3_t
-{
-  poly16x8_t val[3];
-} poly16x8x3_t;
-
-typedef struct int8x8x4_t
-{
-  int8x8_t val[4];
-} int8x8x4_t;
-
-typedef struct int8x16x4_t
-{
-  int8x16_t val[4];
-} int8x16x4_t;
-
-typedef struct int16x4x4_t
-{
-  int16x4_t val[4];
-} int16x4x4_t;
-
-typedef struct int16x8x4_t
-{
-  int16x8_t val[4];
-} int16x8x4_t;
-
-typedef struct int32x2x4_t
-{
-  int32x2_t val[4];
-} int32x2x4_t;
-
-typedef struct int32x4x4_t
-{
-  int32x4_t val[4];
-} int32x4x4_t;
-
-typedef struct int64x1x4_t
-{
-  int64x1_t val[4];
-} int64x1x4_t;
-
-typedef struct int64x2x4_t
-{
-  int64x2_t val[4];
-} int64x2x4_t;
-
-typedef struct uint8x8x4_t
-{
-  uint8x8_t val[4];
-} uint8x8x4_t;
-
-typedef struct uint8x16x4_t
-{
-  uint8x16_t val[4];
-} uint8x16x4_t;
-
-typedef struct uint16x4x4_t
-{
-  uint16x4_t val[4];
-} uint16x4x4_t;
-
-typedef struct uint16x8x4_t
-{
-  uint16x8_t val[4];
-} uint16x8x4_t;
-
-typedef struct uint32x2x4_t
-{
-  uint32x2_t val[4];
-} uint32x2x4_t;
-
-typedef struct uint32x4x4_t
-{
-  uint32x4_t val[4];
-} uint32x4x4_t;
-
-typedef struct uint64x1x4_t
-{
-  uint64x1_t val[4];
-} uint64x1x4_t;
-
-typedef struct uint64x2x4_t
-{
-  uint64x2_t val[4];
-} uint64x2x4_t;
-
-typedef struct float16x4x4_t
-{
-  float16x4_t val[4];
-} float16x4x4_t;
-
-typedef struct float16x8x4_t
-{
-  float16x8_t val[4];
-} float16x8x4_t;
-
-typedef struct float32x2x4_t
-{
-  float32x2_t val[4];
-} float32x2x4_t;
-
-typedef struct float32x4x4_t
-{
-  float32x4_t val[4];
-} float32x4x4_t;
-
-typedef struct float64x2x4_t
-{
-  float64x2_t val[4];
-} float64x2x4_t;
-
-typedef struct float64x1x4_t
-{
-  float64x1_t val[4];
-} float64x1x4_t;
-
-typedef struct poly8x8x4_t
-{
-  poly8x8_t val[4];
-} poly8x8x4_t;
-
-typedef struct poly8x16x4_t
-{
-  poly8x16_t val[4];
-} poly8x16x4_t;
-
-typedef struct poly16x4x4_t
-{
-  poly16x4_t val[4];
-} poly16x4x4_t;
-
-typedef struct poly16x8x4_t
-{
-  poly16x8_t val[4];
-} poly16x8x4_t;
-
 /* __aarch64_vdup_lane internal macros.  */
 #define __aarch64_vdup_lane_any(__size, __q, __a, __b) \
   vdup##__q##_n_##__size (__aarch64_vget_lane_any (__a, __b))
@@ -825,546 +377,504 @@ __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_s8 (int8x8_t __a, int8x8_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_saddlv8qi (__a, __b);
+  return __builtin_aarch64_saddlv8qi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_s16 (int16x4_t __a, int16x4_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_saddlv4hi (__a, __b);
+  return __builtin_aarch64_saddlv4hi (__a, __b);
 }
 
 __extension__ extern __inline int64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_s32 (int32x2_t __a, int32x2_t __b)
 {
-  return (int64x2_t) __builtin_aarch64_saddlv2si (__a, __b);
+  return __builtin_aarch64_saddlv2si (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_u8 (uint8x8_t __a, uint8x8_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_uaddlv8qi ((int8x8_t) __a,
-						   (int8x8_t) __b);
+  return __builtin_aarch64_uaddlv8qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_u16 (uint16x4_t __a, uint16x4_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_uaddlv4hi ((int16x4_t) __a,
-						   (int16x4_t) __b);
+  return __builtin_aarch64_uaddlv4hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_u32 (uint32x2_t __a, uint32x2_t __b)
 {
-  return (uint64x2_t) __builtin_aarch64_uaddlv2si ((int32x2_t) __a,
-						   (int32x2_t) __b);
+  return __builtin_aarch64_uaddlv2si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_high_s8 (int8x16_t __a, int8x16_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_saddl2v16qi (__a, __b);
+  return __builtin_aarch64_saddl2v16qi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_high_s16 (int16x8_t __a, int16x8_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_saddl2v8hi (__a, __b);
+  return __builtin_aarch64_saddl2v8hi (__a, __b);
 }
 
 __extension__ extern __inline int64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_high_s32 (int32x4_t __a, int32x4_t __b)
 {
-  return (int64x2_t) __builtin_aarch64_saddl2v4si (__a, __b);
+  return __builtin_aarch64_saddl2v4si (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_high_u8 (uint8x16_t __a, uint8x16_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_uaddl2v16qi ((int8x16_t) __a,
-						     (int8x16_t) __b);
+  return __builtin_aarch64_uaddl2v16qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_high_u16 (uint16x8_t __a, uint16x8_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_uaddl2v8hi ((int16x8_t) __a,
-						    (int16x8_t) __b);
+  return __builtin_aarch64_uaddl2v8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddl_high_u32 (uint32x4_t __a, uint32x4_t __b)
 {
-  return (uint64x2_t) __builtin_aarch64_uaddl2v4si ((int32x4_t) __a,
-						    (int32x4_t) __b);
+  return __builtin_aarch64_uaddl2v4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_s8 (int16x8_t __a, int8x8_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_saddwv8qi (__a, __b);
+  return __builtin_aarch64_saddwv8qi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_s16 (int32x4_t __a, int16x4_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_saddwv4hi (__a, __b);
+  return __builtin_aarch64_saddwv4hi (__a, __b);
 }
 
 __extension__ extern __inline int64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_s32 (int64x2_t __a, int32x2_t __b)
 {
-  return (int64x2_t) __builtin_aarch64_saddwv2si (__a, __b);
+  return __builtin_aarch64_saddwv2si (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_u8 (uint16x8_t __a, uint8x8_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_uaddwv8qi ((int16x8_t) __a,
-						   (int8x8_t) __b);
+  return __builtin_aarch64_uaddwv8qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_u16 (uint32x4_t __a, uint16x4_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_uaddwv4hi ((int32x4_t) __a,
-						   (int16x4_t) __b);
+  return __builtin_aarch64_uaddwv4hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_u32 (uint64x2_t __a, uint32x2_t __b)
 {
-  return (uint64x2_t) __builtin_aarch64_uaddwv2si ((int64x2_t) __a,
-						   (int32x2_t) __b);
+  return __builtin_aarch64_uaddwv2si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_high_s8 (int16x8_t __a, int8x16_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_saddw2v16qi (__a, __b);
+  return __builtin_aarch64_saddw2v16qi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_high_s16 (int32x4_t __a, int16x8_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_saddw2v8hi (__a, __b);
+  return __builtin_aarch64_saddw2v8hi (__a, __b);
 }
 
 __extension__ extern __inline int64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_high_s32 (int64x2_t __a, int32x4_t __b)
 {
-  return (int64x2_t) __builtin_aarch64_saddw2v4si (__a, __b);
+  return __builtin_aarch64_saddw2v4si (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_high_u8 (uint16x8_t __a, uint8x16_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_uaddw2v16qi ((int16x8_t) __a,
-						     (int8x16_t) __b);
+  return __builtin_aarch64_uaddw2v16qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_high_u16 (uint32x4_t __a, uint16x8_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_uaddw2v8hi ((int32x4_t) __a,
-						    (int16x8_t) __b);
+  return __builtin_aarch64_uaddw2v8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddw_high_u32 (uint64x2_t __a, uint32x4_t __b)
 {
-  return (uint64x2_t) __builtin_aarch64_uaddw2v4si ((int64x2_t) __a,
-						    (int32x4_t) __b);
+  return __builtin_aarch64_uaddw2v4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhadd_s8 (int8x8_t __a, int8x8_t __b)
 {
-  return (int8x8_t) __builtin_aarch64_shaddv8qi (__a, __b);
+  return __builtin_aarch64_shaddv8qi (__a, __b);
 }
 
 __extension__ extern __inline int16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhadd_s16 (int16x4_t __a, int16x4_t __b)
 {
-  return (int16x4_t) __builtin_aarch64_shaddv4hi (__a, __b);
+  return __builtin_aarch64_shaddv4hi (__a, __b);
 }
 
 __extension__ extern __inline int32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhadd_s32 (int32x2_t __a, int32x2_t __b)
 {
-  return (int32x2_t) __builtin_aarch64_shaddv2si (__a, __b);
+  return __builtin_aarch64_shaddv2si (__a, __b);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhadd_u8 (uint8x8_t __a, uint8x8_t __b)
 {
-  return (uint8x8_t) __builtin_aarch64_uhaddv8qi ((int8x8_t) __a,
-						  (int8x8_t) __b);
+  return __builtin_aarch64_uhaddv8qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhadd_u16 (uint16x4_t __a, uint16x4_t __b)
 {
-  return (uint16x4_t) __builtin_aarch64_uhaddv4hi ((int16x4_t) __a,
-						   (int16x4_t) __b);
+  return __builtin_aarch64_uhaddv4hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhadd_u32 (uint32x2_t __a, uint32x2_t __b)
 {
-  return (uint32x2_t) __builtin_aarch64_uhaddv2si ((int32x2_t) __a,
-						   (int32x2_t) __b);
+  return __builtin_aarch64_uhaddv2si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhaddq_s8 (int8x16_t __a, int8x16_t __b)
 {
-  return (int8x16_t) __builtin_aarch64_shaddv16qi (__a, __b);
+  return __builtin_aarch64_shaddv16qi (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhaddq_s16 (int16x8_t __a, int16x8_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_shaddv8hi (__a, __b);
+  return __builtin_aarch64_shaddv8hi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhaddq_s32 (int32x4_t __a, int32x4_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_shaddv4si (__a, __b);
+  return __builtin_aarch64_shaddv4si (__a, __b);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhaddq_u8 (uint8x16_t __a, uint8x16_t __b)
 {
-  return (uint8x16_t) __builtin_aarch64_uhaddv16qi ((int8x16_t) __a,
-						    (int8x16_t) __b);
+  return __builtin_aarch64_uhaddv16qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhaddq_u16 (uint16x8_t __a, uint16x8_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_uhaddv8hi ((int16x8_t) __a,
-						   (int16x8_t) __b);
+  return __builtin_aarch64_uhaddv8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhaddq_u32 (uint32x4_t __a, uint32x4_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_uhaddv4si ((int32x4_t) __a,
-						   (int32x4_t) __b);
+  return __builtin_aarch64_uhaddv4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhadd_s8 (int8x8_t __a, int8x8_t __b)
 {
-  return (int8x8_t) __builtin_aarch64_srhaddv8qi (__a, __b);
+  return __builtin_aarch64_srhaddv8qi (__a, __b);
 }
 
 __extension__ extern __inline int16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhadd_s16 (int16x4_t __a, int16x4_t __b)
 {
-  return (int16x4_t) __builtin_aarch64_srhaddv4hi (__a, __b);
+  return __builtin_aarch64_srhaddv4hi (__a, __b);
 }
 
 __extension__ extern __inline int32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhadd_s32 (int32x2_t __a, int32x2_t __b)
 {
-  return (int32x2_t) __builtin_aarch64_srhaddv2si (__a, __b);
+  return __builtin_aarch64_srhaddv2si (__a, __b);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhadd_u8 (uint8x8_t __a, uint8x8_t __b)
 {
-  return (uint8x8_t) __builtin_aarch64_urhaddv8qi ((int8x8_t) __a,
-						   (int8x8_t) __b);
+  return __builtin_aarch64_urhaddv8qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhadd_u16 (uint16x4_t __a, uint16x4_t __b)
 {
-  return (uint16x4_t) __builtin_aarch64_urhaddv4hi ((int16x4_t) __a,
-						    (int16x4_t) __b);
+  return __builtin_aarch64_urhaddv4hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhadd_u32 (uint32x2_t __a, uint32x2_t __b)
 {
-  return (uint32x2_t) __builtin_aarch64_urhaddv2si ((int32x2_t) __a,
-						    (int32x2_t) __b);
+  return __builtin_aarch64_urhaddv2si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhaddq_s8 (int8x16_t __a, int8x16_t __b)
 {
-  return (int8x16_t) __builtin_aarch64_srhaddv16qi (__a, __b);
+  return __builtin_aarch64_srhaddv16qi (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhaddq_s16 (int16x8_t __a, int16x8_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_srhaddv8hi (__a, __b);
+  return __builtin_aarch64_srhaddv8hi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhaddq_s32 (int32x4_t __a, int32x4_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_srhaddv4si (__a, __b);
+  return __builtin_aarch64_srhaddv4si (__a, __b);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhaddq_u8 (uint8x16_t __a, uint8x16_t __b)
 {
-  return (uint8x16_t) __builtin_aarch64_urhaddv16qi ((int8x16_t) __a,
-						     (int8x16_t) __b);
+  return __builtin_aarch64_urhaddv16qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhaddq_u16 (uint16x8_t __a, uint16x8_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_urhaddv8hi ((int16x8_t) __a,
-						    (int16x8_t) __b);
+  return __builtin_aarch64_urhaddv8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrhaddq_u32 (uint32x4_t __a, uint32x4_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_urhaddv4si ((int32x4_t) __a,
-						    (int32x4_t) __b);
+  return __builtin_aarch64_urhaddv4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_s16 (int16x8_t __a, int16x8_t __b)
 {
-  return (int8x8_t) __builtin_aarch64_addhnv8hi (__a, __b);
+  return __builtin_aarch64_addhnv8hi (__a, __b);
 }
 
 __extension__ extern __inline int16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_s32 (int32x4_t __a, int32x4_t __b)
 {
-  return (int16x4_t) __builtin_aarch64_addhnv4si (__a, __b);
+  return __builtin_aarch64_addhnv4si (__a, __b);
 }
 
 __extension__ extern __inline int32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_s64 (int64x2_t __a, int64x2_t __b)
 {
-  return (int32x2_t) __builtin_aarch64_addhnv2di (__a, __b);
+  return __builtin_aarch64_addhnv2di (__a, __b);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_u16 (uint16x8_t __a, uint16x8_t __b)
 {
-  return (uint8x8_t) __builtin_aarch64_addhnv8hi ((int16x8_t) __a,
-						  (int16x8_t) __b);
+  return __builtin_aarch64_addhnv8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_u32 (uint32x4_t __a, uint32x4_t __b)
 {
-  return (uint16x4_t) __builtin_aarch64_addhnv4si ((int32x4_t) __a,
-						   (int32x4_t) __b);
+  return __builtin_aarch64_addhnv4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_u64 (uint64x2_t __a, uint64x2_t __b)
 {
-  return (uint32x2_t) __builtin_aarch64_addhnv2di ((int64x2_t) __a,
-						   (int64x2_t) __b);
+  return __builtin_aarch64_addhnv2di_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_s16 (int16x8_t __a, int16x8_t __b)
 {
-  return (int8x8_t) __builtin_aarch64_raddhnv8hi (__a, __b);
+  return __builtin_aarch64_raddhnv8hi (__a, __b);
 }
 
 __extension__ extern __inline int16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_s32 (int32x4_t __a, int32x4_t __b)
 {
-  return (int16x4_t) __builtin_aarch64_raddhnv4si (__a, __b);
+  return __builtin_aarch64_raddhnv4si (__a, __b);
 }
 
 __extension__ extern __inline int32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_s64 (int64x2_t __a, int64x2_t __b)
 {
-  return (int32x2_t) __builtin_aarch64_raddhnv2di (__a, __b);
+  return __builtin_aarch64_raddhnv2di (__a, __b);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_u16 (uint16x8_t __a, uint16x8_t __b)
 {
-  return (uint8x8_t) __builtin_aarch64_raddhnv8hi ((int16x8_t) __a,
-						   (int16x8_t) __b);
+  return __builtin_aarch64_raddhnv8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_u32 (uint32x4_t __a, uint32x4_t __b)
 {
-  return (uint16x4_t) __builtin_aarch64_raddhnv4si ((int32x4_t) __a,
-						    (int32x4_t) __b);
+  return __builtin_aarch64_raddhnv4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_u64 (uint64x2_t __a, uint64x2_t __b)
 {
-  return (uint32x2_t) __builtin_aarch64_raddhnv2di ((int64x2_t) __a,
-						    (int64x2_t) __b);
+  return __builtin_aarch64_raddhnv2di_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_high_s16 (int8x8_t __a, int16x8_t __b, int16x8_t __c)
 {
-  return (int8x16_t) __builtin_aarch64_addhn2v8hi (__a, __b, __c);
+  return __builtin_aarch64_addhn2v8hi (__a, __b, __c);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_high_s32 (int16x4_t __a, int32x4_t __b, int32x4_t __c)
 {
-  return (int16x8_t) __builtin_aarch64_addhn2v4si (__a, __b, __c);
+  return __builtin_aarch64_addhn2v4si (__a, __b, __c);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_high_s64 (int32x2_t __a, int64x2_t __b, int64x2_t __c)
 {
-  return (int32x4_t) __builtin_aarch64_addhn2v2di (__a, __b, __c);
+  return __builtin_aarch64_addhn2v2di (__a, __b, __c);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_high_u16 (uint8x8_t __a, uint16x8_t __b, uint16x8_t __c)
 {
-  return (uint8x16_t) __builtin_aarch64_addhn2v8hi ((int8x8_t) __a,
-						    (int16x8_t) __b,
-						    (int16x8_t) __c);
+  return __builtin_aarch64_addhn2v8hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_high_u32 (uint16x4_t __a, uint32x4_t __b, uint32x4_t __c)
 {
-  return (uint16x8_t) __builtin_aarch64_addhn2v4si ((int16x4_t) __a,
-						    (int32x4_t) __b,
-						    (int32x4_t) __c);
+  return __builtin_aarch64_addhn2v4si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddhn_high_u64 (uint32x2_t __a, uint64x2_t __b, uint64x2_t __c)
 {
-  return (uint32x4_t) __builtin_aarch64_addhn2v2di ((int32x2_t) __a,
-						    (int64x2_t) __b,
-						    (int64x2_t) __c);
+  return __builtin_aarch64_addhn2v2di_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_high_s16 (int8x8_t __a, int16x8_t __b, int16x8_t __c)
 {
-  return (int8x16_t) __builtin_aarch64_raddhn2v8hi (__a, __b, __c);
+  return __builtin_aarch64_raddhn2v8hi (__a, __b, __c);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_high_s32 (int16x4_t __a, int32x4_t __b, int32x4_t __c)
 {
-  return (int16x8_t) __builtin_aarch64_raddhn2v4si (__a, __b, __c);
+  return __builtin_aarch64_raddhn2v4si (__a, __b, __c);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_high_s64 (int32x2_t __a, int64x2_t __b, int64x2_t __c)
 {
-  return (int32x4_t) __builtin_aarch64_raddhn2v2di (__a, __b, __c);
+  return __builtin_aarch64_raddhn2v2di (__a, __b, __c);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_high_u16 (uint8x8_t __a, uint16x8_t __b, uint16x8_t __c)
 {
-  return (uint8x16_t) __builtin_aarch64_raddhn2v8hi ((int8x8_t) __a,
-						     (int16x8_t) __b,
-						     (int16x8_t) __c);
+  return __builtin_aarch64_raddhn2v8hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_high_u32 (uint16x4_t __a, uint32x4_t __b, uint32x4_t __c)
 {
-  return (uint16x8_t) __builtin_aarch64_raddhn2v4si ((int16x4_t) __a,
-						     (int32x4_t) __b,
-						     (int32x4_t) __c);
+  return __builtin_aarch64_raddhn2v4si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vraddhn_high_u64 (uint32x2_t __a, uint64x2_t __b, uint64x2_t __c)
 {
-  return (uint32x4_t) __builtin_aarch64_raddhn2v2di ((int32x2_t) __a,
-						     (int64x2_t) __b,
-						     (int64x2_t) __c);
+  return __builtin_aarch64_raddhn2v2di_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline float32x2_t
@@ -1455,8 +965,7 @@ __extension__ extern __inline poly8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmul_p8 (poly8x8_t __a, poly8x8_t __b)
 {
-  return (poly8x8_t) __builtin_aarch64_pmulv8qi ((int8x8_t) __a,
-						 (int8x8_t) __b);
+  return __builtin_aarch64_pmulv8qi_ppp (__a, __b);
 }
 
 __extension__ extern __inline int8x16_t
@@ -1519,8 +1028,7 @@ __extension__ extern __inline poly8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmulq_p8 (poly8x16_t __a, poly8x16_t __b)
 {
-  return (poly8x16_t) __builtin_aarch64_pmulv16qi ((int8x16_t) __a,
-						   (int8x16_t) __b);
+  return __builtin_aarch64_pmulv16qi_ppp (__a, __b);
 }
 
 __extension__ extern __inline int8x8_t
@@ -2227,180 +1735,168 @@ __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_s8 (int8x8_t __a, int8x8_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_ssublv8qi (__a, __b);
+  return __builtin_aarch64_ssublv8qi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_s16 (int16x4_t __a, int16x4_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_ssublv4hi (__a, __b);
+  return __builtin_aarch64_ssublv4hi (__a, __b);
 }
 
 __extension__ extern __inline int64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_s32 (int32x2_t __a, int32x2_t __b)
 {
-  return (int64x2_t) __builtin_aarch64_ssublv2si (__a, __b);
+  return __builtin_aarch64_ssublv2si (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_u8 (uint8x8_t __a, uint8x8_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_usublv8qi ((int8x8_t) __a,
-						   (int8x8_t) __b);
+  return __builtin_aarch64_usublv8qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_u16 (uint16x4_t __a, uint16x4_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_usublv4hi ((int16x4_t) __a,
-						   (int16x4_t) __b);
+  return __builtin_aarch64_usublv4hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_u32 (uint32x2_t __a, uint32x2_t __b)
 {
-  return (uint64x2_t) __builtin_aarch64_usublv2si ((int32x2_t) __a,
-						   (int32x2_t) __b);
+  return __builtin_aarch64_usublv2si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_high_s8 (int8x16_t __a, int8x16_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_ssubl2v16qi (__a, __b);
+  return __builtin_aarch64_ssubl2v16qi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_high_s16 (int16x8_t __a, int16x8_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_ssubl2v8hi (__a, __b);
+  return __builtin_aarch64_ssubl2v8hi (__a, __b);
 }
 
 __extension__ extern __inline int64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_high_s32 (int32x4_t __a, int32x4_t __b)
 {
-  return (int64x2_t) __builtin_aarch64_ssubl2v4si (__a, __b);
+  return __builtin_aarch64_ssubl2v4si (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_high_u8 (uint8x16_t __a, uint8x16_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_usubl2v16qi ((int8x16_t) __a,
-						     (int8x16_t) __b);
+  return __builtin_aarch64_usubl2v16qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_high_u16 (uint16x8_t __a, uint16x8_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_usubl2v8hi ((int16x8_t) __a,
-						    (int16x8_t) __b);
+  return __builtin_aarch64_usubl2v8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubl_high_u32 (uint32x4_t __a, uint32x4_t __b)
 {
-  return (uint64x2_t) __builtin_aarch64_usubl2v4si ((int32x4_t) __a,
-						    (int32x4_t) __b);
+  return __builtin_aarch64_usubl2v4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_s8 (int16x8_t __a, int8x8_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_ssubwv8qi (__a, __b);
+  return __builtin_aarch64_ssubwv8qi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_s16 (int32x4_t __a, int16x4_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_ssubwv4hi (__a, __b);
+  return __builtin_aarch64_ssubwv4hi (__a, __b);
 }
 
 __extension__ extern __inline int64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_s32 (int64x2_t __a, int32x2_t __b)
 {
-  return (int64x2_t) __builtin_aarch64_ssubwv2si (__a, __b);
+  return __builtin_aarch64_ssubwv2si (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_u8 (uint16x8_t __a, uint8x8_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_usubwv8qi ((int16x8_t) __a,
-						   (int8x8_t) __b);
+  return __builtin_aarch64_usubwv8qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_u16 (uint32x4_t __a, uint16x4_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_usubwv4hi ((int32x4_t) __a,
-						   (int16x4_t) __b);
+  return __builtin_aarch64_usubwv4hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_u32 (uint64x2_t __a, uint32x2_t __b)
 {
-  return (uint64x2_t) __builtin_aarch64_usubwv2si ((int64x2_t) __a,
-						   (int32x2_t) __b);
+  return __builtin_aarch64_usubwv2si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_high_s8 (int16x8_t __a, int8x16_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_ssubw2v16qi (__a, __b);
+  return __builtin_aarch64_ssubw2v16qi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_high_s16 (int32x4_t __a, int16x8_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_ssubw2v8hi (__a, __b);
+  return __builtin_aarch64_ssubw2v8hi (__a, __b);
 }
 
 __extension__ extern __inline int64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_high_s32 (int64x2_t __a, int32x4_t __b)
 {
-  return (int64x2_t) __builtin_aarch64_ssubw2v4si (__a, __b);
+  return __builtin_aarch64_ssubw2v4si (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_high_u8 (uint16x8_t __a, uint8x16_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_usubw2v16qi ((int16x8_t) __a,
-						     (int8x16_t) __b);
+  return __builtin_aarch64_usubw2v16qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_high_u16 (uint32x4_t __a, uint16x8_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_usubw2v8hi ((int32x4_t) __a,
-						    (int16x8_t) __b);
+  return __builtin_aarch64_usubw2v8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubw_high_u32 (uint64x2_t __a, uint32x4_t __b)
 {
-  return (uint64x2_t) __builtin_aarch64_usubw2v4si ((int64x2_t) __a,
-						    (int32x4_t) __b);
+  return __builtin_aarch64_usubw2v4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x8_t
@@ -2442,276 +1938,252 @@ __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsub_s8 (int8x8_t __a, int8x8_t __b)
 {
-  return (int8x8_t)__builtin_aarch64_shsubv8qi (__a, __b);
+  return __builtin_aarch64_shsubv8qi (__a, __b);
 }
 
 __extension__ extern __inline int16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsub_s16 (int16x4_t __a, int16x4_t __b)
 {
-  return (int16x4_t) __builtin_aarch64_shsubv4hi (__a, __b);
+  return __builtin_aarch64_shsubv4hi (__a, __b);
 }
 
 __extension__ extern __inline int32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsub_s32 (int32x2_t __a, int32x2_t __b)
 {
-  return (int32x2_t) __builtin_aarch64_shsubv2si (__a, __b);
+  return __builtin_aarch64_shsubv2si (__a, __b);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsub_u8 (uint8x8_t __a, uint8x8_t __b)
 {
-  return (uint8x8_t) __builtin_aarch64_uhsubv8qi ((int8x8_t) __a,
-						  (int8x8_t) __b);
+  return __builtin_aarch64_uhsubv8qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsub_u16 (uint16x4_t __a, uint16x4_t __b)
 {
-  return (uint16x4_t) __builtin_aarch64_uhsubv4hi ((int16x4_t) __a,
-						   (int16x4_t) __b);
+  return __builtin_aarch64_uhsubv4hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsub_u32 (uint32x2_t __a, uint32x2_t __b)
 {
-  return (uint32x2_t) __builtin_aarch64_uhsubv2si ((int32x2_t) __a,
-						   (int32x2_t) __b);
+  return __builtin_aarch64_uhsubv2si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsubq_s8 (int8x16_t __a, int8x16_t __b)
 {
-  return (int8x16_t) __builtin_aarch64_shsubv16qi (__a, __b);
+  return __builtin_aarch64_shsubv16qi (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsubq_s16 (int16x8_t __a, int16x8_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_shsubv8hi (__a, __b);
+  return __builtin_aarch64_shsubv8hi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsubq_s32 (int32x4_t __a, int32x4_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_shsubv4si (__a, __b);
+  return __builtin_aarch64_shsubv4si (__a, __b);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsubq_u8 (uint8x16_t __a, uint8x16_t __b)
 {
-  return (uint8x16_t) __builtin_aarch64_uhsubv16qi ((int8x16_t) __a,
-						    (int8x16_t) __b);
+  return __builtin_aarch64_uhsubv16qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsubq_u16 (uint16x8_t __a, uint16x8_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_uhsubv8hi ((int16x8_t) __a,
-						   (int16x8_t) __b);
+  return __builtin_aarch64_uhsubv8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vhsubq_u32 (uint32x4_t __a, uint32x4_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_uhsubv4si ((int32x4_t) __a,
-						   (int32x4_t) __b);
+  return __builtin_aarch64_uhsubv4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_s16 (int16x8_t __a, int16x8_t __b)
 {
-  return (int8x8_t) __builtin_aarch64_subhnv8hi (__a, __b);
+  return __builtin_aarch64_subhnv8hi (__a, __b);
 }
 
 __extension__ extern __inline int16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_s32 (int32x4_t __a, int32x4_t __b)
 {
-  return (int16x4_t) __builtin_aarch64_subhnv4si (__a, __b);
+  return __builtin_aarch64_subhnv4si (__a, __b);
 }
 
 __extension__ extern __inline int32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_s64 (int64x2_t __a, int64x2_t __b)
 {
-  return (int32x2_t) __builtin_aarch64_subhnv2di (__a, __b);
+  return __builtin_aarch64_subhnv2di (__a, __b);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_u16 (uint16x8_t __a, uint16x8_t __b)
 {
-  return (uint8x8_t) __builtin_aarch64_subhnv8hi ((int16x8_t) __a,
-						  (int16x8_t) __b);
+  return __builtin_aarch64_subhnv8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_u32 (uint32x4_t __a, uint32x4_t __b)
 {
-  return (uint16x4_t) __builtin_aarch64_subhnv4si ((int32x4_t) __a,
-						   (int32x4_t) __b);
+  return __builtin_aarch64_subhnv4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_u64 (uint64x2_t __a, uint64x2_t __b)
 {
-  return (uint32x2_t) __builtin_aarch64_subhnv2di ((int64x2_t) __a,
-						   (int64x2_t) __b);
+  return __builtin_aarch64_subhnv2di_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_s16 (int16x8_t __a, int16x8_t __b)
 {
-  return (int8x8_t) __builtin_aarch64_rsubhnv8hi (__a, __b);
+  return __builtin_aarch64_rsubhnv8hi (__a, __b);
 }
 
 __extension__ extern __inline int16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_s32 (int32x4_t __a, int32x4_t __b)
 {
-  return (int16x4_t) __builtin_aarch64_rsubhnv4si (__a, __b);
+  return __builtin_aarch64_rsubhnv4si (__a, __b);
 }
 
 __extension__ extern __inline int32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_s64 (int64x2_t __a, int64x2_t __b)
 {
-  return (int32x2_t) __builtin_aarch64_rsubhnv2di (__a, __b);
+  return __builtin_aarch64_rsubhnv2di (__a, __b);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_u16 (uint16x8_t __a, uint16x8_t __b)
 {
-  return (uint8x8_t) __builtin_aarch64_rsubhnv8hi ((int16x8_t) __a,
-						   (int16x8_t) __b);
+  return __builtin_aarch64_rsubhnv8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_u32 (uint32x4_t __a, uint32x4_t __b)
 {
-  return (uint16x4_t) __builtin_aarch64_rsubhnv4si ((int32x4_t) __a,
-						    (int32x4_t) __b);
+  return __builtin_aarch64_rsubhnv4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_u64 (uint64x2_t __a, uint64x2_t __b)
 {
-  return (uint32x2_t) __builtin_aarch64_rsubhnv2di ((int64x2_t) __a,
-						    (int64x2_t) __b);
+  return __builtin_aarch64_rsubhnv2di_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_high_s16 (int8x8_t __a, int16x8_t __b, int16x8_t __c)
 {
-  return (int8x16_t) __builtin_aarch64_rsubhn2v8hi (__a, __b, __c);
+  return __builtin_aarch64_rsubhn2v8hi (__a, __b, __c);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_high_s32 (int16x4_t __a, int32x4_t __b, int32x4_t __c)
 {
-  return (int16x8_t) __builtin_aarch64_rsubhn2v4si (__a, __b, __c);
+  return __builtin_aarch64_rsubhn2v4si (__a, __b, __c);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_high_s64 (int32x2_t __a, int64x2_t __b, int64x2_t __c)
 {
-  return (int32x4_t) __builtin_aarch64_rsubhn2v2di (__a, __b, __c);
+  return __builtin_aarch64_rsubhn2v2di (__a, __b, __c);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_high_u16 (uint8x8_t __a, uint16x8_t __b, uint16x8_t __c)
 {
-  return (uint8x16_t) __builtin_aarch64_rsubhn2v8hi ((int8x8_t) __a,
-						     (int16x8_t) __b,
-						     (int16x8_t) __c);
+  return __builtin_aarch64_rsubhn2v8hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_high_u32 (uint16x4_t __a, uint32x4_t __b, uint32x4_t __c)
 {
-  return (uint16x8_t) __builtin_aarch64_rsubhn2v4si ((int16x4_t) __a,
-						     (int32x4_t) __b,
-						     (int32x4_t) __c);
+  return __builtin_aarch64_rsubhn2v4si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrsubhn_high_u64 (uint32x2_t __a, uint64x2_t __b, uint64x2_t __c)
 {
-  return (uint32x4_t) __builtin_aarch64_rsubhn2v2di ((int32x2_t) __a,
-						     (int64x2_t) __b,
-						     (int64x2_t) __c);
+  return __builtin_aarch64_rsubhn2v2di_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_high_s16 (int8x8_t __a, int16x8_t __b, int16x8_t __c)
 {
-  return (int8x16_t) __builtin_aarch64_subhn2v8hi (__a, __b, __c);
+  return __builtin_aarch64_subhn2v8hi (__a, __b, __c);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_high_s32 (int16x4_t __a, int32x4_t __b, int32x4_t __c)
 {
-  return (int16x8_t) __builtin_aarch64_subhn2v4si (__a, __b, __c);;
+  return __builtin_aarch64_subhn2v4si (__a, __b, __c);;
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_high_s64 (int32x2_t __a, int64x2_t __b, int64x2_t __c)
 {
-  return (int32x4_t) __builtin_aarch64_subhn2v2di (__a, __b, __c);
+  return __builtin_aarch64_subhn2v2di (__a, __b, __c);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_high_u16 (uint8x8_t __a, uint16x8_t __b, uint16x8_t __c)
 {
-  return (uint8x16_t) __builtin_aarch64_subhn2v8hi ((int8x8_t) __a,
-						    (int16x8_t) __b,
-						    (int16x8_t) __c);
+  return __builtin_aarch64_subhn2v8hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_high_u32 (uint16x4_t __a, uint32x4_t __b, uint32x4_t __c)
 {
-  return (uint16x8_t) __builtin_aarch64_subhn2v4si ((int16x4_t) __a,
-						    (int32x4_t) __b,
-						    (int32x4_t) __c);
+  return __builtin_aarch64_subhn2v4si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vsubhn_high_u64 (uint32x2_t __a, uint64x2_t __b, uint64x2_t __c)
 {
-  return (uint32x4_t) __builtin_aarch64_subhn2v2di ((int32x2_t) __a,
-						    (int64x2_t) __b,
-						    (int64x2_t) __c);
+  return __builtin_aarch64_subhn2v2di_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint16x4_t
@@ -6503,21 +5975,21 @@ __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_s8 (int8x8_t __a, int8x8_t __b)
 {
-  return (int8x16_t) __builtin_aarch64_combinev8qi (__a, __b);
+  return __builtin_aarch64_combinev8qi (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_s16 (int16x4_t __a, int16x4_t __b)
 {
-  return (int16x8_t) __builtin_aarch64_combinev4hi (__a, __b);
+  return __builtin_aarch64_combinev4hi (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_s32 (int32x2_t __a, int32x2_t __b)
 {
-  return (int32x4_t) __builtin_aarch64_combinev2si (__a, __b);
+  return __builtin_aarch64_combinev2si (__a, __b);
 }
 
 __extension__ extern __inline int64x2_t
@@ -6538,38 +6010,35 @@ __extension__ extern __inline float32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_f32 (float32x2_t __a, float32x2_t __b)
 {
-  return (float32x4_t) __builtin_aarch64_combinev2sf (__a, __b);
+  return __builtin_aarch64_combinev2sf (__a, __b);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_u8 (uint8x8_t __a, uint8x8_t __b)
 {
-  return (uint8x16_t) __builtin_aarch64_combinev8qi ((int8x8_t) __a,
-						     (int8x8_t) __b);
+  return __builtin_aarch64_combinev8qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_u16 (uint16x4_t __a, uint16x4_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_combinev4hi ((int16x4_t) __a,
-						     (int16x4_t) __b);
+  return __builtin_aarch64_combinev4hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_u32 (uint32x2_t __a, uint32x2_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_combinev2si ((int32x2_t) __a,
-						     (int32x2_t) __b);
+  return __builtin_aarch64_combinev2si_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_u64 (uint64x1_t __a, uint64x1_t __b)
 {
-  return (uint64x2_t) __builtin_aarch64_combinedi (__a[0], __b[0]);
+  return __builtin_aarch64_combinedi_uuu (__a[0], __b[0]);
 }
 
 __extension__ extern __inline float64x2_t
@@ -6583,23 +6052,21 @@ __extension__ extern __inline poly8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_p8 (poly8x8_t __a, poly8x8_t __b)
 {
-  return (poly8x16_t) __builtin_aarch64_combinev8qi ((int8x8_t) __a,
-						     (int8x8_t) __b);
+  return __builtin_aarch64_combinev8qi_ppp (__a, __b);
 }
 
 __extension__ extern __inline poly16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_p16 (poly16x4_t __a, poly16x4_t __b)
 {
-  return (poly16x8_t) __builtin_aarch64_combinev4hi ((int16x4_t) __a,
-						     (int16x4_t) __b);
+  return __builtin_aarch64_combinev4hi_ppp (__a, __b);
 }
 
 __extension__ extern __inline poly64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_p64 (poly64x1_t __a, poly64x1_t __b)
 {
-  return (poly64x2_t) __builtin_aarch64_combinedi_ppp (__a[0], __b[0]);
+  return __builtin_aarch64_combinedi_ppp (__a[0], __b[0]);
 }
 
 /* Start of temporary inline asm implementations.  */
@@ -7056,18 +6523,14 @@ __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmla_n_u16 (uint16x4_t __a, uint16x4_t __b, uint16_t __c)
 {
-  return (uint16x4_t) __builtin_aarch64_mla_nv4hi ((int16x4_t) __a,
-                                                   (int16x4_t) __b,
-                                                   (int16_t) __c);
+  return __builtin_aarch64_mla_nv4hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmla_n_u32 (uint32x2_t __a, uint32x2_t __b, uint32_t __c)
 {
-  return (uint32x2_t) __builtin_aarch64_mla_nv2si ((int32x2_t) __a,
-                                                   (int32x2_t) __b,
-                                                   (int32_t) __c);
+  return __builtin_aarch64_mla_nv2si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline int8x8_t
@@ -7095,27 +6558,21 @@ __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmla_u8 (uint8x8_t __a, uint8x8_t __b, uint8x8_t __c)
 {
-  return (uint8x8_t) __builtin_aarch64_mlav8qi ((int8x8_t) __a,
-                                                (int8x8_t) __b,
-                                                (int8x8_t) __c);
+  return __builtin_aarch64_mlav8qi_uuuu (__a, __b,  __c);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmla_u16 (uint16x4_t __a, uint16x4_t __b, uint16x4_t __c)
 {
-  return (uint16x4_t) __builtin_aarch64_mlav4hi ((int16x4_t) __a,
-                                                 (int16x4_t) __b,
-                                                 (int16x4_t) __c);
+  return __builtin_aarch64_mlav4hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmla_u32 (uint32x2_t __a, uint32x2_t __b, uint32x2_t __c)
 {
-  return (uint32x2_t) __builtin_aarch64_mlav2si ((int32x2_t) __a,
-                                                 (int32x2_t) __b,
-                                                 (int32x2_t) __c);
+  return __builtin_aarch64_mlav2si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline int32x4_t
@@ -7403,18 +6860,14 @@ __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmlaq_n_u16 (uint16x8_t __a, uint16x8_t __b, uint16_t __c)
 {
-  return (uint16x8_t) __builtin_aarch64_mla_nv8hi ((int16x8_t) __a,
-                                                   (int16x8_t) __b,
-                                                   (int16_t) __c);
+  return __builtin_aarch64_mla_nv8hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmlaq_n_u32 (uint32x4_t __a, uint32x4_t __b, uint32_t __c)
 {
-  return (uint32x4_t) __builtin_aarch64_mla_nv4si ((int32x4_t) __a,
-                                                   (int32x4_t) __b,
-                                                   (int32_t) __c);
+  return __builtin_aarch64_mla_nv4si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline int8x16_t
@@ -7442,27 +6895,21 @@ __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmlaq_u8 (uint8x16_t __a, uint8x16_t __b, uint8x16_t __c)
 {
-  return (uint8x16_t) __builtin_aarch64_mlav16qi ((int8x16_t) __a,
-                                                  (int8x16_t) __b,
-                                                  (int8x16_t) __c);
+  return __builtin_aarch64_mlav16qi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmlaq_u16 (uint16x8_t __a, uint16x8_t __b, uint16x8_t __c)
 {
-  return (uint16x8_t) __builtin_aarch64_mlav8hi ((int16x8_t) __a,
-                                                 (int16x8_t) __b,
-                                                 (int16x8_t) __c);
+  return __builtin_aarch64_mlav8hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmlaq_u32 (uint32x4_t __a, uint32x4_t __b, uint32x4_t __c)
 {
-  return (uint32x4_t) __builtin_aarch64_mlav4si ((int32x4_t) __a,
-                                                 (int32x4_t) __b,
-                                                 (int32x4_t) __c);
+  return __builtin_aarch64_mlav4si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline float32x2_t
@@ -7490,18 +6937,14 @@ __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmls_n_u16 (uint16x4_t __a, uint16x4_t __b, uint16_t __c)
 {
-  return (uint16x4_t) __builtin_aarch64_mls_nv4hi ((int16x4_t) __a,
-                                                   (int16x4_t) __b,
-                                                   (int16_t) __c);
+  return __builtin_aarch64_mls_nv4hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmls_n_u32 (uint32x2_t __a, uint32x2_t __b, uint32_t __c)
 {
-  return (uint32x2_t) __builtin_aarch64_mls_nv2si ((int32x2_t) __a,
-                                                   (int32x2_t) __b,
-                                                   (int32_t) __c);
+  return __builtin_aarch64_mls_nv2si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline int8x8_t
@@ -7529,27 +6972,21 @@ __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmls_u8 (uint8x8_t __a, uint8x8_t __b, uint8x8_t __c)
 {
-  return (uint8x8_t) __builtin_aarch64_mlsv8qi ((int8x8_t) __a,
-                                                (int8x8_t) __b,
-                                                (int8x8_t) __c);
+  return __builtin_aarch64_mlsv8qi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmls_u16 (uint16x4_t __a, uint16x4_t __b, uint16x4_t __c)
 {
-  return (uint16x4_t) __builtin_aarch64_mlsv4hi ((int16x4_t) __a,
-                                                 (int16x4_t) __b,
-                                                 (int16x4_t) __c);
+  return __builtin_aarch64_mlsv4hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmls_u32 (uint32x2_t __a, uint32x2_t __b, uint32x2_t __c)
 {
-  return (uint32x2_t) __builtin_aarch64_mlsv2si ((int32x2_t) __a,
-                                                 (int32x2_t) __b,
-                                                 (int32x2_t) __c);
+  return __builtin_aarch64_mlsv2si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline int32x4_t
@@ -7841,18 +7278,14 @@ __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmlsq_n_u16 (uint16x8_t __a, uint16x8_t __b, uint16_t __c)
 {
-  return (uint16x8_t) __builtin_aarch64_mls_nv8hi ((int16x8_t) __a,
-                                                   (int16x8_t) __b,
-                                                   (int16_t) __c);
+  return __builtin_aarch64_mls_nv8hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmlsq_n_u32 (uint32x4_t __a, uint32x4_t __b, uint32_t __c)
 {
-  return (uint32x4_t) __builtin_aarch64_mls_nv4si ((int32x4_t) __a,
-                                                   (int32x4_t) __b,
-                                                   (int32_t) __c);
+  return __builtin_aarch64_mls_nv4si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline int8x16_t
@@ -7880,27 +7313,21 @@ __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmlsq_u8 (uint8x16_t __a, uint8x16_t __b, uint8x16_t __c)
 {
-  return (uint8x16_t) __builtin_aarch64_mlsv16qi ((int8x16_t) __a,
-                                                  (int8x16_t) __b,
-                                                  (int8x16_t) __c);
+  return __builtin_aarch64_mlsv16qi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmlsq_u16 (uint16x8_t __a, uint16x8_t __b, uint16x8_t __c)
 {
-  return (uint16x8_t) __builtin_aarch64_mlsv8hi ((int16x8_t) __a,
-                                                 (int16x8_t) __b,
-                                                 (int16x8_t) __c);
+  return __builtin_aarch64_mlsv8hi_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmlsq_u32 (uint32x4_t __a, uint32x4_t __b, uint32x4_t __c)
 {
-  return (uint32x4_t) __builtin_aarch64_mlsv4si ((int32x4_t) __a,
-                                                 (int32x4_t) __b,
-                                                 (int32x4_t) __c);
+  return __builtin_aarch64_mlsv4si_uuuu (__a, __b, __c);
 }
 
 __extension__ extern __inline int16x8_t
@@ -8012,24 +7439,21 @@ __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmovn_high_u16 (uint8x8_t __a, uint16x8_t __b)
 {
-  return (uint8x16_t)
-	   __builtin_aarch64_xtn2v8hi ((int8x8_t) __a, (int16x8_t) __b);
+  return __builtin_aarch64_xtn2v8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmovn_high_u32 (uint16x4_t __a, uint32x4_t __b)
 {
-  return (uint16x8_t)
-	   __builtin_aarch64_xtn2v4si ((int16x4_t) __a, (int32x4_t) __b);
+  return __builtin_aarch64_xtn2v4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmovn_high_u64 (uint32x2_t __a, uint64x2_t __b)
 {
-  return (uint32x4_t)
-	   __builtin_aarch64_xtn2v2di ((int32x2_t) __a, (int64x2_t) __b);
+  return __builtin_aarch64_xtn2v2di_uuu (__a, __b);
 }
 
 __extension__ extern __inline int8x8_t
@@ -8057,21 +7481,21 @@ __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmovn_u16 (uint16x8_t __a)
 {
-  return (uint8x8_t)__builtin_aarch64_xtnv8hi ((int16x8_t) __a);
+  return __builtin_aarch64_xtnv8hi_uu (__a);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmovn_u32 (uint32x4_t __a)
 {
-  return (uint16x4_t) __builtin_aarch64_xtnv4si ((int32x4_t )__a);
+  return __builtin_aarch64_xtnv4si_uu (__a);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmovn_u64 (uint64x2_t __a)
 {
-  return (uint32x2_t) __builtin_aarch64_xtnv2di ((int64x2_t) __a);
+  return __builtin_aarch64_xtnv2di_uu (__a);
 }
 
 __extension__ extern __inline int8x8_t
@@ -8099,21 +7523,21 @@ __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshrn_n_u16 (uint16x8_t __a, const int __b)
 {
-  return (uint8x8_t)__builtin_aarch64_shrnv8hi ((int16x8_t)__a, __b);
+  return __builtin_aarch64_shrnv8hi_uus (__a, __b);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshrn_n_u32 (uint32x4_t __a, const int __b)
 {
-  return (uint16x4_t)__builtin_aarch64_shrnv4si ((int32x4_t)__a, __b);
+  return __builtin_aarch64_shrnv4si_uus (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshrn_n_u64 (uint64x2_t __a, const int __b)
 {
-  return (uint32x2_t)__builtin_aarch64_shrnv2di ((int64x2_t)__a, __b);
+  return __builtin_aarch64_shrnv2di_uus (__a, __b);
 }
 
 __extension__ extern __inline int32x4_t
@@ -8204,8 +7628,7 @@ __extension__ extern __inline poly16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmull_high_p8 (poly8x16_t __a, poly8x16_t __b)
 {
-  return (poly16x8_t) __builtin_aarch64_pmull_hiv16qi ((int8x16_t) __a,
-						       (int8x16_t) __b);
+  return __builtin_aarch64_pmull_hiv16qi_ppp (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
@@ -8338,8 +7761,7 @@ __extension__ extern __inline poly16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmull_p8 (poly8x8_t __a, poly8x8_t __b)
 {
-  return (poly16x8_t) __builtin_aarch64_pmullv8qi ((int8x8_t) __a,
-						   (int8x8_t) __b);
+  return __builtin_aarch64_pmullv8qi_ppp (__a, __b);
 }
 
 __extension__ extern __inline int16x8_t
@@ -8584,32 +8006,28 @@ __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vpaddq_u8 (uint8x16_t __a, uint8x16_t __b)
 {
-  return (uint8x16_t) __builtin_aarch64_addpv16qi ((int8x16_t) __a,
-						   (int8x16_t) __b);
+  return __builtin_aarch64_addpv16qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vpaddq_u16 (uint16x8_t __a, uint16x8_t __b)
 {
-  return (uint16x8_t) __builtin_aarch64_addpv8hi ((int16x8_t) __a,
-						  (int16x8_t) __b);
+  return __builtin_aarch64_addpv8hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vpaddq_u32 (uint32x4_t __a, uint32x4_t __b)
 {
-  return (uint32x4_t) __builtin_aarch64_addpv4si ((int32x4_t) __a,
-						  (int32x4_t) __b);
+  return __builtin_aarch64_addpv4si_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vpaddq_u64 (uint64x2_t __a, uint64x2_t __b)
 {
-  return (uint64x2_t) __builtin_aarch64_addpv2di ((int64x2_t) __a,
-						  (int64x2_t) __b);
+  return __builtin_aarch64_addpv2di_uuu (__a, __b);
 }
 
 __extension__ extern __inline int16x4_t
@@ -8882,24 +8300,21 @@ __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrshrn_high_n_u16 (uint8x8_t __a, uint16x8_t __b, const int __c)
 {
-  return (uint8x16_t) __builtin_aarch64_rshrn2v8hi ((int8x8_t) __a,
-						    (int16x8_t) __b, __c);
+  return __builtin_aarch64_rshrn2v8hi_uuus (__a, __b, __c);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrshrn_high_n_u32 (uint16x4_t __a, uint32x4_t __b, const int __c)
 {
-  return (uint16x8_t) __builtin_aarch64_rshrn2v4si ((int16x4_t) __a,
-						    (int32x4_t) __b, __c);
+  return __builtin_aarch64_rshrn2v4si_uuus (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrshrn_high_n_u64 (uint32x2_t __a, uint64x2_t __b, const int __c)
 {
-  return (uint32x4_t) __builtin_aarch64_rshrn2v2di ((int32x2_t)__a,
-						    (int64x2_t)__b, __c);
+  return __builtin_aarch64_rshrn2v2di_uuus (__a, __b, __c);
 }
 
 __extension__ extern __inline int8x8_t
@@ -8927,21 +8342,21 @@ __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrshrn_n_u16 (uint16x8_t __a, const int __b)
 {
-  return (uint8x8_t) __builtin_aarch64_rshrnv8hi ((int16x8_t) __a, __b);
+  return __builtin_aarch64_rshrnv8hi_uus (__a, __b);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrshrn_n_u32 (uint32x4_t __a, const int __b)
 {
-  return (uint16x4_t) __builtin_aarch64_rshrnv4si ((int32x4_t) __a, __b);
+  return __builtin_aarch64_rshrnv4si_uus (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vrshrn_n_u64 (uint64x2_t __a, const int __b)
 {
-  return (uint32x2_t) __builtin_aarch64_rshrnv2di ((int64x2_t) __a, __b);
+  return __builtin_aarch64_rshrnv2di_uus (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
@@ -8983,24 +8398,21 @@ __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshrn_high_n_u16 (uint8x8_t __a, uint16x8_t __b, const int __c)
 {
-  return (uint8x16_t)
-    __builtin_aarch64_shrn2v8hi ((int8x8_t) __a, (int16x8_t) __b, __c);
+  return __builtin_aarch64_shrn2v8hi_uuus (__a, __b, __c);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshrn_high_n_u32 (uint16x4_t __a, uint32x4_t __b, const int __c)
 {
-  return (uint16x8_t)
-    __builtin_aarch64_shrn2v4si ((int16x4_t) __a, (int32x4_t) __b, __c);
+  return __builtin_aarch64_shrn2v4si_uuus (__a, __b, __c);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshrn_high_n_u64 (uint32x2_t __a, uint64x2_t __b, const int __c)
 {
-  return (uint32x4_t)
-    __builtin_aarch64_shrn2v2di ((int32x2_t) __a, (int64x2_t) __b, __c);
+  return __builtin_aarch64_shrn2v2di_uuus (__a, __b, __c);
 }
 
 __extension__ extern __inline poly8x8_t
@@ -9120,104 +8532,11 @@ vtstq_p64 (poly64x2_t __a, poly64x2_t __b)
 		       != __AARCH64_INT64_C (0));
 }
 
-/* End of temporary inline asm implementations.  */
-
-/* Start of temporary inline asm for vldn, vstn and friends.  */
-
-/* Create struct element types for duplicating loads.
-
-   Create 2 element structures of:
-
-   +------+----+----+----+----+
-   |      | 8  | 16 | 32 | 64 |
-   +------+----+----+----+----+
-   |int   | Y  | Y  | N  | N  |
-   +------+----+----+----+----+
-   |uint  | Y  | Y  | N  | N  |
-   +------+----+----+----+----+
-   |float | -  | Y  | N  | N  |
-   +------+----+----+----+----+
-   |poly  | Y  | Y  | -  | -  |
-   +------+----+----+----+----+
-
-   Create 3 element structures of:
-
-   +------+----+----+----+----+
-   |      | 8  | 16 | 32 | 64 |
-   +------+----+----+----+----+
-   |int   | Y  | Y  | Y  | Y  |
-   +------+----+----+----+----+
-   |uint  | Y  | Y  | Y  | Y  |
-   +------+----+----+----+----+
-   |float | -  | Y  | Y  | Y  |
-   +------+----+----+----+----+
-   |poly  | Y  | Y  | -  | -  |
-   +------+----+----+----+----+
-
-   Create 4 element structures of:
-
-   +------+----+----+----+----+
-   |      | 8  | 16 | 32 | 64 |
-   +------+----+----+----+----+
-   |int   | Y  | N  | N  | Y  |
-   +------+----+----+----+----+
-   |uint  | Y  | N  | N  | Y  |
-   +------+----+----+----+----+
-   |float | -  | N  | N  | Y  |
-   +------+----+----+----+----+
-   |poly  | Y  | N  | -  | -  |
-   +------+----+----+----+----+
-
-  This is required for casting memory reference.  */
-#define __STRUCTN(t, sz, nelem)			\
-  typedef struct t ## sz ## x ## nelem ## _t {	\
-    t ## sz ## _t val[nelem];			\
-  }  t ## sz ## x ## nelem ## _t;
-
-/* 2-element structs.  */
-__STRUCTN (int, 8, 2)
-__STRUCTN (int, 16, 2)
-__STRUCTN (uint, 8, 2)
-__STRUCTN (uint, 16, 2)
-__STRUCTN (float, 16, 2)
-__STRUCTN (poly, 8, 2)
-__STRUCTN (poly, 16, 2)
-/* 3-element structs.  */
-__STRUCTN (int, 8, 3)
-__STRUCTN (int, 16, 3)
-__STRUCTN (int, 32, 3)
-__STRUCTN (int, 64, 3)
-__STRUCTN (uint, 8, 3)
-__STRUCTN (uint, 16, 3)
-__STRUCTN (uint, 32, 3)
-__STRUCTN (uint, 64, 3)
-__STRUCTN (float, 16, 3)
-__STRUCTN (float, 32, 3)
-__STRUCTN (float, 64, 3)
-__STRUCTN (poly, 8, 3)
-__STRUCTN (poly, 16, 3)
-/* 4-element structs.  */
-__STRUCTN (int, 8, 4)
-__STRUCTN (int, 64, 4)
-__STRUCTN (uint, 8, 4)
-__STRUCTN (uint, 64, 4)
-__STRUCTN (poly, 8, 4)
-__STRUCTN (float, 64, 4)
-#undef __STRUCTN
-
-
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_f16 (float16_t *__ptr, float16x4x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  float16x8x2_t __temp;
-  __temp.val[0] = vcombine_f16 (__val.val[0],
-				vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f16 (__val.val[1],
-				vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanev4hf ((__builtin_aarch64_simd_hf *) __ptr, __o,
+  __builtin_aarch64_st2_lanev4hf ((__builtin_aarch64_simd_hf *) __ptr, __val,
 				  __lane);
 }
 
@@ -9225,14 +8544,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_f32 (float32_t *__ptr, float32x2x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  float32x4x2_t __temp;
-  __temp.val[0] = vcombine_f32 (__val.val[0],
-				vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f32 (__val.val[1],
-				vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanev2sf ((__builtin_aarch64_simd_sf *) __ptr, __o,
+  __builtin_aarch64_st2_lanev2sf ((__builtin_aarch64_simd_sf *) __ptr, __val,
 				  __lane);
 }
 
@@ -9240,14 +8552,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_f64 (float64_t *__ptr, float64x1x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  float64x2x2_t __temp;
-  __temp.val[0] = vcombine_f64 (__val.val[0],
-				vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f64 (__val.val[1],
-				vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanedf ((__builtin_aarch64_simd_df *) __ptr, __o,
+  __builtin_aarch64_st2_lanedf ((__builtin_aarch64_simd_df *) __ptr, __val,
 				__lane);
 }
 
@@ -9255,59 +8560,31 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_p8 (poly8_t *__ptr, poly8x8x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly8x16x2_t __temp;
-  __temp.val[0] = vcombine_p8 (__val.val[0],
-			       vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p8 (__val.val[1],
-			       vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st2_lanev8qi_sps ((__builtin_aarch64_simd_qi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_p16 (poly16_t *__ptr, poly16x4x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly16x8x2_t __temp;
-  __temp.val[0] = vcombine_p16 (__val.val[0],
-				vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p16 (__val.val[1],
-				vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st2_lanev4hi_sps ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_p64 (poly64_t *__ptr, poly64x1x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly64x2x2_t __temp;
-  __temp.val[0] = vcombine_p64 (__val.val[0],
-				vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p64 (__val.val[1],
-				vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanedi ((__builtin_aarch64_simd_di *) __ptr, __o,
-				__lane);
+  __builtin_aarch64_st2_lanedi_sps ((__builtin_aarch64_simd_di *) __ptr,
+				    __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_s8 (int8_t *__ptr, int8x8x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  int8x16x2_t __temp;
-  __temp.val[0] = vcombine_s8 (__val.val[0],
-			       vcreate_s8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s8 (__val.val[1],
-			       vcreate_s8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
+  __builtin_aarch64_st2_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __val,
 				  __lane);
 }
 
@@ -9315,14 +8592,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_s16 (int16_t *__ptr, int16x4x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  int16x8x2_t __temp;
-  __temp.val[0] = vcombine_s16 (__val.val[0],
-				vcreate_s16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s16 (__val.val[1],
-				vcreate_s16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
+  __builtin_aarch64_st2_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __val,
 				  __lane);
 }
 
@@ -9330,14 +8600,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_s32 (int32_t *__ptr, int32x2x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  int32x4x2_t __temp;
-  __temp.val[0] = vcombine_s32 (__val.val[0],
-				vcreate_s32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s32 (__val.val[1],
-				vcreate_s32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanev2si ((__builtin_aarch64_simd_si *) __ptr, __o,
+  __builtin_aarch64_st2_lanev2si ((__builtin_aarch64_simd_si *) __ptr, __val,
 				  __lane);
 }
 
@@ -9345,14 +8608,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_s64 (int64_t *__ptr, int64x1x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  int64x2x2_t __temp;
-  __temp.val[0] = vcombine_s64 (__val.val[0],
-				vcreate_s64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s64 (__val.val[1],
-				vcreate_s64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanedi ((__builtin_aarch64_simd_di *) __ptr, __o,
+  __builtin_aarch64_st2_lanedi ((__builtin_aarch64_simd_di *) __ptr, __val,
 				__lane);
 }
 
@@ -9360,69 +8616,39 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_u8 (uint8_t *__ptr, uint8x8x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint8x16x2_t __temp;
-  __temp.val[0] = vcombine_u8 (__val.val[0],
-			       vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u8 (__val.val[1],
-			       vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st2_lanev8qi_sus ((__builtin_aarch64_simd_qi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_u16 (uint16_t *__ptr, uint16x4x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint16x8x2_t __temp;
-  __temp.val[0] = vcombine_u16 (__val.val[0],
-				vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u16 (__val.val[1],
-				vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st2_lanev4hi_sus ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_u32 (uint32_t *__ptr, uint32x2x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint32x4x2_t __temp;
-  __temp.val[0] = vcombine_u32 (__val.val[0],
-				vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u32 (__val.val[1],
-				vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanev2si ((__builtin_aarch64_simd_si *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st2_lanev2si_sus ((__builtin_aarch64_simd_si *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_u64 (uint64_t *__ptr, uint64x1x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint64x2x2_t __temp;
-  __temp.val[0] = vcombine_u64 (__val.val[0],
-				vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u64 (__val.val[1],
-				vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanedi ((__builtin_aarch64_simd_di *) __ptr, __o,
-				__lane);
+  __builtin_aarch64_st2_lanedi_sus ((__builtin_aarch64_simd_di *) __ptr, __val,
+				    __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_f16 (float16_t *__ptr, float16x8x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev8hf ((__builtin_aarch64_simd_hf *) __ptr, __o,
+  __builtin_aarch64_st2_lanev8hf ((__builtin_aarch64_simd_hf *) __ptr, __val,
 				  __lane);
 }
 
@@ -9430,9 +8656,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_f32 (float32_t *__ptr, float32x4x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev4sf ((__builtin_aarch64_simd_sf *) __ptr, __o,
+  __builtin_aarch64_st2_lanev4sf ((__builtin_aarch64_simd_sf *) __ptr, __val,
 				  __lane);
 }
 
@@ -9440,9 +8664,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_f64 (float64_t *__ptr, float64x2x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev2df ((__builtin_aarch64_simd_df *) __ptr, __o,
+  __builtin_aarch64_st2_lanev2df ((__builtin_aarch64_simd_df *) __ptr, __val,
 				  __lane);
 }
 
@@ -9450,39 +8672,31 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_p8 (poly8_t *__ptr, poly8x16x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				   __lane);
+  __builtin_aarch64_st2_lanev16qi_sps ((__builtin_aarch64_simd_qi *) __ptr,
+				       __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_p16 (poly16_t *__ptr, poly16x8x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st2_lanev8hi_sps ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_p64 (poly64_t *__ptr, poly64x2x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st2_lanev2di_sps ((__builtin_aarch64_simd_di *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_s8 (int8_t *__ptr, int8x16x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
+  __builtin_aarch64_st2_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __val,
 				   __lane);
 }
 
@@ -9490,9 +8704,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_s16 (int16_t *__ptr, int16x8x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
+  __builtin_aarch64_st2_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __val,
 				  __lane);
 }
 
@@ -9500,9 +8712,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_s32 (int32_t *__ptr, int32x4x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev4si ((__builtin_aarch64_simd_si *) __ptr, __o,
+  __builtin_aarch64_st2_lanev4si ((__builtin_aarch64_simd_si *) __ptr, __val,
 				  __lane);
 }
 
@@ -9510,9 +8720,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_s64 (int64_t *__ptr, int64x2x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __o,
+  __builtin_aarch64_st2_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __val,
 				  __lane);
 }
 
@@ -9520,56 +8728,39 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_u8 (uint8_t *__ptr, uint8x16x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				   __lane);
+  __builtin_aarch64_st2_lanev16qi_sus ((__builtin_aarch64_simd_qi *) __ptr,
+				       __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_u16 (uint16_t *__ptr, uint16x8x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st2_lanev8hi_sus ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_u32 (uint32_t *__ptr, uint32x4x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev4si ((__builtin_aarch64_simd_si *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st2_lanev4si_sus ((__builtin_aarch64_simd_si *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_u64 (uint64_t *__ptr, uint64x2x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st2_lanev2di_sus ((__builtin_aarch64_simd_di *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_f16 (float16_t *__ptr, float16x4x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  float16x8x3_t __temp;
-  __temp.val[0] = vcombine_f16 (__val.val[0],
-				vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f16 (__val.val[1],
-				vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f16 (__val.val[2],
-				vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanev4hf ((__builtin_aarch64_simd_hf *) __ptr, __o,
+  __builtin_aarch64_st3_lanev4hf ((__builtin_aarch64_simd_hf *) __ptr, __val,
 				  __lane);
 }
 
@@ -9577,16 +8768,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_f32 (float32_t *__ptr, float32x2x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  float32x4x3_t __temp;
-  __temp.val[0] = vcombine_f32 (__val.val[0],
-				vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f32 (__val.val[1],
-				vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f32 (__val.val[2],
-				vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanev2sf ((__builtin_aarch64_simd_sf *) __ptr, __o,
+  __builtin_aarch64_st3_lanev2sf ((__builtin_aarch64_simd_sf *) __ptr, __val,
 				  __lane);
 }
 
@@ -9594,16 +8776,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_f64 (float64_t *__ptr, float64x1x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  float64x2x3_t __temp;
-  __temp.val[0] = vcombine_f64 (__val.val[0],
-				vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f64 (__val.val[1],
-				vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f64 (__val.val[2],
-				vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanedf ((__builtin_aarch64_simd_df *) __ptr, __o,
+  __builtin_aarch64_st3_lanedf ((__builtin_aarch64_simd_df *) __ptr, __val,
 				__lane);
 }
 
@@ -9611,67 +8784,31 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_p8 (poly8_t *__ptr, poly8x8x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly8x16x3_t __temp;
-  __temp.val[0] = vcombine_p8 (__val.val[0],
-			       vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p8 (__val.val[1],
-			       vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p8 (__val.val[2],
-			       vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st3_lanev8qi_sps ((__builtin_aarch64_simd_qi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_p16 (poly16_t *__ptr, poly16x4x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly16x8x3_t __temp;
-  __temp.val[0] = vcombine_p16 (__val.val[0],
-				vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p16 (__val.val[1],
-				vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p16 (__val.val[2],
-				vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st3_lanev4hi_sps ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_p64 (poly64_t *__ptr, poly64x1x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly64x2x3_t __temp;
-  __temp.val[0] = vcombine_p64 (__val.val[0],
-				vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p64 (__val.val[1],
-				vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p64 (__val.val[2],
-				vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanedi ((__builtin_aarch64_simd_di *) __ptr, __o,
-				__lane);
+  __builtin_aarch64_st3_lanedi_sps ((__builtin_aarch64_simd_di *) __ptr, __val,
+				    __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_s8 (int8_t *__ptr, int8x8x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  int8x16x3_t __temp;
-  __temp.val[0] = vcombine_s8 (__val.val[0],
-			       vcreate_s8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s8 (__val.val[1],
-			       vcreate_s8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_s8 (__val.val[2],
-			       vcreate_s8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
+  __builtin_aarch64_st3_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __val,
 				  __lane);
 }
 
@@ -9679,16 +8816,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_s16 (int16_t *__ptr, int16x4x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  int16x8x3_t __temp;
-  __temp.val[0] = vcombine_s16 (__val.val[0],
-				vcreate_s16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s16 (__val.val[1],
-				vcreate_s16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_s16 (__val.val[2],
-				vcreate_s16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
+  __builtin_aarch64_st3_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __val,
 				  __lane);
 }
 
@@ -9696,16 +8824,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_s32 (int32_t *__ptr, int32x2x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  int32x4x3_t __temp;
-  __temp.val[0] = vcombine_s32 (__val.val[0],
-				vcreate_s32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s32 (__val.val[1],
-				vcreate_s32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_s32 (__val.val[2],
-				vcreate_s32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanev2si ((__builtin_aarch64_simd_si *) __ptr, __o,
+  __builtin_aarch64_st3_lanev2si ((__builtin_aarch64_simd_si *) __ptr, __val,
 				  __lane);
 }
 
@@ -9713,16 +8832,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_s64 (int64_t *__ptr, int64x1x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  int64x2x3_t __temp;
-  __temp.val[0] = vcombine_s64 (__val.val[0],
-				vcreate_s64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s64 (__val.val[1],
-				vcreate_s64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_s64 (__val.val[2],
-				vcreate_s64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanedi ((__builtin_aarch64_simd_di *) __ptr, __o,
+  __builtin_aarch64_st3_lanedi ((__builtin_aarch64_simd_di *) __ptr, __val,
 				__lane);
 }
 
@@ -9730,77 +8840,39 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_u8 (uint8_t *__ptr, uint8x8x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint8x16x3_t __temp;
-  __temp.val[0] = vcombine_u8 (__val.val[0],
-			       vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u8 (__val.val[1],
-			       vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u8 (__val.val[2],
-			       vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st3_lanev8qi_sus ((__builtin_aarch64_simd_qi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_u16 (uint16_t *__ptr, uint16x4x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint16x8x3_t __temp;
-  __temp.val[0] = vcombine_u16 (__val.val[0],
-				vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u16 (__val.val[1],
-				vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u16 (__val.val[2],
-				vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st3_lanev4hi_sus ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_u32 (uint32_t *__ptr, uint32x2x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint32x4x3_t __temp;
-  __temp.val[0] = vcombine_u32 (__val.val[0],
-				vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u32 (__val.val[1],
-				vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u32 (__val.val[2],
-				vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanev2si ((__builtin_aarch64_simd_si *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st3_lanev2si_sus ((__builtin_aarch64_simd_si *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_u64 (uint64_t *__ptr, uint64x1x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint64x2x3_t __temp;
-  __temp.val[0] = vcombine_u64 (__val.val[0],
-				vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u64 (__val.val[1],
-				vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u64 (__val.val[2],
-				vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanedi ((__builtin_aarch64_simd_di *) __ptr, __o,
-				__lane);
+  __builtin_aarch64_st3_lanedi_sus ((__builtin_aarch64_simd_di *) __ptr, __val,
+				    __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_f16 (float16_t *__ptr, float16x8x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev8hf ((__builtin_aarch64_simd_hf *) __ptr, __o,
+  __builtin_aarch64_st3_lanev8hf ((__builtin_aarch64_simd_hf *) __ptr, __val,
 				  __lane);
 }
 
@@ -9808,9 +8880,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_f32 (float32_t *__ptr, float32x4x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev4sf ((__builtin_aarch64_simd_sf *) __ptr, __o,
+  __builtin_aarch64_st3_lanev4sf ((__builtin_aarch64_simd_sf *) __ptr, __val,
 				  __lane);
 }
 
@@ -9818,9 +8888,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_f64 (float64_t *__ptr, float64x2x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev2df ((__builtin_aarch64_simd_df *) __ptr, __o,
+  __builtin_aarch64_st3_lanev2df ((__builtin_aarch64_simd_df *) __ptr, __val,
 				  __lane);
 }
 
@@ -9828,39 +8896,31 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_p8 (poly8_t *__ptr, poly8x16x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				   __lane);
+  __builtin_aarch64_st3_lanev16qi_sps ((__builtin_aarch64_simd_qi *) __ptr,
+				       __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_p16 (poly16_t *__ptr, poly16x8x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st3_lanev8hi_sps ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_p64 (poly64_t *__ptr, poly64x2x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st3_lanev2di_sps ((__builtin_aarch64_simd_di *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_s8 (int8_t *__ptr, int8x16x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
+  __builtin_aarch64_st3_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __val,
 				   __lane);
 }
 
@@ -9868,9 +8928,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_s16 (int16_t *__ptr, int16x8x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
+  __builtin_aarch64_st3_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __val,
 				  __lane);
 }
 
@@ -9878,9 +8936,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_s32 (int32_t *__ptr, int32x4x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev4si ((__builtin_aarch64_simd_si *) __ptr, __o,
+  __builtin_aarch64_st3_lanev4si ((__builtin_aarch64_simd_si *) __ptr, __val,
 				  __lane);
 }
 
@@ -9888,9 +8944,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_s64 (int64_t *__ptr, int64x2x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __o,
+  __builtin_aarch64_st3_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __val,
 				  __lane);
 }
 
@@ -9898,58 +8952,39 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_u8 (uint8_t *__ptr, uint8x16x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				   __lane);
+  __builtin_aarch64_st3_lanev16qi_sus ((__builtin_aarch64_simd_qi *) __ptr,
+				       __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_u16 (uint16_t *__ptr, uint16x8x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st3_lanev8hi_sus ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_u32 (uint32_t *__ptr, uint32x4x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev4si ((__builtin_aarch64_simd_si *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st3_lanev4si_sus ((__builtin_aarch64_simd_si *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_u64 (uint64_t *__ptr, uint64x2x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st3_lanev2di_sus ((__builtin_aarch64_simd_di *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_f16 (float16_t *__ptr, float16x4x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  float16x8x4_t __temp;
-  __temp.val[0] = vcombine_f16 (__val.val[0],
-				vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f16 (__val.val[1],
-				vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f16 (__val.val[2],
-				vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_f16 (__val.val[3],
-				vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanev4hf ((__builtin_aarch64_simd_hf *) __ptr, __o,
+  __builtin_aarch64_st4_lanev4hf ((__builtin_aarch64_simd_hf *) __ptr, __val,
 				  __lane);
 }
 
@@ -9957,18 +8992,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_f32 (float32_t *__ptr, float32x2x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  float32x4x4_t __temp;
-  __temp.val[0] = vcombine_f32 (__val.val[0],
-				vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f32 (__val.val[1],
-				vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f32 (__val.val[2],
-				vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_f32 (__val.val[3],
-				vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanev2sf ((__builtin_aarch64_simd_sf *) __ptr, __o,
+  __builtin_aarch64_st4_lanev2sf ((__builtin_aarch64_simd_sf *) __ptr, __val,
 				  __lane);
 }
 
@@ -9976,18 +9000,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_f64 (float64_t *__ptr, float64x1x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  float64x2x4_t __temp;
-  __temp.val[0] = vcombine_f64 (__val.val[0],
-				vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f64 (__val.val[1],
-				vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f64 (__val.val[2],
-				vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_f64 (__val.val[3],
-				vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanedf ((__builtin_aarch64_simd_df *) __ptr, __o,
+  __builtin_aarch64_st4_lanedf ((__builtin_aarch64_simd_df *) __ptr, __val,
 				__lane);
 }
 
@@ -9995,75 +9008,31 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_p8 (poly8_t *__ptr, poly8x8x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly8x16x4_t __temp;
-  __temp.val[0] = vcombine_p8 (__val.val[0],
-			       vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p8 (__val.val[1],
-			       vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p8 (__val.val[2],
-			       vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_p8 (__val.val[3],
-			       vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st4_lanev8qi_sps ((__builtin_aarch64_simd_qi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_p16 (poly16_t *__ptr, poly16x4x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly16x8x4_t __temp;
-  __temp.val[0] = vcombine_p16 (__val.val[0],
-				vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p16 (__val.val[1],
-				vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p16 (__val.val[2],
-				vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_p16 (__val.val[3],
-				vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st4_lanev4hi_sps ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_p64 (poly64_t *__ptr, poly64x1x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly64x2x4_t __temp;
-  __temp.val[0] = vcombine_p64 (__val.val[0],
-				vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p64 (__val.val[1],
-				vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p64 (__val.val[2],
-				vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_p64 (__val.val[3],
-				vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanedi ((__builtin_aarch64_simd_di *) __ptr, __o,
-				__lane);
+  __builtin_aarch64_st4_lanedi_sps ((__builtin_aarch64_simd_di *) __ptr, __val,
+				    __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_s8 (int8_t *__ptr, int8x8x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  int8x16x4_t __temp;
-  __temp.val[0] = vcombine_s8 (__val.val[0],
-			       vcreate_s8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s8 (__val.val[1],
-			       vcreate_s8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_s8 (__val.val[2],
-			       vcreate_s8 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_s8 (__val.val[3],
-			       vcreate_s8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
+  __builtin_aarch64_st4_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __val,
 				  __lane);
 }
 
@@ -10071,18 +9040,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_s16 (int16_t *__ptr, int16x4x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  int16x8x4_t __temp;
-  __temp.val[0] = vcombine_s16 (__val.val[0],
-				vcreate_s16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s16 (__val.val[1],
-				vcreate_s16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_s16 (__val.val[2],
-				vcreate_s16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_s16 (__val.val[3],
-				vcreate_s16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
+  __builtin_aarch64_st4_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __val,
 				  __lane);
 }
 
@@ -10090,18 +9048,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_s32 (int32_t *__ptr, int32x2x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  int32x4x4_t __temp;
-  __temp.val[0] = vcombine_s32 (__val.val[0],
-				vcreate_s32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s32 (__val.val[1],
-				vcreate_s32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_s32 (__val.val[2],
-				vcreate_s32 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_s32 (__val.val[3],
-				vcreate_s32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanev2si ((__builtin_aarch64_simd_si *) __ptr, __o,
+  __builtin_aarch64_st4_lanev2si ((__builtin_aarch64_simd_si *) __ptr, __val,
 				  __lane);
 }
 
@@ -10109,18 +9056,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_s64 (int64_t *__ptr, int64x1x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  int64x2x4_t __temp;
-  __temp.val[0] = vcombine_s64 (__val.val[0],
-				vcreate_s64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_s64 (__val.val[1],
-				vcreate_s64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_s64 (__val.val[2],
-				vcreate_s64 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_s64 (__val.val[3],
-				vcreate_s64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanedi ((__builtin_aarch64_simd_di *) __ptr, __o,
+  __builtin_aarch64_st4_lanedi ((__builtin_aarch64_simd_di *) __ptr, __val,
 				__lane);
 }
 
@@ -10128,85 +9064,39 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_u8 (uint8_t *__ptr, uint8x8x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint8x16x4_t __temp;
-  __temp.val[0] = vcombine_u8 (__val.val[0],
-			       vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u8 (__val.val[1],
-			       vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u8 (__val.val[2],
-			       vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u8 (__val.val[3],
-			       vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanev8qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st4_lanev8qi_sus ((__builtin_aarch64_simd_qi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_u16 (uint16_t *__ptr, uint16x4x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint16x8x4_t __temp;
-  __temp.val[0] = vcombine_u16 (__val.val[0],
-				vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u16 (__val.val[1],
-				vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u16 (__val.val[2],
-				vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u16 (__val.val[3],
-				vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanev4hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st4_lanev4hi_sus ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_u32 (uint32_t *__ptr, uint32x2x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint32x4x4_t __temp;
-  __temp.val[0] = vcombine_u32 (__val.val[0],
-				vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u32 (__val.val[1],
-				vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u32 (__val.val[2],
-				vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u32 (__val.val[3],
-				vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanev2si ((__builtin_aarch64_simd_si *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st4_lanev2si_sus ((__builtin_aarch64_simd_si *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_u64 (uint64_t *__ptr, uint64x1x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint64x2x4_t __temp;
-  __temp.val[0] = vcombine_u64 (__val.val[0],
-				vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u64 (__val.val[1],
-				vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u64 (__val.val[2],
-				vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u64 (__val.val[3],
-				vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanedi ((__builtin_aarch64_simd_di *) __ptr, __o,
-				__lane);
+  __builtin_aarch64_st4_lanedi_sus ((__builtin_aarch64_simd_di *) __ptr, __val,
+				    __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_f16 (float16_t *__ptr, float16x8x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev8hf ((__builtin_aarch64_simd_hf *) __ptr, __o,
+  __builtin_aarch64_st4_lanev8hf ((__builtin_aarch64_simd_hf *) __ptr, __val,
 				  __lane);
 }
 
@@ -10214,9 +9104,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_f32 (float32_t *__ptr, float32x4x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev4sf ((__builtin_aarch64_simd_sf *) __ptr, __o,
+  __builtin_aarch64_st4_lanev4sf ((__builtin_aarch64_simd_sf *) __ptr, __val,
 				  __lane);
 }
 
@@ -10224,9 +9112,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_f64 (float64_t *__ptr, float64x2x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev2df ((__builtin_aarch64_simd_df *) __ptr, __o,
+  __builtin_aarch64_st4_lanev2df ((__builtin_aarch64_simd_df *) __ptr, __val,
 				  __lane);
 }
 
@@ -10234,39 +9120,31 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_p8 (poly8_t *__ptr, poly8x16x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				   __lane);
+  __builtin_aarch64_st4_lanev16qi_sps ((__builtin_aarch64_simd_qi *) __ptr,
+				       __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_p16 (poly16_t *__ptr, poly16x8x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st4_lanev8hi_sps ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_p64 (poly64_t *__ptr, poly64x2x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st4_lanev2di_sps ((__builtin_aarch64_simd_di *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_s8 (int8_t *__ptr, int8x16x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
+  __builtin_aarch64_st4_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __val,
 				   __lane);
 }
 
@@ -10274,9 +9152,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_s16 (int16_t *__ptr, int16x8x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
+  __builtin_aarch64_st4_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __val,
 				  __lane);
 }
 
@@ -10284,9 +9160,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_s32 (int32_t *__ptr, int32x4x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev4si ((__builtin_aarch64_simd_si *) __ptr, __o,
+  __builtin_aarch64_st4_lanev4si ((__builtin_aarch64_simd_si *) __ptr, __val,
 				  __lane);
 }
 
@@ -10294,9 +9168,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_s64 (int64_t *__ptr, int64x2x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __o,
+  __builtin_aarch64_st4_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __val,
 				  __lane);
 }
 
@@ -10304,40 +9176,32 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_u8 (uint8_t *__ptr, uint8x16x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev16qi ((__builtin_aarch64_simd_qi *) __ptr, __o,
-				   __lane);
+  __builtin_aarch64_st4_lanev16qi_sus ((__builtin_aarch64_simd_qi *) __ptr,
+				       __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_u16 (uint16_t *__ptr, uint16x8x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev8hi ((__builtin_aarch64_simd_hi *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st4_lanev8hi_sus ((__builtin_aarch64_simd_hi *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_u32 (uint32_t *__ptr, uint32x4x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev4si ((__builtin_aarch64_simd_si *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st4_lanev4si_sus ((__builtin_aarch64_simd_si *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_u64 (uint64_t *__ptr, uint64x2x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev2di ((__builtin_aarch64_simd_di *) __ptr, __o,
-				  __lane);
+  __builtin_aarch64_st4_lanev2di_sus ((__builtin_aarch64_simd_di *) __ptr,
+				      __val, __lane);
 }
 
 __extension__ extern __inline int64_t
@@ -10554,12 +9418,10 @@ __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vtbl3_s8 (int8x8x3_t __tab, int8x8_t __idx)
 {
   int8x16x2_t __temp;
-  __builtin_aarch64_simd_oi __o;
   __temp.val[0] = vcombine_s8 (__tab.val[0], __tab.val[1]);
   __temp.val[1] = vcombine_s8 (__tab.val[2],
 			       vcreate_s8 (__AARCH64_UINT64_C (0x0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  return __builtin_aarch64_qtbl2v8qi (__o, __idx);
+  return __builtin_aarch64_qtbl2v8qi (__temp, __idx);
 }
 
 __extension__ extern __inline uint8x8_t
@@ -10567,12 +9429,10 @@ __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vtbl3_u8 (uint8x8x3_t __tab, uint8x8_t __idx)
 {
   uint8x16x2_t __temp;
-  __builtin_aarch64_simd_oi __o;
   __temp.val[0] = vcombine_u8 (__tab.val[0], __tab.val[1]);
   __temp.val[1] = vcombine_u8 (__tab.val[2],
 			       vcreate_u8 (__AARCH64_UINT64_C (0x0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  return (uint8x8_t)__builtin_aarch64_qtbl2v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl2v8qi_uuu (__temp, __idx);
 }
 
 __extension__ extern __inline poly8x8_t
@@ -10580,12 +9440,10 @@ __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vtbl3_p8 (poly8x8x3_t __tab, uint8x8_t __idx)
 {
   poly8x16x2_t __temp;
-  __builtin_aarch64_simd_oi __o;
   __temp.val[0] = vcombine_p8 (__tab.val[0], __tab.val[1]);
   __temp.val[1] = vcombine_p8 (__tab.val[2],
 			       vcreate_p8 (__AARCH64_UINT64_C (0x0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  return (poly8x8_t)__builtin_aarch64_qtbl2v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl2v8qi_ppu (__temp, __idx);
 }
 
 __extension__ extern __inline int8x8_t
@@ -10593,11 +9451,9 @@ __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vtbl4_s8 (int8x8x4_t __tab, int8x8_t __idx)
 {
   int8x16x2_t __temp;
-  __builtin_aarch64_simd_oi __o;
   __temp.val[0] = vcombine_s8 (__tab.val[0], __tab.val[1]);
   __temp.val[1] = vcombine_s8 (__tab.val[2], __tab.val[3]);
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  return __builtin_aarch64_qtbl2v8qi (__o, __idx);
+  return __builtin_aarch64_qtbl2v8qi (__temp, __idx);
 }
 
 __extension__ extern __inline uint8x8_t
@@ -10605,11 +9461,9 @@ __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vtbl4_u8 (uint8x8x4_t __tab, uint8x8_t __idx)
 {
   uint8x16x2_t __temp;
-  __builtin_aarch64_simd_oi __o;
   __temp.val[0] = vcombine_u8 (__tab.val[0], __tab.val[1]);
   __temp.val[1] = vcombine_u8 (__tab.val[2], __tab.val[3]);
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  return (uint8x8_t)__builtin_aarch64_qtbl2v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl2v8qi_uuu (__temp, __idx);
 }
 
 __extension__ extern __inline poly8x8_t
@@ -10617,11 +9471,9 @@ __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vtbl4_p8 (poly8x8x4_t __tab, uint8x8_t __idx)
 {
   poly8x16x2_t __temp;
-  __builtin_aarch64_simd_oi __o;
   __temp.val[0] = vcombine_p8 (__tab.val[0], __tab.val[1]);
   __temp.val[1] = vcombine_p8 (__tab.val[2], __tab.val[3]);
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  return(poly8x8_t)__builtin_aarch64_qtbl2v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl2v8qi_ppu (__temp, __idx);
 }
 
 __extension__ extern __inline int8x8_t
@@ -10838,21 +9690,21 @@ __extension__ extern __inline uint8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddv_u8 (uint8x8_t __a)
 {
-  return (uint8_t) __builtin_aarch64_reduc_plus_scal_v8qi ((int8x8_t) __a);
+  return __builtin_aarch64_reduc_plus_scal_v8qi_uu (__a);
 }
 
 __extension__ extern __inline uint16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddv_u16 (uint16x4_t __a)
 {
-  return (uint16_t) __builtin_aarch64_reduc_plus_scal_v4hi ((int16x4_t) __a);
+  return __builtin_aarch64_reduc_plus_scal_v4hi_uu (__a);
 }
 
 __extension__ extern __inline uint32_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddv_u32 (uint32x2_t __a)
 {
-  return (int32_t) __builtin_aarch64_reduc_plus_scal_v2si ((int32x2_t) __a);
+  return __builtin_aarch64_reduc_plus_scal_v2si_uu (__a);
 }
 
 __extension__ extern __inline int8_t
@@ -10887,28 +9739,28 @@ __extension__ extern __inline uint8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddvq_u8 (uint8x16_t __a)
 {
-  return (uint8_t) __builtin_aarch64_reduc_plus_scal_v16qi ((int8x16_t) __a);
+  return __builtin_aarch64_reduc_plus_scal_v16qi_uu (__a);
 }
 
 __extension__ extern __inline uint16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddvq_u16 (uint16x8_t __a)
 {
-  return (uint16_t) __builtin_aarch64_reduc_plus_scal_v8hi ((int16x8_t) __a);
+  return __builtin_aarch64_reduc_plus_scal_v8hi_uu (__a);
 }
 
 __extension__ extern __inline uint32_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddvq_u32 (uint32x4_t __a)
 {
-  return (uint32_t) __builtin_aarch64_reduc_plus_scal_v4si ((int32x4_t) __a);
+  return __builtin_aarch64_reduc_plus_scal_v4si_uu (__a);
 }
 
 __extension__ extern __inline uint64_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vaddvq_u64 (uint64x2_t __a)
 {
-  return (uint64_t) __builtin_aarch64_reduc_plus_scal_v2di ((int64x2_t) __a);
+  return __builtin_aarch64_reduc_plus_scal_v2di_uu (__a);
 }
 
 __extension__ extern __inline float32_t
@@ -16103,16 +14955,16 @@ __extension__ extern __inline poly8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_p8 (const poly8_t *__a)
 {
-  return (poly8x8_t)
-    __builtin_aarch64_ld1v8qi ((const __builtin_aarch64_simd_qi *) __a);
+  return __builtin_aarch64_ld1v8qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_p16 (const poly16_t *__a)
 {
-  return (poly16x4_t)
-    __builtin_aarch64_ld1v4hi ((const __builtin_aarch64_simd_hi *) __a);
+  return __builtin_aarch64_ld1v4hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly64x1_t
@@ -16154,24 +15006,24 @@ __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u8 (const uint8_t *__a)
 {
-  return (uint8x8_t)
-    __builtin_aarch64_ld1v8qi ((const __builtin_aarch64_simd_qi *) __a);
+  return __builtin_aarch64_ld1v8qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u16 (const uint16_t *__a)
 {
-  return (uint16x4_t)
-    __builtin_aarch64_ld1v4hi ((const __builtin_aarch64_simd_hi *) __a);
+  return __builtin_aarch64_ld1v4hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u32 (const uint32_t *__a)
 {
-  return (uint32x2_t)
-    __builtin_aarch64_ld1v2si ((const __builtin_aarch64_simd_si *) __a);
+  return __builtin_aarch64_ld1v2si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x1_t
@@ -16187,366 +15039,211 @@ __extension__ extern __inline uint8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u8_x3 (const uint8_t *__a)
 {
-  uint8x8x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = (__builtin_aarch64_simd_ci)__builtin_aarch64_ld1x3v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  __i.val[0] = (uint8x8_t) __builtin_aarch64_get_dregciv8qi  (__o, 0);
-  __i.val[1] = (uint8x8_t) __builtin_aarch64_get_dregciv8qi  (__o, 1);
-  __i.val[2] = (uint8x8_t) __builtin_aarch64_get_dregciv8qi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v8qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s8_x3 (const int8_t *__a)
 {
-  int8x8x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  __i.val[0] = (int8x8_t) __builtin_aarch64_get_dregciv8qi  (__o, 0);
-  __i.val[1] = (int8x8_t) __builtin_aarch64_get_dregciv8qi  (__o, 1);
-  __i.val[2] = (int8x8_t) __builtin_aarch64_get_dregciv8qi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v8qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u16_x3 (const uint16_t *__a)
 {
-  uint16x4x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  __i.val[0] = (uint16x4_t) __builtin_aarch64_get_dregciv4hi  (__o, 0);
-  __i.val[1] = (uint16x4_t) __builtin_aarch64_get_dregciv4hi  (__o, 1);
-  __i.val[2] = (uint16x4_t) __builtin_aarch64_get_dregciv4hi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v4hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s16_x3 (const int16_t *__a)
 {
-  int16x4x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  __i.val[0] = (int16x4_t) __builtin_aarch64_get_dregciv4hi  (__o, 0);
-  __i.val[1] = (int16x4_t) __builtin_aarch64_get_dregciv4hi  (__o, 1);
-  __i.val[2] = (int16x4_t) __builtin_aarch64_get_dregciv4hi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v4hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u32_x3 (const uint32_t *__a)
 {
-  uint32x2x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v2si ((const __builtin_aarch64_simd_si *) __a);
-  __i.val[0] = (uint32x2_t) __builtin_aarch64_get_dregciv2si  (__o, 0);
-  __i.val[1] = (uint32x2_t) __builtin_aarch64_get_dregciv2si  (__o, 1);
-  __i.val[2] = (uint32x2_t) __builtin_aarch64_get_dregciv2si  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v2si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline int32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s32_x3 (const int32_t *__a)
 {
-  int32x2x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v2si ((const __builtin_aarch64_simd_si *) __a);
-  __i.val[0] = (int32x2_t) __builtin_aarch64_get_dregciv2si  (__o, 0);
-  __i.val[1] = (int32x2_t) __builtin_aarch64_get_dregciv2si  (__o, 1);
-  __i.val[2] = (int32x2_t) __builtin_aarch64_get_dregciv2si  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v2si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u64_x3 (const uint64_t *__a)
 {
-  uint64x1x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3di ((const __builtin_aarch64_simd_di *) __a);
-  __i.val[0] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __i.val[1] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __i.val[2] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s64_x3 (const int64_t *__a)
 {
-  int64x1x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3di ((const __builtin_aarch64_simd_di *) __a);
-  __i.val[0] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __i.val[1] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __i.val[2] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-
-  return __i;
+  return __builtin_aarch64_ld1x3di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_f16_x3 (const float16_t *__a)
 {
-  float16x4x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v4hf ((const __builtin_aarch64_simd_hf *) __a);
-  __i.val[0] = (float16x4_t) __builtin_aarch64_get_dregciv4hf  (__o, 0);
-  __i.val[1] = (float16x4_t) __builtin_aarch64_get_dregciv4hf  (__o, 1);
-  __i.val[2] = (float16x4_t) __builtin_aarch64_get_dregciv4hf  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v4hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_f32_x3 (const float32_t *__a)
 {
-  float32x2x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v2sf ((const __builtin_aarch64_simd_sf *) __a);
-  __i.val[0] = (float32x2_t) __builtin_aarch64_get_dregciv2sf  (__o, 0);
-  __i.val[1] = (float32x2_t) __builtin_aarch64_get_dregciv2sf  (__o, 1);
-  __i.val[2] = (float32x2_t) __builtin_aarch64_get_dregciv2sf  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v2sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_f64_x3 (const float64_t *__a)
 {
-  float64x1x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3df ((const __builtin_aarch64_simd_df *) __a);
-  __i.val[0] = (float64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __i.val[1] = (float64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __i.val[2] = (float64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline poly8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_p8_x3 (const poly8_t *__a)
 {
-  poly8x8x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  __i.val[0] = (poly8x8_t) __builtin_aarch64_get_dregciv8qi  (__o, 0);
-  __i.val[1] = (poly8x8_t) __builtin_aarch64_get_dregciv8qi  (__o, 1);
-  __i.val[2] = (poly8x8_t) __builtin_aarch64_get_dregciv8qi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v8qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_p16_x3 (const poly16_t *__a)
 {
-  poly16x4x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  __i.val[0] = (poly16x4_t) __builtin_aarch64_get_dregciv4hi  (__o, 0);
-  __i.val[1] = (poly16x4_t) __builtin_aarch64_get_dregciv4hi  (__o, 1);
-  __i.val[2] = (poly16x4_t) __builtin_aarch64_get_dregciv4hi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v4hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_p64_x3 (const poly64_t *__a)
 {
-  poly64x1x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3di ((const __builtin_aarch64_simd_di *) __a);
-  __i.val[0] = (poly64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __i.val[1] = (poly64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __i.val[2] = (poly64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-
-return __i;
+  return __builtin_aarch64_ld1x3di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u8_x3 (const uint8_t *__a)
 {
-  uint8x16x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  __i.val[0] = (uint8x16_t) __builtin_aarch64_get_qregciv16qi  (__o, 0);
-  __i.val[1] = (uint8x16_t) __builtin_aarch64_get_qregciv16qi  (__o, 1);
-  __i.val[2] = (uint8x16_t) __builtin_aarch64_get_qregciv16qi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v16qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s8_x3 (const int8_t *__a)
 {
-  int8x16x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  __i.val[0] = (int8x16_t) __builtin_aarch64_get_qregciv16qi  (__o, 0);
-  __i.val[1] = (int8x16_t) __builtin_aarch64_get_qregciv16qi  (__o, 1);
-  __i.val[2] = (int8x16_t) __builtin_aarch64_get_qregciv16qi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v16qi (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u16_x3 (const uint16_t *__a)
 {
-  uint16x8x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  __i.val[0] = (uint16x8_t) __builtin_aarch64_get_qregciv8hi  (__o, 0);
-  __i.val[1] = (uint16x8_t) __builtin_aarch64_get_qregciv8hi  (__o, 1);
-  __i.val[2] = (uint16x8_t) __builtin_aarch64_get_qregciv8hi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v8hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s16_x3 (const int16_t *__a)
 {
-  int16x8x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  __i.val[0] = (int16x8_t) __builtin_aarch64_get_qregciv8hi  (__o, 0);
-  __i.val[1] = (int16x8_t) __builtin_aarch64_get_qregciv8hi  (__o, 1);
-  __i.val[2] = (int16x8_t) __builtin_aarch64_get_qregciv8hi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v8hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u32_x3 (const uint32_t *__a)
 {
-  uint32x4x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v4si ((const __builtin_aarch64_simd_si *) __a);
-  __i.val[0] = (uint32x4_t) __builtin_aarch64_get_qregciv4si  (__o, 0);
-  __i.val[1] = (uint32x4_t) __builtin_aarch64_get_qregciv4si  (__o, 1);
-  __i.val[2] = (uint32x4_t) __builtin_aarch64_get_qregciv4si  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v4si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline int32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s32_x3 (const int32_t *__a)
 {
-  int32x4x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v4si ((const __builtin_aarch64_simd_si *) __a);
-  __i.val[0] = (int32x4_t) __builtin_aarch64_get_qregciv4si  (__o, 0);
-  __i.val[1] = (int32x4_t) __builtin_aarch64_get_qregciv4si  (__o, 1);
-  __i.val[2] = (int32x4_t) __builtin_aarch64_get_qregciv4si  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v4si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u64_x3 (const uint64_t *__a)
 {
-  uint64x2x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v2di ((const __builtin_aarch64_simd_di *) __a);
-  __i.val[0] = (uint64x2_t) __builtin_aarch64_get_qregciv2di  (__o, 0);
-  __i.val[1] = (uint64x2_t) __builtin_aarch64_get_qregciv2di  (__o, 1);
-  __i.val[2] = (uint64x2_t) __builtin_aarch64_get_qregciv2di  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v2di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s64_x3 (const int64_t *__a)
 {
-  int64x2x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v2di ((const __builtin_aarch64_simd_di *) __a);
-  __i.val[0] = (int64x2_t) __builtin_aarch64_get_qregciv2di  (__o, 0);
-  __i.val[1] = (int64x2_t) __builtin_aarch64_get_qregciv2di  (__o, 1);
-  __i.val[2] = (int64x2_t) __builtin_aarch64_get_qregciv2di  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v2di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_f16_x3 (const float16_t *__a)
 {
-  float16x8x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v8hf ((const __builtin_aarch64_simd_hf *) __a);
-  __i.val[0] = (float16x8_t) __builtin_aarch64_get_qregciv8hf  (__o, 0);
-  __i.val[1] = (float16x8_t) __builtin_aarch64_get_qregciv8hf  (__o, 1);
-  __i.val[2] = (float16x8_t) __builtin_aarch64_get_qregciv8hf  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v8hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_f32_x3 (const float32_t *__a)
 {
-  float32x4x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v4sf ((const __builtin_aarch64_simd_sf *) __a);
-  __i.val[0] = (float32x4_t) __builtin_aarch64_get_qregciv4sf  (__o, 0);
-  __i.val[1] = (float32x4_t) __builtin_aarch64_get_qregciv4sf  (__o, 1);
-  __i.val[2] = (float32x4_t) __builtin_aarch64_get_qregciv4sf  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v4sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_f64_x3 (const float64_t *__a)
 {
-  float64x2x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v2df ((const __builtin_aarch64_simd_df *) __a);
-  __i.val[0] = (float64x2_t) __builtin_aarch64_get_qregciv2df  (__o, 0);
-  __i.val[1] = (float64x2_t) __builtin_aarch64_get_qregciv2df  (__o, 1);
-  __i.val[2] = (float64x2_t) __builtin_aarch64_get_qregciv2df  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v2df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline poly8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p8_x3 (const poly8_t *__a)
 {
-  poly8x16x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  __i.val[0] = (poly8x16_t) __builtin_aarch64_get_qregciv16qi  (__o, 0);
-  __i.val[1] = (poly8x16_t) __builtin_aarch64_get_qregciv16qi  (__o, 1);
-  __i.val[2] = (poly8x16_t) __builtin_aarch64_get_qregciv16qi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v16qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p16_x3 (const poly16_t *__a)
 {
-  poly16x8x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  __i.val[0] = (poly16x8_t) __builtin_aarch64_get_qregciv8hi  (__o, 0);
-  __i.val[1] = (poly16x8_t) __builtin_aarch64_get_qregciv8hi  (__o, 1);
-  __i.val[2] = (poly16x8_t) __builtin_aarch64_get_qregciv8hi  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v8hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p64_x3 (const poly64_t *__a)
 {
-  poly64x2x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v2di ((const __builtin_aarch64_simd_di *) __a);
-  __i.val[0] = (poly64x2_t) __builtin_aarch64_get_qregciv2di  (__o, 0);
-  __i.val[1] = (poly64x2_t) __builtin_aarch64_get_qregciv2di  (__o, 1);
-  __i.val[2] = (poly64x2_t) __builtin_aarch64_get_qregciv2di  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v2di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 /* vld1q */
@@ -16576,24 +15273,24 @@ __extension__ extern __inline poly8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p8 (const poly8_t *__a)
 {
-  return (poly8x16_t)
-    __builtin_aarch64_ld1v16qi ((const __builtin_aarch64_simd_qi *) __a);
+  return __builtin_aarch64_ld1v16qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p16 (const poly16_t *__a)
 {
-  return (poly16x8_t)
-    __builtin_aarch64_ld1v8hi ((const __builtin_aarch64_simd_hi *) __a);
+  return __builtin_aarch64_ld1v8hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p64 (const poly64_t *__a)
 {
-  return (poly64x2_t)
-    __builtin_aarch64_ld1v2di ((const __builtin_aarch64_simd_di *) __a);
+  return __builtin_aarch64_ld1v2di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int8x16_t
@@ -16628,368 +15325,243 @@ __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u8 (const uint8_t *__a)
 {
-  return (uint8x16_t)
-    __builtin_aarch64_ld1v16qi ((const __builtin_aarch64_simd_qi *) __a);
+  return __builtin_aarch64_ld1v16qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u8_x2 (const uint8_t *__a)
 {
-  uint8x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 0);
-  ret.val[1] = (uint8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v8qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s8_x2 (const int8_t *__a)
 {
-  int8x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 0);
-  ret.val[1] = (int8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v8qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u16_x2 (const uint16_t *__a)
 {
-  uint16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 0);
-  ret.val[1] = (uint16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v4hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s16_x2 (const int16_t *__a)
 {
-  int16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 0);
-  ret.val[1] = (int16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v4hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u32_x2 (const uint32_t *__a)
 {
-  uint32x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 0);
-  ret.val[1] = (uint32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v2si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline int32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s32_x2 (const int32_t *__a)
 {
-  int32x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 0);
-  ret.val[1] = (int32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v2si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u64_x2 (const uint64_t *__a)
 {
-  uint64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x1_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  ret.val[1] = (uint64x1_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s64_x2 (const int64_t *__a)
 {
-  int64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x1_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  ret.val[1] = (int64x1_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_f16_x2 (const float16_t *__a)
 {
-  float16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v4hf ((const __builtin_aarch64_simd_hf *) __a);
-  ret.val[0] = (float16x4_t) __builtin_aarch64_get_dregoiv4hf (__o, 0);
-  ret.val[1] = (float16x4_t) __builtin_aarch64_get_dregoiv4hf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v4hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_f32_x2 (const float32_t *__a)
 {
-  float32x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v2sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x2_t) __builtin_aarch64_get_dregoiv2sf (__o, 0);
-  ret.val[1] = (float32x2_t) __builtin_aarch64_get_dregoiv2sf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v2sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_f64_x2 (const float64_t *__a)
 {
-  float64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2df ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x1_t) {__builtin_aarch64_get_dregoidf (__o, 0)};
-  ret.val[1] = (float64x1_t) {__builtin_aarch64_get_dregoidf (__o, 1)};
-  return ret;
+  return __builtin_aarch64_ld1x2df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline poly8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_p8_x2 (const poly8_t *__a)
 {
-  poly8x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 0);
-  ret.val[1] = (poly8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v8qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_p16_x2 (const poly16_t *__a)
 {
-  poly16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 0);
-  ret.val[1] = (poly16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v4hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_p64_x2 (const poly64_t *__a)
 {
-  poly64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x1_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  ret.val[1] = (poly64x1_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u8_x2 (const uint8_t *__a)
 {
-  uint8x16x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 0);
-  ret.val[1] = (uint8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v16qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s8_x2 (const int8_t *__a)
 {
-  int8x16x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 0);
-  ret.val[1] = (int8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v16qi (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u16_x2 (const uint16_t *__a)
 {
-  uint16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 0);
-  ret.val[1] = (uint16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v8hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s16_x2 (const int16_t *__a)
 {
-  int16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 0);
-  ret.val[1] = (int16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v8hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u32_x2 (const uint32_t *__a)
 {
-  uint32x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (uint32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v4si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline int32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s32_x2 (const int32_t *__a)
 {
-  int32x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (int32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v4si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u64_x2 (const uint64_t *__a)
 {
-  uint64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 0);
-  ret.val[1] = (uint64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v2di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s64_x2 (const int64_t *__a)
 {
-  int64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 0);
-  ret.val[1] = (int64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v2di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_f16_x2 (const float16_t *__a)
 {
-  float16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v8hf ((const __builtin_aarch64_simd_hf *) __a);
-  ret.val[0] = (float16x8_t) __builtin_aarch64_get_qregoiv8hf (__o, 0);
-  ret.val[1] = (float16x8_t) __builtin_aarch64_get_qregoiv8hf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v8hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_f32_x2 (const float32_t *__a)
 {
-  float32x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v4sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x4_t) __builtin_aarch64_get_qregoiv4sf (__o, 0);
-  ret.val[1] = (float32x4_t) __builtin_aarch64_get_qregoiv4sf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v4sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_f64_x2 (const float64_t *__a)
 {
-  float64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v2df ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x2_t) __builtin_aarch64_get_qregoiv2df (__o, 0);
-  ret.val[1] = (float64x2_t) __builtin_aarch64_get_qregoiv2df (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v2df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline poly8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p8_x2 (const poly8_t *__a)
 {
-  poly8x16x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 0);
-  ret.val[1] = (poly8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v16qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p16_x2 (const poly16_t *__a)
 {
-  poly16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 0);
-  ret.val[1] = (poly16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v8hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p64_x2 (const poly64_t *__a)
 {
-  poly64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 0);
-  ret.val[1] = (poly64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v2di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u16 (const uint16_t *__a)
 {
-  return (uint16x8_t)
-    __builtin_aarch64_ld1v8hi ((const __builtin_aarch64_simd_hi *) __a);
+  return __builtin_aarch64_ld1v8hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u32 (const uint32_t *__a)
 {
-  return (uint32x4_t)
-    __builtin_aarch64_ld1v4si ((const __builtin_aarch64_simd_si *) __a);
+  return __builtin_aarch64_ld1v4si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u64 (const uint64_t *__a)
 {
-  return (uint64x2_t)
-    __builtin_aarch64_ld1v2di ((const __builtin_aarch64_simd_di *) __a);
+  return __builtin_aarch64_ld1v2di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 /* vld1(q)_x4.  */
@@ -16998,280 +15570,211 @@ __extension__ extern __inline int8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s8_x4 (const int8_t *__a)
 {
-  union { int8x8x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v8qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s8_x4 (const int8_t *__a)
 {
-  union { int8x16x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v16qi (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s16_x4 (const int16_t *__a)
 {
-  union { int16x4x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v4hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s16_x4 (const int16_t *__a)
 {
-  union { int16x8x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v8hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s32_x4 (const int32_t *__a)
 {
-  union { int32x2x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-  = __builtin_aarch64_ld1x4v2si ((const __builtin_aarch64_simd_si *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v2si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline int32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s32_x4 (const int32_t *__a)
 {
-  union { int32x4x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-  = __builtin_aarch64_ld1x4v4si ((const __builtin_aarch64_simd_si *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v4si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u8_x4 (const uint8_t *__a)
 {
-  union { uint8x8x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v8qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u8_x4 (const uint8_t *__a)
 {
-  union { uint8x16x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v16qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u16_x4 (const uint16_t *__a)
 {
-  union { uint16x4x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v4hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u16_x4 (const uint16_t *__a)
 {
-  union { uint16x8x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v8hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u32_x4 (const uint32_t *__a)
 {
-  union { uint32x2x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v2si ((const __builtin_aarch64_simd_si *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v2si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u32_x4 (const uint32_t *__a)
 {
-  union { uint32x4x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v4si ((const __builtin_aarch64_simd_si *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v4si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline float16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_f16_x4 (const float16_t *__a)
 {
-  union { float16x4x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v4hf ((const __builtin_aarch64_simd_hf *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v4hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_f16_x4 (const float16_t *__a)
 {
-  union { float16x8x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v8hf ((const __builtin_aarch64_simd_hf *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v8hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_f32_x4 (const float32_t *__a)
 {
-  union { float32x2x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v2sf ((const __builtin_aarch64_simd_sf *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v2sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_f32_x4 (const float32_t *__a)
 {
-  union { float32x4x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v4sf ((const __builtin_aarch64_simd_sf *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v4sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline poly8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_p8_x4 (const poly8_t *__a)
 {
-  union { poly8x8x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v8qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p8_x4 (const poly8_t *__a)
 {
-  union { poly8x16x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v16qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_p16_x4 (const poly16_t *__a)
 {
-  union { poly16x4x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v4hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p16_x4 (const poly16_t *__a)
 {
-  union { poly16x8x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v8hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_s64_x4 (const int64_t *__a)
 {
-  union { int64x1x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4di ((const __builtin_aarch64_simd_di *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_u64_x4 (const uint64_t *__a)
 {
-  union { uint64x1x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4di ((const __builtin_aarch64_simd_di *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline poly64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_p64_x4 (const poly64_t *__a)
 {
-  union { poly64x1x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4di ((const __builtin_aarch64_simd_di *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_s64_x4 (const int64_t *__a)
 {
-  union { int64x2x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v2di ((const __builtin_aarch64_simd_di *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v2di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_u64_x4 (const uint64_t *__a)
 {
-  union { uint64x2x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v2di ((const __builtin_aarch64_simd_di *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v2di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline poly64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_p64_x4 (const poly64_t *__a)
 {
-  union { poly64x2x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v2di ((const __builtin_aarch64_simd_di *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v2di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_f64_x4 (const float64_t *__a)
 {
-  union { float64x1x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4df ((const __builtin_aarch64_simd_df *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline float64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_f64_x4 (const float64_t *__a)
 {
-  union { float64x2x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v2df ((const __builtin_aarch64_simd_df *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v2df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 /* vld1_dup  */
@@ -17680,1092 +16183,622 @@ __extension__ extern __inline int64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_s64 (const int64_t * __a)
 {
-  int64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x1_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  ret.val[1] = (int64x1_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_u64 (const uint64_t * __a)
 {
-  uint64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x1_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  ret.val[1] = (uint64x1_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2di_us ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_f64 (const float64_t * __a)
 {
-  float64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2df ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x1_t) {__builtin_aarch64_get_dregoidf (__o, 0)};
-  ret.val[1] = (float64x1_t) {__builtin_aarch64_get_dregoidf (__o, 1)};
-  return ret;
+  return __builtin_aarch64_ld2df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline int8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_s8 (const int8_t * __a)
 {
-  int8x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 0);
-  ret.val[1] = (int8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v8qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_p8 (const poly8_t * __a)
 {
-  poly8x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 0);
-  ret.val[1] = (poly8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v8qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_p64 (const poly64_t * __a)
 {
-  poly64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x1_t) __builtin_aarch64_get_dregoidi_pss (__o, 0);
-  ret.val[1] = (poly64x1_t) __builtin_aarch64_get_dregoidi_pss (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2di_ps ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_s16 (const int16_t * __a)
 {
-  int16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 0);
-  ret.val[1] = (int16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v4hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_p16 (const poly16_t * __a)
 {
-  poly16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 0);
-  ret.val[1] = (poly16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v4hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_s32 (const int32_t * __a)
 {
-  int32x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 0);
-  ret.val[1] = (int32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v2si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_u8 (const uint8_t * __a)
 {
-  uint8x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 0);
-  ret.val[1] = (uint8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v8qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_u16 (const uint16_t * __a)
 {
-  uint16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 0);
-  ret.val[1] = (uint16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v4hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_u32 (const uint32_t * __a)
 {
-  uint32x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 0);
-  ret.val[1] = (uint32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v2si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline float16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_f16 (const float16_t * __a)
 {
-  float16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v4hf (__a);
-  ret.val[0] = __builtin_aarch64_get_dregoiv4hf (__o, 0);
-  ret.val[1] = __builtin_aarch64_get_dregoiv4hf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v4hf (__a);
 }
 
 __extension__ extern __inline float32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_f32 (const float32_t * __a)
 {
-  float32x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v2sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x2_t) __builtin_aarch64_get_dregoiv2sf (__o, 0);
-  ret.val[1] = (float32x2_t) __builtin_aarch64_get_dregoiv2sf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v2sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline int8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_s8 (const int8_t * __a)
 {
-  int8x16x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 0);
-  ret.val[1] = (int8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v16qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_p8 (const poly8_t * __a)
 {
-  poly8x16x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 0);
-  ret.val[1] = (poly8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v16qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_s16 (const int16_t * __a)
 {
-  int16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 0);
-  ret.val[1] = (int16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v8hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_p16 (const poly16_t * __a)
 {
-  poly16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 0);
-  ret.val[1] = (poly16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v8hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_p64 (const poly64_t * __a)
 {
-  poly64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x2_t) __builtin_aarch64_get_qregoiv2di_pss (__o, 0);
-  ret.val[1] = (poly64x2_t) __builtin_aarch64_get_qregoiv2di_pss (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v2di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_s32 (const int32_t * __a)
 {
-  int32x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (int32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v4si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline int64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_s64 (const int64_t * __a)
 {
-  int64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 0);
-  ret.val[1] = (int64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v2di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_u8 (const uint8_t * __a)
 {
-  uint8x16x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 0);
-  ret.val[1] = (uint8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v16qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_u16 (const uint16_t * __a)
 {
-  uint16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 0);
-  ret.val[1] = (uint16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v8hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_u32 (const uint32_t * __a)
 {
-  uint32x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (uint32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v4si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_u64 (const uint64_t * __a)
 {
-  uint64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 0);
-  ret.val[1] = (uint64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v2di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_f16 (const float16_t * __a)
 {
-  float16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v8hf (__a);
-  ret.val[0] = __builtin_aarch64_get_qregoiv8hf (__o, 0);
-  ret.val[1] = __builtin_aarch64_get_qregoiv8hf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v8hf (__a);
 }
 
 __extension__ extern __inline float32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_f32 (const float32_t * __a)
 {
-  float32x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v4sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x4_t) __builtin_aarch64_get_qregoiv4sf (__o, 0);
-  ret.val[1] = (float32x4_t) __builtin_aarch64_get_qregoiv4sf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v4sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_f64 (const float64_t * __a)
 {
-  float64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v2df ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x2_t) __builtin_aarch64_get_qregoiv2df (__o, 0);
-  ret.val[1] = (float64x2_t) __builtin_aarch64_get_qregoiv2df (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v2df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline int64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_s64 (const int64_t * __a)
 {
-  int64x1x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  ret.val[1] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  ret.val[2] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_u64 (const uint64_t * __a)
 {
-  uint64x1x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  ret.val[1] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  ret.val[2] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3di_us ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_f64 (const float64_t * __a)
 {
-  float64x1x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3df ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x1_t) {__builtin_aarch64_get_dregcidf (__o, 0)};
-  ret.val[1] = (float64x1_t) {__builtin_aarch64_get_dregcidf (__o, 1)};
-  ret.val[2] = (float64x1_t) {__builtin_aarch64_get_dregcidf (__o, 2)};
-  return ret;
+  return __builtin_aarch64_ld3df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline int8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_s8 (const int8_t * __a)
 {
-  int8x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 0);
-  ret.val[1] = (int8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 1);
-  ret.val[2] = (int8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v8qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_p8 (const poly8_t * __a)
 {
-  poly8x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 0);
-  ret.val[1] = (poly8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 1);
-  ret.val[2] = (poly8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v8qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_s16 (const int16_t * __a)
 {
-  int16x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 0);
-  ret.val[1] = (int16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 1);
-  ret.val[2] = (int16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v4hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_p16 (const poly16_t * __a)
 {
-  poly16x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 0);
-  ret.val[1] = (poly16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 1);
-  ret.val[2] = (poly16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v4hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_s32 (const int32_t * __a)
 {
-  int32x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x2_t) __builtin_aarch64_get_dregciv2si (__o, 0);
-  ret.val[1] = (int32x2_t) __builtin_aarch64_get_dregciv2si (__o, 1);
-  ret.val[2] = (int32x2_t) __builtin_aarch64_get_dregciv2si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v2si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_u8 (const uint8_t * __a)
 {
-  uint8x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 0);
-  ret.val[1] = (uint8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 1);
-  ret.val[2] = (uint8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v8qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_u16 (const uint16_t * __a)
 {
-  uint16x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 0);
-  ret.val[1] = (uint16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 1);
-  ret.val[2] = (uint16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v4hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_u32 (const uint32_t * __a)
 {
-  uint32x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x2_t) __builtin_aarch64_get_dregciv2si (__o, 0);
-  ret.val[1] = (uint32x2_t) __builtin_aarch64_get_dregciv2si (__o, 1);
-  ret.val[2] = (uint32x2_t) __builtin_aarch64_get_dregciv2si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v2si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline float16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_f16 (const float16_t * __a)
 {
-  float16x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v4hf (__a);
-  ret.val[0] = __builtin_aarch64_get_dregciv4hf (__o, 0);
-  ret.val[1] = __builtin_aarch64_get_dregciv4hf (__o, 1);
-  ret.val[2] = __builtin_aarch64_get_dregciv4hf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v4hf (__a);
 }
 
 __extension__ extern __inline float32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_f32 (const float32_t * __a)
 {
-  float32x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v2sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x2_t) __builtin_aarch64_get_dregciv2sf (__o, 0);
-  ret.val[1] = (float32x2_t) __builtin_aarch64_get_dregciv2sf (__o, 1);
-  ret.val[2] = (float32x2_t) __builtin_aarch64_get_dregciv2sf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v2sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline poly64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_p64 (const poly64_t * __a)
 {
-  poly64x1x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x1_t) __builtin_aarch64_get_dregcidi_pss (__o, 0);
-  ret.val[1] = (poly64x1_t) __builtin_aarch64_get_dregcidi_pss (__o, 1);
-  ret.val[2] = (poly64x1_t) __builtin_aarch64_get_dregcidi_pss (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3di_ps ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_s8 (const int8_t * __a)
 {
-  int8x16x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 0);
-  ret.val[1] = (int8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 1);
-  ret.val[2] = (int8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v16qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_p8 (const poly8_t * __a)
 {
-  poly8x16x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 0);
-  ret.val[1] = (poly8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 1);
-  ret.val[2] = (poly8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v16qi_ps ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_s16 (const int16_t * __a)
 {
-  int16x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 0);
-  ret.val[1] = (int16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 1);
-  ret.val[2] = (int16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v8hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_p16 (const poly16_t * __a)
 {
-  poly16x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 0);
-  ret.val[1] = (poly16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 1);
-  ret.val[2] = (poly16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v8hi_ps ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_s32 (const int32_t * __a)
 {
-  int32x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x4_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (int32x4_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (int32x4_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v4si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline int64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_s64 (const int64_t * __a)
 {
-  int64x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x2_t) __builtin_aarch64_get_qregciv2di (__o, 0);
-  ret.val[1] = (int64x2_t) __builtin_aarch64_get_qregciv2di (__o, 1);
-  ret.val[2] = (int64x2_t) __builtin_aarch64_get_qregciv2di (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v2di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_u8 (const uint8_t * __a)
 {
-  uint8x16x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 0);
-  ret.val[1] = (uint8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 1);
-  ret.val[2] = (uint8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v16qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_u16 (const uint16_t * __a)
 {
-  uint16x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 0);
-  ret.val[1] = (uint16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 1);
-  ret.val[2] = (uint16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v8hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_u32 (const uint32_t * __a)
 {
-  uint32x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x4_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (uint32x4_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (uint32x4_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v4si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_u64 (const uint64_t * __a)
 {
-  uint64x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x2_t) __builtin_aarch64_get_qregciv2di (__o, 0);
-  ret.val[1] = (uint64x2_t) __builtin_aarch64_get_qregciv2di (__o, 1);
-  ret.val[2] = (uint64x2_t) __builtin_aarch64_get_qregciv2di (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v2di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_f16 (const float16_t * __a)
 {
-  float16x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v8hf (__a);
-  ret.val[0] = __builtin_aarch64_get_qregciv8hf (__o, 0);
-  ret.val[1] = __builtin_aarch64_get_qregciv8hf (__o, 1);
-  ret.val[2] = __builtin_aarch64_get_qregciv8hf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v8hf (__a);
 }
 
 __extension__ extern __inline float32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_f32 (const float32_t * __a)
 {
-  float32x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v4sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x4_t) __builtin_aarch64_get_qregciv4sf (__o, 0);
-  ret.val[1] = (float32x4_t) __builtin_aarch64_get_qregciv4sf (__o, 1);
-  ret.val[2] = (float32x4_t) __builtin_aarch64_get_qregciv4sf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v4sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_f64 (const float64_t * __a)
 {
-  float64x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v2df ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x2_t) __builtin_aarch64_get_qregciv2df (__o, 0);
-  ret.val[1] = (float64x2_t) __builtin_aarch64_get_qregciv2df (__o, 1);
-  ret.val[2] = (float64x2_t) __builtin_aarch64_get_qregciv2df (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v2df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline poly64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_p64 (const poly64_t * __a)
 {
-  poly64x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x2_t) __builtin_aarch64_get_qregciv2di_pss (__o, 0);
-  ret.val[1] = (poly64x2_t) __builtin_aarch64_get_qregciv2di_pss (__o, 1);
-  ret.val[2] = (poly64x2_t) __builtin_aarch64_get_qregciv2di_pss (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v2di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_s64 (const int64_t * __a)
 {
-  int64x1x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  ret.val[1] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  ret.val[2] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  ret.val[3] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_u64 (const uint64_t * __a)
 {
-  uint64x1x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  ret.val[1] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  ret.val[2] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  ret.val[3] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4di_us ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_f64 (const float64_t * __a)
 {
-  float64x1x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4df ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x1_t) {__builtin_aarch64_get_dregxidf (__o, 0)};
-  ret.val[1] = (float64x1_t) {__builtin_aarch64_get_dregxidf (__o, 1)};
-  ret.val[2] = (float64x1_t) {__builtin_aarch64_get_dregxidf (__o, 2)};
-  ret.val[3] = (float64x1_t) {__builtin_aarch64_get_dregxidf (__o, 3)};
-  return ret;
+  return __builtin_aarch64_ld4df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline int8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_s8 (const int8_t * __a)
 {
-  int8x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 0);
-  ret.val[1] = (int8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 1);
-  ret.val[2] = (int8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 2);
-  ret.val[3] = (int8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v8qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_p8 (const poly8_t * __a)
 {
-  poly8x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 0);
-  ret.val[1] = (poly8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 1);
-  ret.val[2] = (poly8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 2);
-  ret.val[3] = (poly8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v8qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_s16 (const int16_t * __a)
 {
-  int16x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 0);
-  ret.val[1] = (int16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 1);
-  ret.val[2] = (int16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 2);
-  ret.val[3] = (int16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v4hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_p16 (const poly16_t * __a)
 {
-  poly16x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 0);
-  ret.val[1] = (poly16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 1);
-  ret.val[2] = (poly16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 2);
-  ret.val[3] = (poly16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v4hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_s32 (const int32_t * __a)
 {
-  int32x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 0);
-  ret.val[1] = (int32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 1);
-  ret.val[2] = (int32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 2);
-  ret.val[3] = (int32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v2si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_u8 (const uint8_t * __a)
 {
-  uint8x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 0);
-  ret.val[1] = (uint8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 1);
-  ret.val[2] = (uint8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 2);
-  ret.val[3] = (uint8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v8qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_u16 (const uint16_t * __a)
 {
-  uint16x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 0);
-  ret.val[1] = (uint16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 1);
-  ret.val[2] = (uint16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 2);
-  ret.val[3] = (uint16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v4hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_u32 (const uint32_t * __a)
 {
-  uint32x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 0);
-  ret.val[1] = (uint32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 1);
-  ret.val[2] = (uint32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 2);
-  ret.val[3] = (uint32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v2si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline float16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_f16 (const float16_t * __a)
 {
-  float16x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v4hf (__a);
-  ret.val[0] = __builtin_aarch64_get_dregxiv4hf (__o, 0);
-  ret.val[1] = __builtin_aarch64_get_dregxiv4hf (__o, 1);
-  ret.val[2] = __builtin_aarch64_get_dregxiv4hf (__o, 2);
-  ret.val[3] = __builtin_aarch64_get_dregxiv4hf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v4hf (__a);
 }
 
 __extension__ extern __inline float32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_f32 (const float32_t * __a)
 {
-  float32x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v2sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x2_t) __builtin_aarch64_get_dregxiv2sf (__o, 0);
-  ret.val[1] = (float32x2_t) __builtin_aarch64_get_dregxiv2sf (__o, 1);
-  ret.val[2] = (float32x2_t) __builtin_aarch64_get_dregxiv2sf (__o, 2);
-  ret.val[3] = (float32x2_t) __builtin_aarch64_get_dregxiv2sf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v2sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline poly64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_p64 (const poly64_t * __a)
 {
-  poly64x1x4_t  ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x1_t) __builtin_aarch64_get_dregxidi_pss (__o, 0);
-  ret.val[1] = (poly64x1_t) __builtin_aarch64_get_dregxidi_pss (__o, 1);
-  ret.val[2] = (poly64x1_t) __builtin_aarch64_get_dregxidi_pss (__o, 2);
-  ret.val[3] = (poly64x1_t) __builtin_aarch64_get_dregxidi_pss (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4di_ps ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_s8 (const int8_t * __a)
 {
-  int8x16x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 0);
-  ret.val[1] = (int8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 1);
-  ret.val[2] = (int8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 2);
-  ret.val[3] = (int8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v16qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_p8 (const poly8_t * __a)
 {
-  poly8x16x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 0);
-  ret.val[1] = (poly8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 1);
-  ret.val[2] = (poly8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 2);
-  ret.val[3] = (poly8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v16qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_s16 (const int16_t * __a)
 {
-  int16x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 0);
-  ret.val[1] = (int16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 1);
-  ret.val[2] = (int16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 2);
-  ret.val[3] = (int16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v8hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_p16 (const poly16_t * __a)
 {
-  poly16x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 0);
-  ret.val[1] = (poly16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 1);
-  ret.val[2] = (poly16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 2);
-  ret.val[3] = (poly16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v8hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_s32 (const int32_t * __a)
 {
-  int32x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v4si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline int64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_s64 (const int64_t * __a)
 {
-  int64x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 0);
-  ret.val[1] = (int64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 1);
-  ret.val[2] = (int64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 2);
-  ret.val[3] = (int64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v2di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_u8 (const uint8_t * __a)
 {
-  uint8x16x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 0);
-  ret.val[1] = (uint8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 1);
-  ret.val[2] = (uint8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 2);
-  ret.val[3] = (uint8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v16qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_u16 (const uint16_t * __a)
 {
-  uint16x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 0);
-  ret.val[1] = (uint16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 1);
-  ret.val[2] = (uint16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 2);
-  ret.val[3] = (uint16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v8hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_u32 (const uint32_t * __a)
 {
-  uint32x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v4si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_u64 (const uint64_t * __a)
 {
-  uint64x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 0);
-  ret.val[1] = (uint64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 1);
-  ret.val[2] = (uint64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 2);
-  ret.val[3] = (uint64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v2di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_f16 (const float16_t * __a)
 {
-  float16x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v8hf (__a);
-  ret.val[0] = __builtin_aarch64_get_qregxiv8hf (__o, 0);
-  ret.val[1] = __builtin_aarch64_get_qregxiv8hf (__o, 1);
-  ret.val[2] = __builtin_aarch64_get_qregxiv8hf (__o, 2);
-  ret.val[3] = __builtin_aarch64_get_qregxiv8hf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v8hf (__a);
 }
 
 __extension__ extern __inline float32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_f32 (const float32_t * __a)
 {
-  float32x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v4sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x4_t) __builtin_aarch64_get_qregxiv4sf (__o, 0);
-  ret.val[1] = (float32x4_t) __builtin_aarch64_get_qregxiv4sf (__o, 1);
-  ret.val[2] = (float32x4_t) __builtin_aarch64_get_qregxiv4sf (__o, 2);
-  ret.val[3] = (float32x4_t) __builtin_aarch64_get_qregxiv4sf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v4sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_f64 (const float64_t * __a)
 {
-  float64x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v2df ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x2_t) __builtin_aarch64_get_qregxiv2df (__o, 0);
-  ret.val[1] = (float64x2_t) __builtin_aarch64_get_qregxiv2df (__o, 1);
-  ret.val[2] = (float64x2_t) __builtin_aarch64_get_qregxiv2df (__o, 2);
-  ret.val[3] = (float64x2_t) __builtin_aarch64_get_qregxiv2df (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v2df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline poly64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_p64 (const poly64_t * __a)
 {
-  poly64x2x4_t  ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x2_t) __builtin_aarch64_get_qregxiv2di_pss (__o, 0);
-  ret.val[1] = (poly64x2_t) __builtin_aarch64_get_qregxiv2di_pss (__o, 1);
-  ret.val[2] = (poly64x2_t) __builtin_aarch64_get_qregxiv2di_pss (__o, 2);
-  ret.val[3] = (poly64x2_t) __builtin_aarch64_get_qregxiv2di_pss (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v2di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline poly128_t
@@ -18781,1093 +16814,624 @@ __extension__ extern __inline int8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_s8 (const int8_t * __a)
 {
-  int8x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 0);
-  ret.val[1] = (int8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv8qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_s16 (const int16_t * __a)
 {
-  int16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 0);
-  ret.val[1] = (int16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv4hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_s32 (const int32_t * __a)
 {
-  int32x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 0);
-  ret.val[1] = (int32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv2si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline float16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_f16 (const float16_t * __a)
 {
-  float16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv4hf ((const __builtin_aarch64_simd_hf *) __a);
-  ret.val[0] = __builtin_aarch64_get_dregoiv4hf (__o, 0);
-  ret.val[1] = (float16x4_t) __builtin_aarch64_get_dregoiv4hf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv4hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_f32 (const float32_t * __a)
 {
-  float32x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv2sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x2_t) __builtin_aarch64_get_dregoiv2sf (__o, 0);
-  ret.val[1] = (float32x2_t) __builtin_aarch64_get_dregoiv2sf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv2sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_f64 (const float64_t * __a)
 {
-  float64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rdf ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x1_t) {__builtin_aarch64_get_dregoidf (__o, 0)};
-  ret.val[1] = (float64x1_t) {__builtin_aarch64_get_dregoidf (__o, 1)};
-  return ret;
+  return __builtin_aarch64_ld2rdf ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline uint8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_u8 (const uint8_t * __a)
 {
-  uint8x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 0);
-  ret.val[1] = (uint8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv8qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_u16 (const uint16_t * __a)
 {
-  uint16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 0);
-  ret.val[1] = (uint16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv4hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_u32 (const uint32_t * __a)
 {
-  uint32x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 0);
-  ret.val[1] = (uint32x2_t) __builtin_aarch64_get_dregoiv2si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv2si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline poly8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_p8 (const poly8_t * __a)
 {
-  poly8x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 0);
-  ret.val[1] = (poly8x8_t) __builtin_aarch64_get_dregoiv8qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv8qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_p16 (const poly16_t * __a)
 {
-  poly16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 0);
-  ret.val[1] = (poly16x4_t) __builtin_aarch64_get_dregoiv4hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv4hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_p64 (const poly64_t * __a)
 {
-  poly64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x1_t) __builtin_aarch64_get_dregoidi_pss (__o, 0);
-  ret.val[1] = (poly64x1_t) __builtin_aarch64_get_dregoidi_pss (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rdi_ps ((const __builtin_aarch64_simd_di *) __a);
 }
 
-
 __extension__ extern __inline int64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_s64 (const int64_t * __a)
 {
-  int64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rdi ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x1_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  ret.val[1] = (int64x1_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rdi ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_u64 (const uint64_t * __a)
 {
-  uint64x1x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rdi ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x1_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  ret.val[1] = (uint64x1_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rdi_us ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_s8 (const int8_t * __a)
 {
-  int8x16x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 0);
-  ret.val[1] = (int8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv16qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_p8 (const poly8_t * __a)
 {
-  poly8x16x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 0);
-  ret.val[1] = (poly8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv16qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_s16 (const int16_t * __a)
 {
-  int16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 0);
-  ret.val[1] = (int16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv8hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_p16 (const poly16_t * __a)
 {
-  poly16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 0);
-  ret.val[1] = (poly16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv8hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_s32 (const int32_t * __a)
 {
-  int32x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (int32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv4si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline int64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_s64 (const int64_t * __a)
 {
-  int64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 0);
-  ret.val[1] = (int64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv2di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_u8 (const uint8_t * __a)
 {
-  uint8x16x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 0);
-  ret.val[1] = (uint8x16_t) __builtin_aarch64_get_qregoiv16qi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv16qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_u16 (const uint16_t * __a)
 {
-  uint16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 0);
-  ret.val[1] = (uint16x8_t) __builtin_aarch64_get_qregoiv8hi (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv8hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_u32 (const uint32_t * __a)
 {
-  uint32x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (uint32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv4si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_u64 (const uint64_t * __a)
 {
-  uint64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 0);
-  ret.val[1] = (uint64x2_t) __builtin_aarch64_get_qregoiv2di (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv2di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_f16 (const float16_t * __a)
 {
-  float16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv8hf ((const __builtin_aarch64_simd_hf *) __a);
-  ret.val[0] = (float16x8_t) __builtin_aarch64_get_qregoiv8hf (__o, 0);
-  ret.val[1] = __builtin_aarch64_get_qregoiv8hf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv8hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_f32 (const float32_t * __a)
 {
-  float32x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv4sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x4_t) __builtin_aarch64_get_qregoiv4sf (__o, 0);
-  ret.val[1] = (float32x4_t) __builtin_aarch64_get_qregoiv4sf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv4sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_f64 (const float64_t * __a)
 {
-  float64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv2df ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x2_t) __builtin_aarch64_get_qregoiv2df (__o, 0);
-  ret.val[1] = (float64x2_t) __builtin_aarch64_get_qregoiv2df (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv2df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline poly64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_p64 (const poly64_t * __a)
 {
-  poly64x2x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x2_t) __builtin_aarch64_get_qregoiv2di_pss (__o, 0);
-  ret.val[1] = (poly64x2_t) __builtin_aarch64_get_qregoiv2di_pss (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv2di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_s64 (const int64_t * __a)
 {
-  int64x1x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rdi ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  ret.val[1] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  ret.val[2] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rdi ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_u64 (const uint64_t * __a)
 {
-  uint64x1x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rdi ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  ret.val[1] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  ret.val[2] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rdi_us ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_f64 (const float64_t * __a)
 {
-  float64x1x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rdf ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x1_t) {__builtin_aarch64_get_dregcidf (__o, 0)};
-  ret.val[1] = (float64x1_t) {__builtin_aarch64_get_dregcidf (__o, 1)};
-  ret.val[2] = (float64x1_t) {__builtin_aarch64_get_dregcidf (__o, 2)};
-  return ret;
+  return __builtin_aarch64_ld3rdf ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline int8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_s8 (const int8_t * __a)
 {
-  int8x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 0);
-  ret.val[1] = (int8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 1);
-  ret.val[2] = (int8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv8qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_p8 (const poly8_t * __a)
 {
-  poly8x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 0);
-  ret.val[1] = (poly8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 1);
-  ret.val[2] = (poly8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv8qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_s16 (const int16_t * __a)
 {
-  int16x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 0);
-  ret.val[1] = (int16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 1);
-  ret.val[2] = (int16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv4hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_p16 (const poly16_t * __a)
 {
-  poly16x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 0);
-  ret.val[1] = (poly16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 1);
-  ret.val[2] = (poly16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv4hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_s32 (const int32_t * __a)
 {
-  int32x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x2_t) __builtin_aarch64_get_dregciv2si (__o, 0);
-  ret.val[1] = (int32x2_t) __builtin_aarch64_get_dregciv2si (__o, 1);
-  ret.val[2] = (int32x2_t) __builtin_aarch64_get_dregciv2si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv2si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_u8 (const uint8_t * __a)
 {
-  uint8x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 0);
-  ret.val[1] = (uint8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 1);
-  ret.val[2] = (uint8x8_t) __builtin_aarch64_get_dregciv8qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv8qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_u16 (const uint16_t * __a)
 {
-  uint16x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 0);
-  ret.val[1] = (uint16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 1);
-  ret.val[2] = (uint16x4_t) __builtin_aarch64_get_dregciv4hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv4hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_u32 (const uint32_t * __a)
 {
-  uint32x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x2_t) __builtin_aarch64_get_dregciv2si (__o, 0);
-  ret.val[1] = (uint32x2_t) __builtin_aarch64_get_dregciv2si (__o, 1);
-  ret.val[2] = (uint32x2_t) __builtin_aarch64_get_dregciv2si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv2si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline float16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_f16 (const float16_t * __a)
 {
-  float16x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv4hf ((const __builtin_aarch64_simd_hf *) __a);
-  ret.val[0] = (float16x4_t) __builtin_aarch64_get_dregciv4hf (__o, 0);
-  ret.val[1] = (float16x4_t) __builtin_aarch64_get_dregciv4hf (__o, 1);
-  ret.val[2] = (float16x4_t) __builtin_aarch64_get_dregciv4hf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv4hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_f32 (const float32_t * __a)
 {
-  float32x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv2sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x2_t) __builtin_aarch64_get_dregciv2sf (__o, 0);
-  ret.val[1] = (float32x2_t) __builtin_aarch64_get_dregciv2sf (__o, 1);
-  ret.val[2] = (float32x2_t) __builtin_aarch64_get_dregciv2sf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv2sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline poly64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_p64 (const poly64_t * __a)
 {
-  poly64x1x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x1_t) __builtin_aarch64_get_dregcidi_pss (__o, 0);
-  ret.val[1] = (poly64x1_t) __builtin_aarch64_get_dregcidi_pss (__o, 1);
-  ret.val[2] = (poly64x1_t) __builtin_aarch64_get_dregcidi_pss (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rdi_ps ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_s8 (const int8_t * __a)
 {
-  int8x16x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 0);
-  ret.val[1] = (int8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 1);
-  ret.val[2] = (int8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv16qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_p8 (const poly8_t * __a)
 {
-  poly8x16x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 0);
-  ret.val[1] = (poly8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 1);
-  ret.val[2] = (poly8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv16qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_s16 (const int16_t * __a)
 {
-  int16x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 0);
-  ret.val[1] = (int16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 1);
-  ret.val[2] = (int16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv8hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_p16 (const poly16_t * __a)
 {
-  poly16x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 0);
-  ret.val[1] = (poly16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 1);
-  ret.val[2] = (poly16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv8hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_s32 (const int32_t * __a)
 {
-  int32x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x4_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (int32x4_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (int32x4_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv4si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline int64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_s64 (const int64_t * __a)
 {
-  int64x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x2_t) __builtin_aarch64_get_qregciv2di (__o, 0);
-  ret.val[1] = (int64x2_t) __builtin_aarch64_get_qregciv2di (__o, 1);
-  ret.val[2] = (int64x2_t) __builtin_aarch64_get_qregciv2di (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv2di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_u8 (const uint8_t * __a)
 {
-  uint8x16x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 0);
-  ret.val[1] = (uint8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 1);
-  ret.val[2] = (uint8x16_t) __builtin_aarch64_get_qregciv16qi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv16qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_u16 (const uint16_t * __a)
 {
-  uint16x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 0);
-  ret.val[1] = (uint16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 1);
-  ret.val[2] = (uint16x8_t) __builtin_aarch64_get_qregciv8hi (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv8hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_u32 (const uint32_t * __a)
 {
-  uint32x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x4_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (uint32x4_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (uint32x4_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv4si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_u64 (const uint64_t * __a)
 {
-  uint64x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x2_t) __builtin_aarch64_get_qregciv2di (__o, 0);
-  ret.val[1] = (uint64x2_t) __builtin_aarch64_get_qregciv2di (__o, 1);
-  ret.val[2] = (uint64x2_t) __builtin_aarch64_get_qregciv2di (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv2di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_f16 (const float16_t * __a)
 {
-  float16x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv8hf ((const __builtin_aarch64_simd_hf *) __a);
-  ret.val[0] = (float16x8_t) __builtin_aarch64_get_qregciv8hf (__o, 0);
-  ret.val[1] = (float16x8_t) __builtin_aarch64_get_qregciv8hf (__o, 1);
-  ret.val[2] = (float16x8_t) __builtin_aarch64_get_qregciv8hf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv8hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_f32 (const float32_t * __a)
 {
-  float32x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv4sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x4_t) __builtin_aarch64_get_qregciv4sf (__o, 0);
-  ret.val[1] = (float32x4_t) __builtin_aarch64_get_qregciv4sf (__o, 1);
-  ret.val[2] = (float32x4_t) __builtin_aarch64_get_qregciv4sf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv4sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_f64 (const float64_t * __a)
 {
-  float64x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv2df ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x2_t) __builtin_aarch64_get_qregciv2df (__o, 0);
-  ret.val[1] = (float64x2_t) __builtin_aarch64_get_qregciv2df (__o, 1);
-  ret.val[2] = (float64x2_t) __builtin_aarch64_get_qregciv2df (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv2df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline poly64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_p64 (const poly64_t * __a)
 {
-  poly64x2x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x2_t) __builtin_aarch64_get_qregciv2di_pss (__o, 0);
-  ret.val[1] = (poly64x2_t) __builtin_aarch64_get_qregciv2di_pss (__o, 1);
-  ret.val[2] = (poly64x2_t) __builtin_aarch64_get_qregciv2di_pss (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv2di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_s64 (const int64_t * __a)
 {
-  int64x1x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rdi ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  ret.val[1] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  ret.val[2] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  ret.val[3] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rdi ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_u64 (const uint64_t * __a)
 {
-  uint64x1x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rdi ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  ret.val[1] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  ret.val[2] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  ret.val[3] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rdi_us ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_f64 (const float64_t * __a)
 {
-  float64x1x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rdf ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x1_t) {__builtin_aarch64_get_dregxidf (__o, 0)};
-  ret.val[1] = (float64x1_t) {__builtin_aarch64_get_dregxidf (__o, 1)};
-  ret.val[2] = (float64x1_t) {__builtin_aarch64_get_dregxidf (__o, 2)};
-  ret.val[3] = (float64x1_t) {__builtin_aarch64_get_dregxidf (__o, 3)};
-  return ret;
+  return __builtin_aarch64_ld4rdf ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline int8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_s8 (const int8_t * __a)
 {
-  int8x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 0);
-  ret.val[1] = (int8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 1);
-  ret.val[2] = (int8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 2);
-  ret.val[3] = (int8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv8qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_p8 (const poly8_t * __a)
 {
-  poly8x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 0);
-  ret.val[1] = (poly8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 1);
-  ret.val[2] = (poly8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 2);
-  ret.val[3] = (poly8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv8qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_s16 (const int16_t * __a)
 {
-  int16x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 0);
-  ret.val[1] = (int16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 1);
-  ret.val[2] = (int16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 2);
-  ret.val[3] = (int16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv4hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_p16 (const poly16_t * __a)
 {
-  poly16x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 0);
-  ret.val[1] = (poly16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 1);
-  ret.val[2] = (poly16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 2);
-  ret.val[3] = (poly16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv4hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_s32 (const int32_t * __a)
 {
-  int32x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 0);
-  ret.val[1] = (int32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 1);
-  ret.val[2] = (int32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 2);
-  ret.val[3] = (int32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv2si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_u8 (const uint8_t * __a)
 {
-  uint8x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv8qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 0);
-  ret.val[1] = (uint8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 1);
-  ret.val[2] = (uint8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 2);
-  ret.val[3] = (uint8x8_t) __builtin_aarch64_get_dregxiv8qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv8qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_u16 (const uint16_t * __a)
 {
-  uint16x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv4hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 0);
-  ret.val[1] = (uint16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 1);
-  ret.val[2] = (uint16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 2);
-  ret.val[3] = (uint16x4_t) __builtin_aarch64_get_dregxiv4hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv4hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_u32 (const uint32_t * __a)
 {
-  uint32x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv2si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 0);
-  ret.val[1] = (uint32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 1);
-  ret.val[2] = (uint32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 2);
-  ret.val[3] = (uint32x2_t) __builtin_aarch64_get_dregxiv2si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv2si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline float16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_f16 (const float16_t * __a)
 {
-  float16x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv4hf ((const __builtin_aarch64_simd_hf *) __a);
-  ret.val[0] = (float16x4_t) __builtin_aarch64_get_dregxiv4hf (__o, 0);
-  ret.val[1] = (float16x4_t) __builtin_aarch64_get_dregxiv4hf (__o, 1);
-  ret.val[2] = (float16x4_t) __builtin_aarch64_get_dregxiv4hf (__o, 2);
-  ret.val[3] = (float16x4_t) __builtin_aarch64_get_dregxiv4hf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv4hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_f32 (const float32_t * __a)
 {
-  float32x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv2sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x2_t) __builtin_aarch64_get_dregxiv2sf (__o, 0);
-  ret.val[1] = (float32x2_t) __builtin_aarch64_get_dregxiv2sf (__o, 1);
-  ret.val[2] = (float32x2_t) __builtin_aarch64_get_dregxiv2sf (__o, 2);
-  ret.val[3] = (float32x2_t) __builtin_aarch64_get_dregxiv2sf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv2sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline poly64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_p64 (const poly64_t * __a)
 {
-  poly64x1x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x1_t) __builtin_aarch64_get_dregxidi_pss (__o, 0);
-  ret.val[1] = (poly64x1_t) __builtin_aarch64_get_dregxidi_pss (__o, 1);
-  ret.val[2] = (poly64x1_t) __builtin_aarch64_get_dregxidi_pss (__o, 2);
-  ret.val[3] = (poly64x1_t) __builtin_aarch64_get_dregxidi_pss (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rdi_ps ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline int8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_s8 (const int8_t * __a)
 {
-  int8x16x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (int8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 0);
-  ret.val[1] = (int8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 1);
-  ret.val[2] = (int8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 2);
-  ret.val[3] = (int8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv16qi ((const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline poly8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_p8 (const poly8_t * __a)
 {
-  poly8x16x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (poly8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 0);
-  ret.val[1] = (poly8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 1);
-  ret.val[2] = (poly8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 2);
-  ret.val[3] = (poly8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv16qi_ps (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline int16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_s16 (const int16_t * __a)
 {
-  int16x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (int16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 0);
-  ret.val[1] = (int16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 1);
-  ret.val[2] = (int16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 2);
-  ret.val[3] = (int16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv8hi ((const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline poly16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_p16 (const poly16_t * __a)
 {
-  poly16x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (poly16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 0);
-  ret.val[1] = (poly16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 1);
-  ret.val[2] = (poly16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 2);
-  ret.val[3] = (poly16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv8hi_ps (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline int32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_s32 (const int32_t * __a)
 {
-  int32x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv4si ((const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline int64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_s64 (const int64_t * __a)
 {
-  int64x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (int64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 0);
-  ret.val[1] = (int64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 1);
-  ret.val[2] = (int64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 2);
-  ret.val[3] = (int64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv2di ((const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline uint8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_u8 (const uint8_t * __a)
 {
-  uint8x16x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv16qi ((const __builtin_aarch64_simd_qi *) __a);
-  ret.val[0] = (uint8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 0);
-  ret.val[1] = (uint8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 1);
-  ret.val[2] = (uint8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 2);
-  ret.val[3] = (uint8x16_t) __builtin_aarch64_get_qregxiv16qi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv16qi_us (
+				(const __builtin_aarch64_simd_qi *) __a);
 }
 
 __extension__ extern __inline uint16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_u16 (const uint16_t * __a)
 {
-  uint16x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv8hi ((const __builtin_aarch64_simd_hi *) __a);
-  ret.val[0] = (uint16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 0);
-  ret.val[1] = (uint16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 1);
-  ret.val[2] = (uint16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 2);
-  ret.val[3] = (uint16x8_t) __builtin_aarch64_get_qregxiv8hi (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv8hi_us (
+				(const __builtin_aarch64_simd_hi *) __a);
 }
 
 __extension__ extern __inline uint32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_u32 (const uint32_t * __a)
 {
-  uint32x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv4si ((const __builtin_aarch64_simd_si *) __a);
-  ret.val[0] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv4si_us (
+				(const __builtin_aarch64_simd_si *) __a);
 }
 
 __extension__ extern __inline uint64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_u64 (const uint64_t * __a)
 {
-  uint64x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (uint64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 0);
-  ret.val[1] = (uint64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 1);
-  ret.val[2] = (uint64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 2);
-  ret.val[3] = (uint64x2_t) __builtin_aarch64_get_qregxiv2di (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv2di_us (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 __extension__ extern __inline float16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_f16 (const float16_t * __a)
 {
-  float16x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv8hf ((const __builtin_aarch64_simd_hf *) __a);
-  ret.val[0] = (float16x8_t) __builtin_aarch64_get_qregxiv8hf (__o, 0);
-  ret.val[1] = (float16x8_t) __builtin_aarch64_get_qregxiv8hf (__o, 1);
-  ret.val[2] = (float16x8_t) __builtin_aarch64_get_qregxiv8hf (__o, 2);
-  ret.val[3] = (float16x8_t) __builtin_aarch64_get_qregxiv8hf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv8hf ((const __builtin_aarch64_simd_hf *) __a);
 }
 
 __extension__ extern __inline float32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_f32 (const float32_t * __a)
 {
-  float32x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv4sf ((const __builtin_aarch64_simd_sf *) __a);
-  ret.val[0] = (float32x4_t) __builtin_aarch64_get_qregxiv4sf (__o, 0);
-  ret.val[1] = (float32x4_t) __builtin_aarch64_get_qregxiv4sf (__o, 1);
-  ret.val[2] = (float32x4_t) __builtin_aarch64_get_qregxiv4sf (__o, 2);
-  ret.val[3] = (float32x4_t) __builtin_aarch64_get_qregxiv4sf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv4sf ((const __builtin_aarch64_simd_sf *) __a);
 }
 
 __extension__ extern __inline float64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_f64 (const float64_t * __a)
 {
-  float64x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv2df ((const __builtin_aarch64_simd_df *) __a);
-  ret.val[0] = (float64x2_t) __builtin_aarch64_get_qregxiv2df (__o, 0);
-  ret.val[1] = (float64x2_t) __builtin_aarch64_get_qregxiv2df (__o, 1);
-  ret.val[2] = (float64x2_t) __builtin_aarch64_get_qregxiv2df (__o, 2);
-  ret.val[3] = (float64x2_t) __builtin_aarch64_get_qregxiv2df (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv2df ((const __builtin_aarch64_simd_df *) __a);
 }
 
 __extension__ extern __inline poly64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_p64 (const poly64_t * __a)
 {
-  poly64x2x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv2di ((const __builtin_aarch64_simd_di *) __a);
-  ret.val[0] = (poly64x2_t) __builtin_aarch64_get_qregxiv2di_pss (__o, 0);
-  ret.val[1] = (poly64x2_t) __builtin_aarch64_get_qregxiv2di_pss (__o, 1);
-  ret.val[2] = (poly64x2_t) __builtin_aarch64_get_qregxiv2di_pss (__o, 2);
-  ret.val[3] = (poly64x2_t) __builtin_aarch64_get_qregxiv2di_pss (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv2di_ps (
+				(const __builtin_aarch64_simd_di *) __a);
 }
 
 /* vld2_lane */
@@ -19876,238 +17440,112 @@ __extension__ extern __inline uint8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_u8 (const uint8_t * __ptr, uint8x8x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint8x16x2_t __temp;
-  __temp.val[0] = vcombine_u8 (__b.val[0], vcreate_u8 (0));
-  __temp.val[1] = vcombine_u8 (__b.val[1], vcreate_u8 (0));
-  __o = __builtin_aarch64_set_qregoiv16qi (__o, (int8x16_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv16qi (__o, (int8x16_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanev8qi (
-	  (__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  __b.val[0] = (uint8x8_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (uint8x8_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanev8qi_usus (
+	  (__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_u16 (const uint16_t * __ptr, uint16x4x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint16x8x2_t __temp;
-  __temp.val[0] = vcombine_u16 (__b.val[0], vcreate_u16 (0));
-  __temp.val[1] = vcombine_u16 (__b.val[1], vcreate_u16 (0));
-  __o = __builtin_aarch64_set_qregoiv8hi (__o, (int16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv8hi (__o, (int16x8_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanev4hi (
-	  (__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  __b.val[0] = (uint16x4_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (uint16x4_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanev4hi_usus (
+	  (__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_u32 (const uint32_t * __ptr, uint32x2x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint32x4x2_t __temp;
-  __temp.val[0] = vcombine_u32 (__b.val[0], vcreate_u32 (0));
-  __temp.val[1] = vcombine_u32 (__b.val[1], vcreate_u32 (0));
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanev2si (
-	  (__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  __b.val[0] = (uint32x2_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (uint32x2_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanev2si_usus (
+	  (__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_u64 (const uint64_t * __ptr, uint64x1x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint64x2x2_t __temp;
-  __temp.val[0] = vcombine_u64 (__b.val[0], vcreate_u64 (0));
-  __temp.val[1] = vcombine_u64 (__b.val[1], vcreate_u64 (0));
-  __o = __builtin_aarch64_set_qregoiv2di (__o, (int64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv2di (__o, (int64x2_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanedi (
-	  (__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  __b.val[0] = (uint64x1_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (uint64x1_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanedi_usus (
+	  (__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_s8 (const int8_t * __ptr, int8x8x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  int8x16x2_t __temp;
-  __temp.val[0] = vcombine_s8 (__b.val[0], vcreate_s8 (0));
-  __temp.val[1] = vcombine_s8 (__b.val[1], vcreate_s8 (0));
-  __o = __builtin_aarch64_set_qregoiv16qi (__o, (int8x16_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv16qi (__o, (int8x16_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanev8qi (
-	  (__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  __b.val[0] = (int8x8_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (int8x8_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanev8qi (
+	  (__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_s16 (const int16_t * __ptr, int16x4x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  int16x8x2_t __temp;
-  __temp.val[0] = vcombine_s16 (__b.val[0], vcreate_s16 (0));
-  __temp.val[1] = vcombine_s16 (__b.val[1], vcreate_s16 (0));
-  __o = __builtin_aarch64_set_qregoiv8hi (__o, (int16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv8hi (__o, (int16x8_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanev4hi (
-	  (__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  __b.val[0] = (int16x4_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (int16x4_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanev4hi (
+	  (__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_s32 (const int32_t * __ptr, int32x2x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  int32x4x2_t __temp;
-  __temp.val[0] = vcombine_s32 (__b.val[0], vcreate_s32 (0));
-  __temp.val[1] = vcombine_s32 (__b.val[1], vcreate_s32 (0));
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanev2si (
-	  (__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  __b.val[0] = (int32x2_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (int32x2_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanev2si (
+	  (__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_s64 (const int64_t * __ptr, int64x1x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  int64x2x2_t __temp;
-  __temp.val[0] = vcombine_s64 (__b.val[0], vcreate_s64 (0));
-  __temp.val[1] = vcombine_s64 (__b.val[1], vcreate_s64 (0));
-  __o = __builtin_aarch64_set_qregoiv2di (__o, (int64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv2di (__o, (int64x2_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanedi (
-	  (__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  __b.val[0] = (int64x1_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (int64x1_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanedi (
+	  (__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_f16 (const float16_t * __ptr, float16x4x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  float16x8x2_t __temp;
-  __temp.val[0] = vcombine_f16 (__b.val[0], vcreate_f16 (0));
-  __temp.val[1] = vcombine_f16 (__b.val[1], vcreate_f16 (0));
-  __o = __builtin_aarch64_set_qregoiv8hf (__o, (float16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv8hf (__o, (float16x8_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanev4hf (
-	  (__builtin_aarch64_simd_hf *) __ptr, __o, __c);
-  __b.val[0] = (float16x4_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (float16x4_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanev4hf (
+	  (__builtin_aarch64_simd_hf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float32x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_f32 (const float32_t * __ptr, float32x2x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  float32x4x2_t __temp;
-  __temp.val[0] = vcombine_f32 (__b.val[0], vcreate_f32 (0));
-  __temp.val[1] = vcombine_f32 (__b.val[1], vcreate_f32 (0));
-  __o = __builtin_aarch64_set_qregoiv4sf (__o, (float32x4_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4sf (__o, (float32x4_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanev2sf (
-	  (__builtin_aarch64_simd_sf *) __ptr, __o, __c);
-  __b.val[0] = (float32x2_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (float32x2_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanev2sf (
+	  (__builtin_aarch64_simd_sf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_f64 (const float64_t * __ptr, float64x1x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  float64x2x2_t __temp;
-  __temp.val[0] = vcombine_f64 (__b.val[0], vcreate_f64 (0));
-  __temp.val[1] = vcombine_f64 (__b.val[1], vcreate_f64 (0));
-  __o = __builtin_aarch64_set_qregoiv2df (__o, (float64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv2df (__o, (float64x2_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanedf (
-	  (__builtin_aarch64_simd_df *) __ptr, __o, __c);
-  __b.val[0] = (float64x1_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (float64x1_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanedf (
+	  (__builtin_aarch64_simd_df *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly8x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_p8 (const poly8_t * __ptr, poly8x8x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly8x16x2_t __temp;
-  __temp.val[0] = vcombine_p8 (__b.val[0], vcreate_p8 (0));
-  __temp.val[1] = vcombine_p8 (__b.val[1], vcreate_p8 (0));
-  __o = __builtin_aarch64_set_qregoiv16qi (__o, (int8x16_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv16qi (__o, (int8x16_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanev8qi (
-	  (__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  __b.val[0] = (poly8x8_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (poly8x8_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanev8qi_psps (
+	  (__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_p16 (const poly16_t * __ptr, poly16x4x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly16x8x2_t __temp;
-  __temp.val[0] = vcombine_p16 (__b.val[0], vcreate_p16 (0));
-  __temp.val[1] = vcombine_p16 (__b.val[1], vcreate_p16 (0));
-  __o = __builtin_aarch64_set_qregoiv8hi (__o, (int16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv8hi (__o, (int16x8_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanev4hi (
-	  (__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  __b.val[0] = (poly16x4_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (poly16x4_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanev4hi_psps (
+	  (__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly64x1x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_p64 (const poly64_t * __ptr, poly64x1x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly64x2x2_t __temp;
-  __temp.val[0] = vcombine_p64 (__b.val[0], vcreate_p64 (0));
-  __temp.val[1] = vcombine_p64 (__b.val[1], vcreate_p64 (0));
-  __o = __builtin_aarch64_set_qregoiv2di (__o, (int64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv2di (__o, (int64x2_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanedi (
-	  (__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  __b.val[0] = (poly64x1_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (poly64x1_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanedi_psps (
+	  (__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 /* vld2q_lane */
@@ -20116,210 +17554,112 @@ __extension__ extern __inline uint8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_u8 (const uint8_t * __ptr, uint8x16x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint8x16x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev16qi (
-	(__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  ret.val[0] = (uint8x16_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (uint8x16_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev16qi_usus (
+	(__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_u16 (const uint16_t * __ptr, uint16x8x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint16x8x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev8hi (
-	(__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  ret.val[0] = (uint16x8_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (uint16x8_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev8hi_usus (
+	(__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_u32 (const uint32_t * __ptr, uint32x4x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint32x4x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev4si (
-	(__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  ret.val[0] = (uint32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (uint32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev4si_usus (
+	(__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_u64 (const uint64_t * __ptr, uint64x2x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint64x2x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev2di (
-	(__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  ret.val[0] = (uint64x2_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (uint64x2_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev2di_usus (
+	(__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_s8 (const int8_t * __ptr, int8x16x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  int8x16x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev16qi (
-	(__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  ret.val[0] = (int8x16_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (int8x16_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev16qi (
+	(__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_s16 (const int16_t * __ptr, int16x8x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  int16x8x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev8hi (
-	(__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  ret.val[0] = (int16x8_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (int16x8_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev8hi (
+	(__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_s32 (const int32_t * __ptr, int32x4x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  int32x4x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev4si (
-	(__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  ret.val[0] = __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev4si (
+	(__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_s64 (const int64_t * __ptr, int64x2x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  int64x2x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev2di (
-	(__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  ret.val[0] = (int64x2_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (int64x2_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev2di (
+	(__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_f16 (const float16_t * __ptr, float16x8x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  float16x8x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev8hf (
-	(__builtin_aarch64_simd_hf *) __ptr, __o, __c);
-  ret.val[0] = (float16x8_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (float16x8_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev8hf (
+	(__builtin_aarch64_simd_hf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float32x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_f32 (const float32_t * __ptr, float32x4x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  float32x4x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev4sf (
-	(__builtin_aarch64_simd_sf *) __ptr, __o, __c);
-  ret.val[0] = (float32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (float32x4_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev4sf (
+	(__builtin_aarch64_simd_sf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_f64 (const float64_t * __ptr, float64x2x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  float64x2x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev2df (
-	(__builtin_aarch64_simd_df *) __ptr, __o, __c);
-  ret.val[0] = (float64x2_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (float64x2_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev2df (
+	(__builtin_aarch64_simd_df *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly8x16x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_p8 (const poly8_t * __ptr, poly8x16x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly8x16x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev16qi (
-	(__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  ret.val[0] = (poly8x16_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (poly8x16_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev16qi_psps (
+	(__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_p16 (const poly16_t * __ptr, poly16x8x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly16x8x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev8hi (
-	(__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  ret.val[0] = (poly16x8_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (poly16x8_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev8hi_psps (
+	(__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly64x2x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_p64 (const poly64_t * __ptr, poly64x2x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly64x2x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev2di (
-	(__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  ret.val[0] = (poly64x2_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (poly64x2_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev2di_psps (
+	(__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 /* vld3_lane */
@@ -20328,280 +17668,112 @@ __extension__ extern __inline uint8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_u8 (const uint8_t * __ptr, uint8x8x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint8x16x3_t __temp;
-  __temp.val[0] = vcombine_u8 (__b.val[0], vcreate_u8 (0));
-  __temp.val[1] = vcombine_u8 (__b.val[1], vcreate_u8 (0));
-  __temp.val[2] = vcombine_u8 (__b.val[2], vcreate_u8 (0));
-  __o = __builtin_aarch64_set_qregciv16qi (__o, (int8x16_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv16qi (__o, (int8x16_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv16qi (__o, (int8x16_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanev8qi (
-	  (__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  __b.val[0] = (uint8x8_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (uint8x8_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (uint8x8_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanev8qi_usus (
+	  (__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_u16 (const uint16_t * __ptr, uint16x4x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint16x8x3_t __temp;
-  __temp.val[0] = vcombine_u16 (__b.val[0], vcreate_u16 (0));
-  __temp.val[1] = vcombine_u16 (__b.val[1], vcreate_u16 (0));
-  __temp.val[2] = vcombine_u16 (__b.val[2], vcreate_u16 (0));
-  __o = __builtin_aarch64_set_qregciv8hi (__o, (int16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv8hi (__o, (int16x8_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv8hi (__o, (int16x8_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanev4hi (
-	  (__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  __b.val[0] = (uint16x4_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (uint16x4_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (uint16x4_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanev4hi_usus (
+	  (__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_u32 (const uint32_t * __ptr, uint32x2x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint32x4x3_t __temp;
-  __temp.val[0] = vcombine_u32 (__b.val[0], vcreate_u32 (0));
-  __temp.val[1] = vcombine_u32 (__b.val[1], vcreate_u32 (0));
-  __temp.val[2] = vcombine_u32 (__b.val[2], vcreate_u32 (0));
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanev2si (
-	  (__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  __b.val[0] = (uint32x2_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (uint32x2_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (uint32x2_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanev2si_usus (
+	  (__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_u64 (const uint64_t * __ptr, uint64x1x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint64x2x3_t __temp;
-  __temp.val[0] = vcombine_u64 (__b.val[0], vcreate_u64 (0));
-  __temp.val[1] = vcombine_u64 (__b.val[1], vcreate_u64 (0));
-  __temp.val[2] = vcombine_u64 (__b.val[2], vcreate_u64 (0));
-  __o = __builtin_aarch64_set_qregciv2di (__o, (int64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv2di (__o, (int64x2_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv2di (__o, (int64x2_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanedi (
-	  (__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  __b.val[0] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (uint64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanedi_usus (
+	  (__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_s8 (const int8_t * __ptr, int8x8x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  int8x16x3_t __temp;
-  __temp.val[0] = vcombine_s8 (__b.val[0], vcreate_s8 (0));
-  __temp.val[1] = vcombine_s8 (__b.val[1], vcreate_s8 (0));
-  __temp.val[2] = vcombine_s8 (__b.val[2], vcreate_s8 (0));
-  __o = __builtin_aarch64_set_qregciv16qi (__o, (int8x16_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv16qi (__o, (int8x16_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv16qi (__o, (int8x16_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanev8qi (
-	  (__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  __b.val[0] = (int8x8_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (int8x8_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (int8x8_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanev8qi (
+	  (__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_s16 (const int16_t * __ptr, int16x4x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  int16x8x3_t __temp;
-  __temp.val[0] = vcombine_s16 (__b.val[0], vcreate_s16 (0));
-  __temp.val[1] = vcombine_s16 (__b.val[1], vcreate_s16 (0));
-  __temp.val[2] = vcombine_s16 (__b.val[2], vcreate_s16 (0));
-  __o = __builtin_aarch64_set_qregciv8hi (__o, (int16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv8hi (__o, (int16x8_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv8hi (__o, (int16x8_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanev4hi (
-	  (__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  __b.val[0] = (int16x4_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (int16x4_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (int16x4_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanev4hi (
+	  (__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_s32 (const int32_t * __ptr, int32x2x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  int32x4x3_t __temp;
-  __temp.val[0] = vcombine_s32 (__b.val[0], vcreate_s32 (0));
-  __temp.val[1] = vcombine_s32 (__b.val[1], vcreate_s32 (0));
-  __temp.val[2] = vcombine_s32 (__b.val[2], vcreate_s32 (0));
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanev2si (
-	  (__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  __b.val[0] = (int32x2_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (int32x2_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (int32x2_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanev2si (
+	  (__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_s64 (const int64_t * __ptr, int64x1x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  int64x2x3_t __temp;
-  __temp.val[0] = vcombine_s64 (__b.val[0], vcreate_s64 (0));
-  __temp.val[1] = vcombine_s64 (__b.val[1], vcreate_s64 (0));
-  __temp.val[2] = vcombine_s64 (__b.val[2], vcreate_s64 (0));
-  __o = __builtin_aarch64_set_qregciv2di (__o, (int64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv2di (__o, (int64x2_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv2di (__o, (int64x2_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanedi (
-	  (__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  __b.val[0] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (int64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanedi (
+	  (__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_f16 (const float16_t * __ptr, float16x4x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  float16x8x3_t __temp;
-  __temp.val[0] = vcombine_f16 (__b.val[0], vcreate_f16 (0));
-  __temp.val[1] = vcombine_f16 (__b.val[1], vcreate_f16 (0));
-  __temp.val[2] = vcombine_f16 (__b.val[2], vcreate_f16 (0));
-  __o = __builtin_aarch64_set_qregciv8hf (__o, (float16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv8hf (__o, (float16x8_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv8hf (__o, (float16x8_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanev4hf (
-	  (__builtin_aarch64_simd_hf *) __ptr, __o, __c);
-  __b.val[0] = (float16x4_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (float16x4_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (float16x4_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanev4hf (
+	  (__builtin_aarch64_simd_hf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float32x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_f32 (const float32_t * __ptr, float32x2x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  float32x4x3_t __temp;
-  __temp.val[0] = vcombine_f32 (__b.val[0], vcreate_f32 (0));
-  __temp.val[1] = vcombine_f32 (__b.val[1], vcreate_f32 (0));
-  __temp.val[2] = vcombine_f32 (__b.val[2], vcreate_f32 (0));
-  __o = __builtin_aarch64_set_qregciv4sf (__o, (float32x4_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4sf (__o, (float32x4_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4sf (__o, (float32x4_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanev2sf (
-	  (__builtin_aarch64_simd_sf *) __ptr, __o, __c);
-  __b.val[0] = (float32x2_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (float32x2_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (float32x2_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanev2sf (
+	  (__builtin_aarch64_simd_sf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_f64 (const float64_t * __ptr, float64x1x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  float64x2x3_t __temp;
-  __temp.val[0] = vcombine_f64 (__b.val[0], vcreate_f64 (0));
-  __temp.val[1] = vcombine_f64 (__b.val[1], vcreate_f64 (0));
-  __temp.val[2] = vcombine_f64 (__b.val[2], vcreate_f64 (0));
-  __o = __builtin_aarch64_set_qregciv2df (__o, (float64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv2df (__o, (float64x2_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv2df (__o, (float64x2_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanedf (
-	  (__builtin_aarch64_simd_df *) __ptr, __o, __c);
-  __b.val[0] = (float64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (float64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (float64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanedf (
+	  (__builtin_aarch64_simd_df *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly8x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_p8 (const poly8_t * __ptr, poly8x8x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly8x16x3_t __temp;
-  __temp.val[0] = vcombine_p8 (__b.val[0], vcreate_p8 (0));
-  __temp.val[1] = vcombine_p8 (__b.val[1], vcreate_p8 (0));
-  __temp.val[2] = vcombine_p8 (__b.val[2], vcreate_p8 (0));
-  __o = __builtin_aarch64_set_qregciv16qi (__o, (int8x16_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv16qi (__o, (int8x16_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv16qi (__o, (int8x16_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanev8qi (
-	  (__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  __b.val[0] = (poly8x8_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (poly8x8_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (poly8x8_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanev8qi_psps (
+	  (__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_p16 (const poly16_t * __ptr, poly16x4x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly16x8x3_t __temp;
-  __temp.val[0] = vcombine_p16 (__b.val[0], vcreate_p16 (0));
-  __temp.val[1] = vcombine_p16 (__b.val[1], vcreate_p16 (0));
-  __temp.val[2] = vcombine_p16 (__b.val[2], vcreate_p16 (0));
-  __o = __builtin_aarch64_set_qregciv8hi (__o, (int16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv8hi (__o, (int16x8_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv8hi (__o, (int16x8_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanev4hi (
-	  (__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  __b.val[0] = (poly16x4_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (poly16x4_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (poly16x4_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanev4hi_psps (
+	  (__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly64x1x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_p64 (const poly64_t * __ptr, poly64x1x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly64x2x3_t __temp;
-  __temp.val[0] = vcombine_p64 (__b.val[0], vcreate_p64 (0));
-  __temp.val[1] = vcombine_p64 (__b.val[1], vcreate_p64 (0));
-  __temp.val[2] = vcombine_p64 (__b.val[2], vcreate_p64 (0));
-  __o = __builtin_aarch64_set_qregciv2di (__o, (int64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv2di (__o, (int64x2_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv2di (__o, (int64x2_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanedi (
-	  (__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  __b.val[0] = (poly64x1_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (poly64x1_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (poly64x1_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanedi_psps (
+	  (__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 /* vld3q_lane */
@@ -20610,238 +17782,112 @@ __extension__ extern __inline uint8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_u8 (const uint8_t * __ptr, uint8x16x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint8x16x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev16qi (
-	(__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  ret.val[0] = (uint8x16_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (uint8x16_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (uint8x16_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev16qi_usus (
+	(__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_u16 (const uint16_t * __ptr, uint16x8x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint16x8x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev8hi (
-	(__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  ret.val[0] = (uint16x8_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (uint16x8_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (uint16x8_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev8hi_usus (
+	(__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_u32 (const uint32_t * __ptr, uint32x4x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint32x4x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev4si (
-	(__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  ret.val[0] = (uint32x4_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (uint32x4_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (uint32x4_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev4si_usus (
+	(__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_u64 (const uint64_t * __ptr, uint64x2x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint64x2x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev2di (
-	(__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  ret.val[0] = (uint64x2_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (uint64x2_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (uint64x2_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev2di_usus (
+	(__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_s8 (const int8_t * __ptr, int8x16x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  int8x16x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev16qi (
-	(__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  ret.val[0] = (int8x16_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (int8x16_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (int8x16_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev16qi (
+	(__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_s16 (const int16_t * __ptr, int16x8x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  int16x8x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev8hi (
-	(__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  ret.val[0] = (int16x8_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (int16x8_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (int16x8_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev8hi (
+	(__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_s32 (const int32_t * __ptr, int32x4x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  int32x4x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev4si (
-	(__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  ret.val[0] = (int32x4_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (int32x4_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (int32x4_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev4si (
+	(__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_s64 (const int64_t * __ptr, int64x2x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  int64x2x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev2di (
-	(__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  ret.val[0] = (int64x2_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (int64x2_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (int64x2_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev2di (
+	(__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_f16 (const float16_t * __ptr, float16x8x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  float16x8x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev8hf (
-	(__builtin_aarch64_simd_hf *) __ptr, __o, __c);
-  ret.val[0] = (float16x8_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (float16x8_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (float16x8_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev8hf (
+	(__builtin_aarch64_simd_hf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float32x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_f32 (const float32_t * __ptr, float32x4x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  float32x4x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev4sf (
-	(__builtin_aarch64_simd_sf *) __ptr, __o, __c);
-  ret.val[0] = (float32x4_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (float32x4_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (float32x4_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev4sf (
+	(__builtin_aarch64_simd_sf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_f64 (const float64_t * __ptr, float64x2x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  float64x2x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev2df (
-	(__builtin_aarch64_simd_df *) __ptr, __o, __c);
-  ret.val[0] = (float64x2_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (float64x2_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (float64x2_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev2df (
+	(__builtin_aarch64_simd_df *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly8x16x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_p8 (const poly8_t * __ptr, poly8x16x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly8x16x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev16qi (
-	(__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  ret.val[0] = (poly8x16_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (poly8x16_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (poly8x16_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev16qi_psps (
+	(__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_p16 (const poly16_t * __ptr, poly16x8x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly16x8x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev8hi (
-	(__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  ret.val[0] = (poly16x8_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (poly16x8_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (poly16x8_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev8hi_psps (
+	(__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly64x2x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_p64 (const poly64_t * __ptr, poly64x2x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly64x2x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev2di (
-	(__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  ret.val[0] = (poly64x2_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (poly64x2_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (poly64x2_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev2di_psps (
+	(__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 /* vld4_lane */
@@ -20850,322 +17896,112 @@ __extension__ extern __inline uint8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_u8 (const uint8_t * __ptr, uint8x8x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint8x16x4_t __temp;
-  __temp.val[0] = vcombine_u8 (__b.val[0], vcreate_u8 (0));
-  __temp.val[1] = vcombine_u8 (__b.val[1], vcreate_u8 (0));
-  __temp.val[2] = vcombine_u8 (__b.val[2], vcreate_u8 (0));
-  __temp.val[3] = vcombine_u8 (__b.val[3], vcreate_u8 (0));
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanev8qi (
-	  (__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  __b.val[0] = (uint8x8_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (uint8x8_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (uint8x8_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (uint8x8_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanev8qi_usus (
+	  (__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_u16 (const uint16_t * __ptr, uint16x4x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint16x8x4_t __temp;
-  __temp.val[0] = vcombine_u16 (__b.val[0], vcreate_u16 (0));
-  __temp.val[1] = vcombine_u16 (__b.val[1], vcreate_u16 (0));
-  __temp.val[2] = vcombine_u16 (__b.val[2], vcreate_u16 (0));
-  __temp.val[3] = vcombine_u16 (__b.val[3], vcreate_u16 (0));
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanev4hi (
-	  (__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  __b.val[0] = (uint16x4_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (uint16x4_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (uint16x4_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (uint16x4_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanev4hi_usus (
+	  (__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_u32 (const uint32_t * __ptr, uint32x2x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint32x4x4_t __temp;
-  __temp.val[0] = vcombine_u32 (__b.val[0], vcreate_u32 (0));
-  __temp.val[1] = vcombine_u32 (__b.val[1], vcreate_u32 (0));
-  __temp.val[2] = vcombine_u32 (__b.val[2], vcreate_u32 (0));
-  __temp.val[3] = vcombine_u32 (__b.val[3], vcreate_u32 (0));
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanev2si (
-	  (__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  __b.val[0] = (uint32x2_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (uint32x2_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (uint32x2_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (uint32x2_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanev2si_usus (
+	  (__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_u64 (const uint64_t * __ptr, uint64x1x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint64x2x4_t __temp;
-  __temp.val[0] = vcombine_u64 (__b.val[0], vcreate_u64 (0));
-  __temp.val[1] = vcombine_u64 (__b.val[1], vcreate_u64 (0));
-  __temp.val[2] = vcombine_u64 (__b.val[2], vcreate_u64 (0));
-  __temp.val[3] = vcombine_u64 (__b.val[3], vcreate_u64 (0));
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanedi (
-	  (__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  __b.val[0] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (uint64x1_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanedi_usus (
+	  (__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_s8 (const int8_t * __ptr, int8x8x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  int8x16x4_t __temp;
-  __temp.val[0] = vcombine_s8 (__b.val[0], vcreate_s8 (0));
-  __temp.val[1] = vcombine_s8 (__b.val[1], vcreate_s8 (0));
-  __temp.val[2] = vcombine_s8 (__b.val[2], vcreate_s8 (0));
-  __temp.val[3] = vcombine_s8 (__b.val[3], vcreate_s8 (0));
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanev8qi (
-	  (__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  __b.val[0] = (int8x8_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (int8x8_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (int8x8_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (int8x8_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanev8qi (
+	  (__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_s16 (const int16_t * __ptr, int16x4x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  int16x8x4_t __temp;
-  __temp.val[0] = vcombine_s16 (__b.val[0], vcreate_s16 (0));
-  __temp.val[1] = vcombine_s16 (__b.val[1], vcreate_s16 (0));
-  __temp.val[2] = vcombine_s16 (__b.val[2], vcreate_s16 (0));
-  __temp.val[3] = vcombine_s16 (__b.val[3], vcreate_s16 (0));
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanev4hi (
-	  (__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  __b.val[0] = (int16x4_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (int16x4_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (int16x4_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (int16x4_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanev4hi (
+	  (__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_s32 (const int32_t * __ptr, int32x2x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  int32x4x4_t __temp;
-  __temp.val[0] = vcombine_s32 (__b.val[0], vcreate_s32 (0));
-  __temp.val[1] = vcombine_s32 (__b.val[1], vcreate_s32 (0));
-  __temp.val[2] = vcombine_s32 (__b.val[2], vcreate_s32 (0));
-  __temp.val[3] = vcombine_s32 (__b.val[3], vcreate_s32 (0));
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanev2si (
-	  (__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  __b.val[0] = (int32x2_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (int32x2_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (int32x2_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (int32x2_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanev2si (
+	  (__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_s64 (const int64_t * __ptr, int64x1x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  int64x2x4_t __temp;
-  __temp.val[0] = vcombine_s64 (__b.val[0], vcreate_s64 (0));
-  __temp.val[1] = vcombine_s64 (__b.val[1], vcreate_s64 (0));
-  __temp.val[2] = vcombine_s64 (__b.val[2], vcreate_s64 (0));
-  __temp.val[3] = vcombine_s64 (__b.val[3], vcreate_s64 (0));
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanedi (
-	  (__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  __b.val[0] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (int64x1_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanedi (
+	  (__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_f16 (const float16_t * __ptr, float16x4x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  float16x8x4_t __temp;
-  __temp.val[0] = vcombine_f16 (__b.val[0], vcreate_f16 (0));
-  __temp.val[1] = vcombine_f16 (__b.val[1], vcreate_f16 (0));
-  __temp.val[2] = vcombine_f16 (__b.val[2], vcreate_f16 (0));
-  __temp.val[3] = vcombine_f16 (__b.val[3], vcreate_f16 (0));
-  __o = __builtin_aarch64_set_qregxiv8hf (__o, (float16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv8hf (__o, (float16x8_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv8hf (__o, (float16x8_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv8hf (__o, (float16x8_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanev4hf (
-	  (__builtin_aarch64_simd_hf *) __ptr, __o, __c);
-  __b.val[0] = (float16x4_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (float16x4_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (float16x4_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (float16x4_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanev4hf (
+	  (__builtin_aarch64_simd_hf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float32x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_f32 (const float32_t * __ptr, float32x2x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  float32x4x4_t __temp;
-  __temp.val[0] = vcombine_f32 (__b.val[0], vcreate_f32 (0));
-  __temp.val[1] = vcombine_f32 (__b.val[1], vcreate_f32 (0));
-  __temp.val[2] = vcombine_f32 (__b.val[2], vcreate_f32 (0));
-  __temp.val[3] = vcombine_f32 (__b.val[3], vcreate_f32 (0));
-  __o = __builtin_aarch64_set_qregxiv4sf (__o, (float32x4_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4sf (__o, (float32x4_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4sf (__o, (float32x4_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4sf (__o, (float32x4_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanev2sf (
-	  (__builtin_aarch64_simd_sf *) __ptr, __o, __c);
-  __b.val[0] = (float32x2_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (float32x2_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (float32x2_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (float32x2_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanev2sf (
+	  (__builtin_aarch64_simd_sf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_f64 (const float64_t * __ptr, float64x1x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  float64x2x4_t __temp;
-  __temp.val[0] = vcombine_f64 (__b.val[0], vcreate_f64 (0));
-  __temp.val[1] = vcombine_f64 (__b.val[1], vcreate_f64 (0));
-  __temp.val[2] = vcombine_f64 (__b.val[2], vcreate_f64 (0));
-  __temp.val[3] = vcombine_f64 (__b.val[3], vcreate_f64 (0));
-  __o = __builtin_aarch64_set_qregxiv2df (__o, (float64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv2df (__o, (float64x2_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv2df (__o, (float64x2_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv2df (__o, (float64x2_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanedf (
-	  (__builtin_aarch64_simd_df *) __ptr, __o, __c);
-  __b.val[0] = (float64x1_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (float64x1_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (float64x1_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (float64x1_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanedf (
+	  (__builtin_aarch64_simd_df *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly8x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_p8 (const poly8_t * __ptr, poly8x8x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly8x16x4_t __temp;
-  __temp.val[0] = vcombine_p8 (__b.val[0], vcreate_p8 (0));
-  __temp.val[1] = vcombine_p8 (__b.val[1], vcreate_p8 (0));
-  __temp.val[2] = vcombine_p8 (__b.val[2], vcreate_p8 (0));
-  __temp.val[3] = vcombine_p8 (__b.val[3], vcreate_p8 (0));
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv16qi (__o, (int8x16_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanev8qi (
-	  (__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  __b.val[0] = (poly8x8_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (poly8x8_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (poly8x8_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (poly8x8_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanev8qi_psps (
+	  (__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_p16 (const poly16_t * __ptr, poly16x4x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly16x8x4_t __temp;
-  __temp.val[0] = vcombine_p16 (__b.val[0], vcreate_p16 (0));
-  __temp.val[1] = vcombine_p16 (__b.val[1], vcreate_p16 (0));
-  __temp.val[2] = vcombine_p16 (__b.val[2], vcreate_p16 (0));
-  __temp.val[3] = vcombine_p16 (__b.val[3], vcreate_p16 (0));
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv8hi (__o, (int16x8_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanev4hi (
-	  (__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  __b.val[0] = (poly16x4_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (poly16x4_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (poly16x4_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (poly16x4_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanev4hi_psps (
+	  (__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly64x1x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_p64 (const poly64_t * __ptr, poly64x1x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly64x2x4_t __temp;
-  __temp.val[0] = vcombine_p64 (__b.val[0], vcreate_p64 (0));
-  __temp.val[1] = vcombine_p64 (__b.val[1], vcreate_p64 (0));
-  __temp.val[2] = vcombine_p64 (__b.val[2], vcreate_p64 (0));
-  __temp.val[3] = vcombine_p64 (__b.val[3], vcreate_p64 (0));
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv2di (__o, (int64x2_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanedi (
-	  (__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  __b.val[0] = (poly64x1_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (poly64x1_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (poly64x1_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (poly64x1_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanedi_psps (
+	  (__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 /* vld4q_lane */
@@ -21174,266 +18010,112 @@ __extension__ extern __inline uint8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_u8 (const uint8_t * __ptr, uint8x16x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint8x16x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev16qi (
-	(__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  ret.val[0] = (uint8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (uint8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (uint8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (uint8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev16qi_usus (
+	(__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_u16 (const uint16_t * __ptr, uint16x8x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint16x8x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev8hi (
-	(__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  ret.val[0] = (uint16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (uint16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (uint16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (uint16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev8hi_usus (
+	(__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_u32 (const uint32_t * __ptr, uint32x4x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint32x4x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev4si (
-	(__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  ret.val[0] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (uint32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev4si_usus (
+	(__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline uint64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_u64 (const uint64_t * __ptr, uint64x2x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint64x2x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev2di (
-	(__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  ret.val[0] = (uint64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (uint64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (uint64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (uint64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev2di_usus (
+	(__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_s8 (const int8_t * __ptr, int8x16x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  int8x16x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev16qi (
-	(__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  ret.val[0] = (int8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (int8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (int8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (int8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev16qi (
+	(__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_s16 (const int16_t * __ptr, int16x8x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  int16x8x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev8hi (
-	(__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  ret.val[0] = (int16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (int16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (int16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (int16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev8hi (
+	(__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_s32 (const int32_t * __ptr, int32x4x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  int32x4x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev4si (
-	(__builtin_aarch64_simd_si *) __ptr, __o, __c);
-  ret.val[0] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (int32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev4si (
+	(__builtin_aarch64_simd_si *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline int64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_s64 (const int64_t * __ptr, int64x2x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  int64x2x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev2di (
-	(__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  ret.val[0] = (int64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (int64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (int64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (int64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev2di (
+	(__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_f16 (const float16_t * __ptr, float16x8x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  float16x8x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev8hf (
-	(__builtin_aarch64_simd_hf *) __ptr, __o, __c);
-  ret.val[0] = (float16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (float16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (float16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (float16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev8hf (
+	(__builtin_aarch64_simd_hf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float32x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_f32 (const float32_t * __ptr, float32x4x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  float32x4x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev4sf (
-	(__builtin_aarch64_simd_sf *) __ptr, __o, __c);
-  ret.val[0] = (float32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (float32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (float32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (float32x4_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev4sf (
+	(__builtin_aarch64_simd_sf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline float64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_f64 (const float64_t * __ptr, float64x2x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  float64x2x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev2df (
-	(__builtin_aarch64_simd_df *) __ptr, __o, __c);
-  ret.val[0] = (float64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (float64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (float64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (float64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev2df (
+	(__builtin_aarch64_simd_df *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly8x16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_p8 (const poly8_t * __ptr, poly8x16x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly8x16x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev16qi (
-	(__builtin_aarch64_simd_qi *) __ptr, __o, __c);
-  ret.val[0] = (poly8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (poly8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (poly8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (poly8x16_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev16qi_psps (
+	(__builtin_aarch64_simd_qi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_p16 (const poly16_t * __ptr, poly16x8x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly16x8x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev8hi (
-	(__builtin_aarch64_simd_hi *) __ptr, __o, __c);
-  ret.val[0] = (poly16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (poly16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (poly16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (poly16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev8hi_psps (
+	(__builtin_aarch64_simd_hi *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline poly64x2x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_p64 (const poly64_t * __ptr, poly64x2x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly64x2x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev2di (
-	(__builtin_aarch64_simd_di *) __ptr, __o, __c);
-  ret.val[0] = (poly64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (poly64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (poly64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (poly64x2_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev2di_psps (
+	(__builtin_aarch64_simd_di *) __ptr, __b, __c);
 }
 
 /* vmax */
@@ -21442,7 +18124,7 @@ __extension__ extern __inline float32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmax_f32 (float32x2_t __a, float32x2_t __b)
 {
-  return __builtin_aarch64_smax_nanv2sf (__a, __b);
+  return __builtin_aarch64_fmax_nanv2sf (__a, __b);
 }
 
 __extension__ extern __inline float64x1_t
@@ -21450,7 +18132,7 @@ __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmax_f64 (float64x1_t __a, float64x1_t __b)
 {
     return (float64x1_t)
-      { __builtin_aarch64_smax_nandf (vget_lane_f64 (__a, 0),
+      { __builtin_aarch64_fmax_nandf (vget_lane_f64 (__a, 0),
 				      vget_lane_f64 (__b, 0)) };
 }
 
@@ -21503,14 +18185,14 @@ __extension__ extern __inline float32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmaxq_f32 (float32x4_t __a, float32x4_t __b)
 {
-  return __builtin_aarch64_smax_nanv4sf (__a, __b);
+  return __builtin_aarch64_fmax_nanv4sf (__a, __b);
 }
 
 __extension__ extern __inline float64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmaxq_f64 (float64x2_t __a, float64x2_t __b)
 {
-  return __builtin_aarch64_smax_nanv2df (__a, __b);
+  return __builtin_aarch64_fmax_nanv2df (__a, __b);
 }
 
 __extension__ extern __inline int8x16_t
@@ -22181,7 +18863,7 @@ __extension__ extern __inline float32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmin_f32 (float32x2_t __a, float32x2_t __b)
 {
-  return __builtin_aarch64_smin_nanv2sf (__a, __b);
+  return __builtin_aarch64_fmin_nanv2sf (__a, __b);
 }
 
 __extension__ extern __inline float64x1_t
@@ -22189,7 +18871,7 @@ __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmin_f64 (float64x1_t __a, float64x1_t __b)
 {
     return (float64x1_t)
-	  { __builtin_aarch64_smin_nandf (vget_lane_f64 (__a, 0),
+	  { __builtin_aarch64_fmin_nandf (vget_lane_f64 (__a, 0),
 					  vget_lane_f64 (__b, 0)) };
 }
 
@@ -22242,14 +18924,14 @@ __extension__ extern __inline float32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vminq_f32 (float32x4_t __a, float32x4_t __b)
 {
-  return __builtin_aarch64_smin_nanv4sf (__a, __b);
+  return __builtin_aarch64_fmin_nanv4sf (__a, __b);
 }
 
 __extension__ extern __inline float64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vminq_f64 (float64x2_t __a, float64x2_t __b)
 {
-  return __builtin_aarch64_smin_nanv2df (__a, __b);
+  return __builtin_aarch64_fmin_nanv2df (__a, __b);
 }
 
 __extension__ extern __inline int8x16_t
@@ -23602,24 +20284,21 @@ __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vpadd_u8 (uint8x8_t __a, uint8x8_t __b)
 {
-  return (uint8x8_t) __builtin_aarch64_addpv8qi ((int8x8_t) __a,
-						 (int8x8_t) __b);
+  return __builtin_aarch64_addpv8qi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vpadd_u16 (uint16x4_t __a, uint16x4_t __b)
 {
-  return (uint16x4_t) __builtin_aarch64_addpv4hi ((int16x4_t) __a,
-						  (int16x4_t) __b);
+  return __builtin_aarch64_addpv4hi_uuu (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vpadd_u32 (uint32x2_t __a, uint32x2_t __b)
 {
-  return (uint32x2_t) __builtin_aarch64_addpv2si ((int32x2_t) __a,
-						  (int32x2_t) __b);
+  return __builtin_aarch64_addpv2si_uuu (__a, __b);
 }
 
 __extension__ extern __inline float32_t
@@ -23647,7 +20326,7 @@ __extension__ extern __inline uint64_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vpaddd_u64 (uint64x2_t __a)
 {
-  return __builtin_aarch64_addpdi ((int64x2_t) __a);
+  return __builtin_aarch64_addpdi_uu (__a);
 }
 
 /* vqabs */
@@ -25450,54 +22129,42 @@ __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl2_s8 (int8x16x2_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbl2v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl2v8qi_ssu (__tab, __idx);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl2_u8 (uint8x16x2_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x8_t)__builtin_aarch64_qtbl2v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl2v8qi_uuu (__tab, __idx);
 }
 
 __extension__ extern __inline poly8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl2_p8 (poly8x16x2_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x8_t)__builtin_aarch64_qtbl2v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl2v8qi_ppu (__tab, __idx);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl2q_s8 (int8x16x2_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbl2v16qi (__o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbl2v16qi_ssu (__tab, __idx);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl2q_u8 (uint8x16x2_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x16_t)__builtin_aarch64_qtbl2v16qi (__o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbl2v16qi_uuu (__tab, __idx);
 }
 
 __extension__ extern __inline poly8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl2q_p8 (poly8x16x2_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x16_t)__builtin_aarch64_qtbl2v16qi (__o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbl2v16qi_ppu (__tab, __idx);
 }
 
 /* vqtbl3 */
@@ -25506,54 +22173,42 @@ __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl3_s8 (int8x16x3_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbl3v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl3v8qi_ssu (__tab, __idx);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl3_u8 (uint8x16x3_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x8_t)__builtin_aarch64_qtbl3v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl3v8qi_uuu (__tab, __idx);
 }
 
 __extension__ extern __inline poly8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl3_p8 (poly8x16x3_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x8_t)__builtin_aarch64_qtbl3v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl3v8qi_ppu (__tab, __idx);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl3q_s8 (int8x16x3_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbl3v16qi (__o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbl3v16qi_ssu (__tab, __idx);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl3q_u8 (uint8x16x3_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x16_t)__builtin_aarch64_qtbl3v16qi (__o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbl3v16qi_uuu (__tab, __idx);
 }
 
 __extension__ extern __inline poly8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl3q_p8 (poly8x16x3_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x16_t)__builtin_aarch64_qtbl3v16qi (__o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbl3v16qi_ppu (__tab, __idx);
 }
 
 /* vqtbl4 */
@@ -25562,54 +22217,42 @@ __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl4_s8 (int8x16x4_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbl4v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl4v8qi_ssu (__tab, __idx);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl4_u8 (uint8x16x4_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x8_t)__builtin_aarch64_qtbl4v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl4v8qi_uuu (__tab, __idx);
 }
 
 __extension__ extern __inline poly8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl4_p8 (poly8x16x4_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x8_t)__builtin_aarch64_qtbl4v8qi (__o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbl4v8qi_ppu (__tab, __idx);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl4q_s8 (int8x16x4_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbl4v16qi (__o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbl4v16qi_ssu (__tab, __idx);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl4q_u8 (uint8x16x4_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x16_t)__builtin_aarch64_qtbl4v16qi (__o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbl4v16qi_uuu (__tab, __idx);
 }
 
 __extension__ extern __inline poly8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbl4q_p8 (poly8x16x4_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x16_t)__builtin_aarch64_qtbl4v16qi (__o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbl4v16qi_ppu (__tab, __idx);
 }
 
 /* vqtbx2 */
@@ -25618,58 +22261,42 @@ __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx2_s8 (int8x8_t __r, int8x16x2_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbx2v8qi (__r, __o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbx2v8qi_sssu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx2_u8 (uint8x8_t __r, uint8x16x2_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x8_t)__builtin_aarch64_qtbx2v8qi ((int8x8_t)__r, __o,
-						 (int8x8_t)__idx);
+  return __builtin_aarch64_qtbx2v8qi_uuuu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline poly8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx2_p8 (poly8x8_t __r, poly8x16x2_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x8_t)__builtin_aarch64_qtbx2v8qi ((int8x8_t)__r, __o,
-						 (int8x8_t)__idx);
+  return __builtin_aarch64_qtbx2v8qi_pppu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx2q_s8 (int8x16_t __r, int8x16x2_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbx2v16qi (__r, __o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbx2v16qi_sssu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx2q_u8 (uint8x16_t __r, uint8x16x2_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x16_t)__builtin_aarch64_qtbx2v16qi ((int8x16_t)__r, __o,
-						   (int8x16_t)__idx);
+  return __builtin_aarch64_qtbx2v16qi_uuuu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline poly8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx2q_p8 (poly8x16_t __r, poly8x16x2_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x16_t)__builtin_aarch64_qtbx2v16qi ((int8x16_t)__r, __o,
-						   (int8x16_t)__idx);
+  return __builtin_aarch64_qtbx2v16qi_pppu (__r, __tab, __idx);
 }
 
 /* vqtbx3 */
@@ -25678,58 +22305,42 @@ __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx3_s8 (int8x8_t __r, int8x16x3_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbx3v8qi (__r, __o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbx3v8qi_sssu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx3_u8 (uint8x8_t __r, uint8x16x3_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x8_t)__builtin_aarch64_qtbx3v8qi ((int8x8_t)__r, __o,
-						 (int8x8_t)__idx);
+  return __builtin_aarch64_qtbx3v8qi_uuuu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline poly8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx3_p8 (poly8x8_t __r, poly8x16x3_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x8_t)__builtin_aarch64_qtbx3v8qi ((int8x8_t)__r, __o,
-						 (int8x8_t)__idx);
+  return __builtin_aarch64_qtbx3v8qi_pppu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx3q_s8 (int8x16_t __r, int8x16x3_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbx3v16qi (__r, __o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbx3v16qi_sssu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx3q_u8 (uint8x16_t __r, uint8x16x3_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x16_t)__builtin_aarch64_qtbx3v16qi ((int8x16_t)__r, __o,
-						   (int8x16_t)__idx);
+  return __builtin_aarch64_qtbx3v16qi_uuuu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline poly8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx3q_p8 (poly8x16_t __r, poly8x16x3_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x16_t)__builtin_aarch64_qtbx3v16qi ((int8x16_t)__r, __o,
-						   (int8x16_t)__idx);
+  return __builtin_aarch64_qtbx3v16qi_pppu (__r, __tab, __idx);
 }
 
 /* vqtbx4 */
@@ -25738,58 +22349,42 @@ __extension__ extern __inline int8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx4_s8 (int8x8_t __r, int8x16x4_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbx4v8qi (__r, __o, (int8x8_t)__idx);
+  return __builtin_aarch64_qtbx4v8qi_sssu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx4_u8 (uint8x8_t __r, uint8x16x4_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x8_t)__builtin_aarch64_qtbx4v8qi ((int8x8_t)__r, __o,
-						 (int8x8_t)__idx);
+  return __builtin_aarch64_qtbx4v8qi_uuuu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline poly8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx4_p8 (poly8x8_t __r, poly8x16x4_t __tab, uint8x8_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x8_t)__builtin_aarch64_qtbx4v8qi ((int8x8_t)__r, __o,
-						 (int8x8_t)__idx);
+  return __builtin_aarch64_qtbx4v8qi_pppu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline int8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx4q_s8 (int8x16_t __r, int8x16x4_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return __builtin_aarch64_qtbx4v16qi (__r, __o, (int8x16_t)__idx);
+  return __builtin_aarch64_qtbx4v16qi_sssu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx4q_u8 (uint8x16_t __r, uint8x16x4_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (uint8x16_t)__builtin_aarch64_qtbx4v16qi ((int8x16_t)__r, __o,
-						   (int8x16_t)__idx);
+  return __builtin_aarch64_qtbx4v16qi_uuuu (__r, __tab, __idx);
 }
 
 __extension__ extern __inline poly8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vqtbx4q_p8 (poly8x16_t __r, poly8x16x4_t __tab, uint8x16_t __idx)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__tab, sizeof (__tab));
-  return (poly8x16_t)__builtin_aarch64_qtbx4v16qi ((int8x16_t)__r, __o,
-						   (int8x16_t)__idx);
+  return __builtin_aarch64_qtbx4v16qi_pppu (__r, __tab, __idx);
 }
 
 /* vrbit  */
@@ -27390,21 +23985,21 @@ __extension__ extern __inline uint8x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshr_n_u8 (uint8x8_t __a, const int __b)
 {
-  return (uint8x8_t) __builtin_aarch64_lshrv8qi ((int8x8_t) __a, __b);
+  return __builtin_aarch64_lshrv8qi_uus (__a, __b);
 }
 
 __extension__ extern __inline uint16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshr_n_u16 (uint16x4_t __a, const int __b)
 {
-  return (uint16x4_t) __builtin_aarch64_lshrv4hi ((int16x4_t) __a, __b);
+  return __builtin_aarch64_lshrv4hi_uus (__a, __b);
 }
 
 __extension__ extern __inline uint32x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshr_n_u32 (uint32x2_t __a, const int __b)
 {
-  return (uint32x2_t) __builtin_aarch64_lshrv2si ((int32x2_t) __a, __b);
+  return __builtin_aarch64_lshrv2si_uus (__a, __b);
 }
 
 __extension__ extern __inline uint64x1_t
@@ -27446,28 +24041,28 @@ __extension__ extern __inline uint8x16_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshrq_n_u8 (uint8x16_t __a, const int __b)
 {
-  return (uint8x16_t) __builtin_aarch64_lshrv16qi ((int8x16_t) __a, __b);
+  return __builtin_aarch64_lshrv16qi_uus (__a, __b);
 }
 
 __extension__ extern __inline uint16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshrq_n_u16 (uint16x8_t __a, const int __b)
 {
-  return (uint16x8_t) __builtin_aarch64_lshrv8hi ((int16x8_t) __a, __b);
+  return __builtin_aarch64_lshrv8hi_uus (__a, __b);
 }
 
 __extension__ extern __inline uint32x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshrq_n_u32 (uint32x4_t __a, const int __b)
 {
-  return (uint32x4_t) __builtin_aarch64_lshrv4si ((int32x4_t) __a, __b);
+  return __builtin_aarch64_lshrv4si_uus (__a, __b);
 }
 
 __extension__ extern __inline uint64x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vshrq_n_u64 (uint64x2_t __a, const int __b)
 {
-  return (uint64x2_t) __builtin_aarch64_lshrv2di ((int64x2_t) __a, __b);
+  return __builtin_aarch64_lshrv2di_uus (__a, __b);
 }
 
 __extension__ extern __inline int64_t
@@ -28024,16 +24619,14 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_p8 (poly8_t *__a, poly8x8_t __b)
 {
-  __builtin_aarch64_st1v8qi ((__builtin_aarch64_simd_qi *) __a,
-			     (int8x8_t) __b);
+  __builtin_aarch64_st1v8qi_sp ((__builtin_aarch64_simd_qi *) __a, __b);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_p16 (poly16_t *__a, poly16x4_t __b)
 {
-  __builtin_aarch64_st1v4hi ((__builtin_aarch64_simd_hi *) __a,
-			     (int16x4_t) __b);
+  __builtin_aarch64_st1v4hi_sp ((__builtin_aarch64_simd_hi *) __a, __b);
 }
 
 __extension__ extern __inline void
@@ -28075,24 +24668,21 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u8 (uint8_t *__a, uint8x8_t __b)
 {
-  __builtin_aarch64_st1v8qi ((__builtin_aarch64_simd_qi *) __a,
-			     (int8x8_t) __b);
+  __builtin_aarch64_st1v8qi_su ((__builtin_aarch64_simd_qi *) __a, __b);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u16 (uint16_t *__a, uint16x4_t __b)
 {
-  __builtin_aarch64_st1v4hi ((__builtin_aarch64_simd_hi *) __a,
-			     (int16x4_t) __b);
+  __builtin_aarch64_st1v4hi_su ((__builtin_aarch64_simd_hi *) __a, __b);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u32 (uint32_t *__a, uint32x2_t __b)
 {
-  __builtin_aarch64_st1v2si ((__builtin_aarch64_simd_si *) __a,
-			     (int32x2_t) __b);
+  __builtin_aarch64_st1v2si_su ((__builtin_aarch64_simd_si *) __a, __b);
 }
 
 __extension__ extern __inline void
@@ -28129,24 +24719,21 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p8 (poly8_t *__a, poly8x16_t __b)
 {
-  __builtin_aarch64_st1v16qi ((__builtin_aarch64_simd_qi *) __a,
-			      (int8x16_t) __b);
+  __builtin_aarch64_st1v16qi_sp ((__builtin_aarch64_simd_qi *) __a, __b);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p16 (poly16_t *__a, poly16x8_t __b)
 {
-  __builtin_aarch64_st1v8hi ((__builtin_aarch64_simd_hi *) __a,
-			     (int16x8_t) __b);
+  __builtin_aarch64_st1v8hi_sp ((__builtin_aarch64_simd_hi *) __a, __b);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p64 (poly64_t *__a, poly64x2_t __b)
 {
-  __builtin_aarch64_st1v2di_sp ((__builtin_aarch64_simd_di *) __a,
-				(poly64x2_t) __b);
+  __builtin_aarch64_st1v2di_sp ((__builtin_aarch64_simd_di *) __a, __b);
 }
 
 __extension__ extern __inline void
@@ -28181,32 +24768,28 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u8 (uint8_t *__a, uint8x16_t __b)
 {
-  __builtin_aarch64_st1v16qi ((__builtin_aarch64_simd_qi *) __a,
-			      (int8x16_t) __b);
+  __builtin_aarch64_st1v16qi_su ((__builtin_aarch64_simd_qi *) __a, __b);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u16 (uint16_t *__a, uint16x8_t __b)
 {
-  __builtin_aarch64_st1v8hi ((__builtin_aarch64_simd_hi *) __a,
-			     (int16x8_t) __b);
+  __builtin_aarch64_st1v8hi_su ((__builtin_aarch64_simd_hi *) __a, __b);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u32 (uint32_t *__a, uint32x4_t __b)
 {
-  __builtin_aarch64_st1v4si ((__builtin_aarch64_simd_si *) __a,
-			     (int32x4_t) __b);
+  __builtin_aarch64_st1v4si_su ((__builtin_aarch64_simd_si *) __a, __b);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u64 (uint64_t *__a, uint64x2_t __b)
 {
-  __builtin_aarch64_st1v2di ((__builtin_aarch64_simd_di *) __a,
-			     (int64x2_t) __b);
+  __builtin_aarch64_st1v2di_su ((__builtin_aarch64_simd_di *) __a, __b);
 }
 
 /* vst1_lane */
@@ -28415,310 +24998,196 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s64_x2 (int64_t * __a, int64x1x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  int64x2x2_t __temp;
-  __temp.val[0]
-    = vcombine_s64 (__val.val[0], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[1]
-    = vcombine_s64 (__val.val[1], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x2di ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u64_x2 (uint64_t * __a, uint64x1x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint64x2x2_t __temp;
-  __temp.val[0]
-    = vcombine_u64 (__val.val[0], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1]
-    = vcombine_u64 (__val.val[1], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x2di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_f64_x2 (float64_t * __a, float64x1x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  float64x2x2_t __temp;
-  __temp.val[0]
-    = vcombine_f64 (__val.val[0], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1]
-    = vcombine_f64 (__val.val[1], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st1x2df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s8_x2 (int8_t * __a, int8x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  int8x16x2_t __temp;
-  __temp.val[0]
-    = vcombine_s8 (__val.val[0], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[1]
-    = vcombine_s8 (__val.val[1], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x2v8qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_p8_x2 (poly8_t * __a, poly8x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly8x16x2_t __temp;
-  __temp.val[0]
-    = vcombine_p8 (__val.val[0], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1]
-    = vcombine_p8 (__val.val[1], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x2v8qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s16_x2 (int16_t * __a, int16x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  int16x8x2_t __temp;
-  __temp.val[0]
-    = vcombine_s16 (__val.val[0], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[1]
-    = vcombine_s16 (__val.val[1], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x2v4hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_p16_x2 (poly16_t * __a, poly16x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly16x8x2_t __temp;
-  __temp.val[0]
-    = vcombine_p16 (__val.val[0], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1]
-    = vcombine_p16 (__val.val[1], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x2v4hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s32_x2 (int32_t * __a, int32x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  int32x4x2_t __temp;
-  __temp.val[0]
-    = vcombine_s32 (__val.val[0], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[1]
-    = vcombine_s32 (__val.val[1], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x2v2si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u8_x2 (uint8_t * __a, uint8x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint8x16x2_t __temp;
-  __temp.val[0] = vcombine_u8 (__val.val[0], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u8 (__val.val[1], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x2v8qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u16_x2 (uint16_t * __a, uint16x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint16x8x2_t __temp;
-  __temp.val[0] = vcombine_u16 (__val.val[0], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u16 (__val.val[1], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x2v4hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u32_x2 (uint32_t * __a, uint32x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint32x4x2_t __temp;
-  __temp.val[0] = vcombine_u32 (__val.val[0], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u32 (__val.val[1], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x2v2si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_f16_x2 (float16_t * __a, float16x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  float16x8x2_t __temp;
-  __temp.val[0] = vcombine_f16 (__val.val[0], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f16 (__val.val[1], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2v4hf (__a, __o);
+  __builtin_aarch64_st1x2v4hf (__a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_f32_x2 (float32_t * __a, float32x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  float32x4x2_t __temp;
-  __temp.val[0] = vcombine_f32 (__val.val[0], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f32 (__val.val[1], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2v2sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st1x2v2sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_p64_x2 (poly64_t * __a, poly64x1x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly64x2x2_t __temp;
-  __temp.val[0] = vcombine_p64 (__val.val[0], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p64 (__val.val[1], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x2di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s8_x2 (int8_t * __a, int8x16x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x2v16qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p8_x2 (poly8_t * __a, poly8x16x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x2v16qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s16_x2 (int16_t * __a, int16x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x2v8hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p16_x2 (poly16_t * __a, poly16x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x2v8hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s32_x2 (int32_t * __a, int32x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x2v4si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s64_x2 (int64_t * __a, int64x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x2v2di ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u8_x2 (uint8_t * __a, uint8x16x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x2v16qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u16_x2 (uint16_t * __a, uint16x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x2v8hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u32_x2 (uint32_t * __a, uint32x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x2v4si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u64_x2 (uint64_t * __a, uint64x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x2v2di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_f16_x2 (float16_t * __a, float16x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v8hf (__a, __o);
+  __builtin_aarch64_st1x2v8hf (__a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_f32_x2 (float32_t * __a, float32x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v4sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st1x2v4sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_f64_x2 (float64_t * __a, float64x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v2df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st1x2v2df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p64_x2 (poly64_t * __a, poly64x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x2v2di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 /* vst1x3 */
@@ -28727,308 +25196,197 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s64_x3 (int64_t * __a, int64x1x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  int64x2x3_t __temp;
-  __temp.val[0] = vcombine_s64 (__val.val[0], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s64 (__val.val[1], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s64 (__val.val[2], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x3di ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u64_x3 (uint64_t * __a, uint64x1x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint64x2x3_t __temp;
-  __temp.val[0] = vcombine_u64 (__val.val[0], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u64 (__val.val[1], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u64 (__val.val[2], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x3di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_f64_x3 (float64_t * __a, float64x1x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  float64x2x3_t __temp;
-  __temp.val[0] = vcombine_f64 (__val.val[0], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f64 (__val.val[1], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f64 (__val.val[2], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st1x3df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s8_x3 (int8_t * __a, int8x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  int8x16x3_t __temp;
-  __temp.val[0] = vcombine_s8 (__val.val[0], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s8 (__val.val[1], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s8 (__val.val[2], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x3v8qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_p8_x3 (poly8_t * __a, poly8x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly8x16x3_t __temp;
-  __temp.val[0] = vcombine_p8 (__val.val[0], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p8 (__val.val[1], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p8 (__val.val[2], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x3v8qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s16_x3 (int16_t * __a, int16x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  int16x8x3_t __temp;
-  __temp.val[0] = vcombine_s16 (__val.val[0], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s16 (__val.val[1], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s16 (__val.val[2], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x3v4hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_p16_x3 (poly16_t * __a, poly16x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly16x8x3_t __temp;
-  __temp.val[0] = vcombine_p16 (__val.val[0], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p16 (__val.val[1], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p16 (__val.val[2], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x3v4hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s32_x3 (int32_t * __a, int32x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  int32x4x3_t __temp;
-  __temp.val[0] = vcombine_s32 (__val.val[0], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s32 (__val.val[1], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s32 (__val.val[2], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x3v2si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u8_x3 (uint8_t * __a, uint8x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint8x16x3_t __temp;
-  __temp.val[0] = vcombine_u8 (__val.val[0], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u8 (__val.val[1], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u8 (__val.val[2], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x3v8qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u16_x3 (uint16_t * __a, uint16x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint16x8x3_t __temp;
-  __temp.val[0] = vcombine_u16 (__val.val[0], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u16 (__val.val[1], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u16 (__val.val[2], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x3v4hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u32_x3 (uint32_t * __a, uint32x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint32x4x3_t __temp;
-  __temp.val[0] = vcombine_u32 (__val.val[0], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u32 (__val.val[1], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u32 (__val.val[2], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x3v2si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_f16_x3 (float16_t * __a, float16x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  float16x8x3_t __temp;
-  __temp.val[0] = vcombine_f16 (__val.val[0], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f16 (__val.val[1], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f16 (__val.val[2], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3v4hf ((__builtin_aarch64_simd_hf *) __a, __o);
+  __builtin_aarch64_st1x3v4hf ((__builtin_aarch64_simd_hf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_f32_x3 (float32_t * __a, float32x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  float32x4x3_t __temp;
-  __temp.val[0] = vcombine_f32 (__val.val[0], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f32 (__val.val[1], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f32 (__val.val[2], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3v2sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st1x3v2sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_p64_x3 (poly64_t * __a, poly64x1x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly64x2x3_t __temp;
-  __temp.val[0] = vcombine_p64 (__val.val[0], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p64 (__val.val[1], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p64 (__val.val[2], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x3di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s8_x3 (int8_t * __a, int8x16x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x3v16qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p8_x3 (poly8_t * __a, poly8x16x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x3v16qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s16_x3 (int16_t * __a, int16x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x3v8hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p16_x3 (poly16_t * __a, poly16x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x3v8hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s32_x3 (int32_t * __a, int32x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x3v4si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s64_x3 (int64_t * __a, int64x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x3v2di ((__builtin_aarch64_simd_di *) __a,
+			       (int64x2x3_t) __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u8_x3 (uint8_t * __a, uint8x16x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x3v16qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u16_x3 (uint16_t * __a, uint16x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x3v8hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u32_x3 (uint32_t * __a, uint32x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x3v4si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u64_x3 (uint64_t * __a, uint64x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x3v2di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_f16_x3 (float16_t * __a, float16x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v8hf ((__builtin_aarch64_simd_hf *) __a, __o);
+  __builtin_aarch64_st1x3v8hf ((__builtin_aarch64_simd_hf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_f32_x3 (float32_t * __a, float32x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v4sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st1x3v4sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_f64_x3 (float64_t * __a, float64x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v2df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st1x3v2df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p64_x3 (poly64_t * __a, poly64x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x3v2di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 /* vst1(q)_x4.  */
@@ -29037,322 +25395,196 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s8_x4 (int8_t * __a, int8x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  int8x16x4_t __temp;
-  __temp.val[0] = vcombine_s8 (__val.val[0], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s8 (__val.val[1], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s8 (__val.val[2], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[3] = vcombine_s8 (__val.val[3], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x4v8qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s8_x4 (int8_t * __a, int8x16x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x4v16qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s16_x4 (int16_t * __a, int16x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  int16x8x4_t __temp;
-  __temp.val[0] = vcombine_s16 (__val.val[0], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s16 (__val.val[1], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s16 (__val.val[2], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[3] = vcombine_s16 (__val.val[3], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x4v4hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s16_x4 (int16_t * __a, int16x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x4v8hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s32_x4 (int32_t * __a, int32x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  int32x4x4_t __temp;
-  __temp.val[0] = vcombine_s32 (__val.val[0], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s32 (__val.val[1], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s32 (__val.val[2], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[3] = vcombine_s32 (__val.val[3], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x4v2si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s32_x4 (int32_t * __a, int32x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x4v4si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u8_x4 (uint8_t * __a, uint8x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint8x16x4_t __temp;
-  __temp.val[0] = vcombine_u8 (__val.val[0], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u8 (__val.val[1], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u8 (__val.val[2], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u8 (__val.val[3], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x4v8qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u8_x4 (uint8_t * __a, uint8x16x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x4v16qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u16_x4 (uint16_t * __a, uint16x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint16x8x4_t __temp;
-  __temp.val[0] = vcombine_u16 (__val.val[0], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u16 (__val.val[1], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u16 (__val.val[2], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u16 (__val.val[3], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x4v4hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u16_x4 (uint16_t * __a, uint16x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x4v8hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u32_x4 (uint32_t * __a, uint32x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint32x4x4_t __temp;
-  __temp.val[0] = vcombine_u32 (__val.val[0], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u32 (__val.val[1], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u32 (__val.val[2], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u32 (__val.val[3], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x4v2si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u32_x4 (uint32_t * __a, uint32x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st1x4v4si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_f16_x4 (float16_t * __a, float16x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  float16x8x4_t __temp;
-  __temp.val[0] = vcombine_f16 (__val.val[0], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f16 (__val.val[1], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f16 (__val.val[2], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_f16 (__val.val[3], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4v4hf ((__builtin_aarch64_simd_hf *) __a, __o);
+  __builtin_aarch64_st1x4v4hf ((__builtin_aarch64_simd_hf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_f16_x4 (float16_t * __a, float16x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v8hf ((__builtin_aarch64_simd_hf *) __a, __o);
+  __builtin_aarch64_st1x4v8hf ((__builtin_aarch64_simd_hf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_f32_x4 (float32_t * __a, float32x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  float32x4x4_t __temp;
-  __temp.val[0] = vcombine_f32 (__val.val[0], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f32 (__val.val[1], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f32 (__val.val[2], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_f32 (__val.val[3], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4v2sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st1x4v2sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_f32_x4 (float32_t * __a, float32x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v4sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st1x4v4sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_p8_x4 (poly8_t * __a, poly8x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly8x16x4_t __temp;
-  __temp.val[0] = vcombine_p8 (__val.val[0], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p8 (__val.val[1], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p8 (__val.val[2], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_p8 (__val.val[3], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x4v8qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p8_x4 (poly8_t * __a, poly8x16x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st1x4v16qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_p16_x4 (poly16_t * __a, poly16x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly16x8x4_t __temp;
-  __temp.val[0] = vcombine_p16 (__val.val[0], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p16 (__val.val[1], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p16 (__val.val[2], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_p16 (__val.val[3], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x4v4hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p16_x4 (poly16_t * __a, poly16x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st1x4v8hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_s64_x4 (int64_t * __a, int64x1x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  int64x2x4_t __temp;
-  __temp.val[0] = vcombine_s64 (__val.val[0], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s64 (__val.val[1], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s64 (__val.val[2], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[3] = vcombine_s64 (__val.val[3], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x4di ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_u64_x4 (uint64_t * __a, uint64x1x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint64x2x4_t __temp;
-  __temp.val[0] = vcombine_u64 (__val.val[0], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u64 (__val.val[1], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u64 (__val.val[2], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u64 (__val.val[3], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x4di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_p64_x4 (poly64_t * __a, poly64x1x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly64x2x4_t __temp;
-  __temp.val[0] = vcombine_p64 (__val.val[0], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p64 (__val.val[1], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p64 (__val.val[2], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_p64 (__val.val[3], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x4di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_s64_x4 (int64_t * __a, int64x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x4v2di ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_u64_x4 (uint64_t * __a, uint64x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x4v2di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_p64_x4 (poly64_t * __a, poly64x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st1x4v2di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_f64_x4 (float64_t * __a, float64x1x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  float64x2x4_t __temp;
-  __temp.val[0] = vcombine_f64 (__val.val[0], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f64 (__val.val[1], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f64 (__val.val[2], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_f64 (__val.val[3], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st1x4df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_f64_x4 (float64_t * __a, float64x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v2df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st1x4v2df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 /* vstn */
@@ -29361,924 +25593,588 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_s64 (int64_t * __a, int64x1x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  int64x2x2_t __temp;
-  __temp.val[0] = vcombine_s64 (__val.val[0], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s64 (__val.val[1], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st2di ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_u64 (uint64_t * __a, uint64x1x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint64x2x2_t __temp;
-  __temp.val[0] = vcombine_u64 (__val.val[0], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u64 (__val.val[1], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st2di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_f64 (float64_t * __a, float64x1x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  float64x2x2_t __temp;
-  __temp.val[0] = vcombine_f64 (__val.val[0], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f64 (__val.val[1], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st2df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_s8 (int8_t * __a, int8x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  int8x16x2_t __temp;
-  __temp.val[0] = vcombine_s8 (__val.val[0], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s8 (__val.val[1], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st2v8qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_p8 (poly8_t * __a, poly8x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly8x16x2_t __temp;
-  __temp.val[0] = vcombine_p8 (__val.val[0], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p8 (__val.val[1], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st2v8qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_s16 (int16_t * __a, int16x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  int16x8x2_t __temp;
-  __temp.val[0] = vcombine_s16 (__val.val[0], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s16 (__val.val[1], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st2v4hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_p16 (poly16_t * __a, poly16x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly16x8x2_t __temp;
-  __temp.val[0] = vcombine_p16 (__val.val[0], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p16 (__val.val[1], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st2v4hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_s32 (int32_t * __a, int32x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  int32x4x2_t __temp;
-  __temp.val[0] = vcombine_s32 (__val.val[0], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s32 (__val.val[1], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st2v2si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_u8 (uint8_t * __a, uint8x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint8x16x2_t __temp;
-  __temp.val[0] = vcombine_u8 (__val.val[0], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u8 (__val.val[1], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st2v8qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_u16 (uint16_t * __a, uint16x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint16x8x2_t __temp;
-  __temp.val[0] = vcombine_u16 (__val.val[0], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u16 (__val.val[1], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st2v4hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_u32 (uint32_t * __a, uint32x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  uint32x4x2_t __temp;
-  __temp.val[0] = vcombine_u32 (__val.val[0], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u32 (__val.val[1], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st2v2si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_f16 (float16_t * __a, float16x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  float16x8x2_t __temp;
-  __temp.val[0] = vcombine_f16 (__val.val[0], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f16 (__val.val[1], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2v4hf (__a, __o);
+  __builtin_aarch64_st2v4hf ((__builtin_aarch64_simd_hf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_f32 (float32_t * __a, float32x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  float32x4x2_t __temp;
-  __temp.val[0] = vcombine_f32 (__val.val[0], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f32 (__val.val[1], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2v2sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st2v2sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_p64 (poly64_t * __a, poly64x1x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  poly64x2x2_t __temp;
-  __temp.val[0] = vcombine_p64 (__val.val[0], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p64 (__val.val[1], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st2di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_s8 (int8_t * __a, int8x16x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st2v16qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_p8 (poly8_t * __a, poly8x16x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st2v16qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_s16 (int16_t * __a, int16x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st2v8hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_p16 (poly16_t * __a, poly16x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st2v8hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_s32 (int32_t * __a, int32x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st2v4si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_s64 (int64_t * __a, int64x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st2v2di ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_u8 (uint8_t * __a, uint8x16x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st2v16qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_u16 (uint16_t * __a, uint16x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st2v8hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_u32 (uint32_t * __a, uint32x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st2v4si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_u64 (uint64_t * __a, uint64x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st2v2di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_f16 (float16_t * __a, float16x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v8hf (__a, __o);
+  __builtin_aarch64_st2v8hf ((__builtin_aarch64_simd_hf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_f32 (float32_t * __a, float32x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v4sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st2v4sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_f64 (float64_t * __a, float64x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v2df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st2v2df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_p64 (poly64_t * __a, poly64x2x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st2v2di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_s64 (int64_t * __a, int64x1x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  int64x2x3_t __temp;
-  __temp.val[0] = vcombine_s64 (__val.val[0], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s64 (__val.val[1], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s64 (__val.val[2], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st3di ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_u64 (uint64_t * __a, uint64x1x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint64x2x3_t __temp;
-  __temp.val[0] = vcombine_u64 (__val.val[0], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u64 (__val.val[1], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u64 (__val.val[2], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st3di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_f64 (float64_t * __a, float64x1x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  float64x2x3_t __temp;
-  __temp.val[0] = vcombine_f64 (__val.val[0], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f64 (__val.val[1], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f64 (__val.val[2], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st3df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_s8 (int8_t * __a, int8x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  int8x16x3_t __temp;
-  __temp.val[0] = vcombine_s8 (__val.val[0], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s8 (__val.val[1], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s8 (__val.val[2], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st3v8qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_p8 (poly8_t * __a, poly8x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly8x16x3_t __temp;
-  __temp.val[0] = vcombine_p8 (__val.val[0], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p8 (__val.val[1], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p8 (__val.val[2], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st3v8qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_s16 (int16_t * __a, int16x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  int16x8x3_t __temp;
-  __temp.val[0] = vcombine_s16 (__val.val[0], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s16 (__val.val[1], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s16 (__val.val[2], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st3v4hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_p16 (poly16_t * __a, poly16x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly16x8x3_t __temp;
-  __temp.val[0] = vcombine_p16 (__val.val[0], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p16 (__val.val[1], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p16 (__val.val[2], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st3v4hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_s32 (int32_t * __a, int32x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  int32x4x3_t __temp;
-  __temp.val[0] = vcombine_s32 (__val.val[0], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s32 (__val.val[1], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s32 (__val.val[2], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st3v2si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_u8 (uint8_t * __a, uint8x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint8x16x3_t __temp;
-  __temp.val[0] = vcombine_u8 (__val.val[0], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u8 (__val.val[1], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u8 (__val.val[2], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st3v8qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_u16 (uint16_t * __a, uint16x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint16x8x3_t __temp;
-  __temp.val[0] = vcombine_u16 (__val.val[0], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u16 (__val.val[1], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u16 (__val.val[2], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st3v4hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_u32 (uint32_t * __a, uint32x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  uint32x4x3_t __temp;
-  __temp.val[0] = vcombine_u32 (__val.val[0], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u32 (__val.val[1], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u32 (__val.val[2], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st3v2si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_f16 (float16_t * __a, float16x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  float16x8x3_t __temp;
-  __temp.val[0] = vcombine_f16 (__val.val[0], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f16 (__val.val[1], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f16 (__val.val[2], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3v4hf ((__builtin_aarch64_simd_hf *) __a, __o);
+  __builtin_aarch64_st3v4hf ((__builtin_aarch64_simd_hf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_f32 (float32_t * __a, float32x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  float32x4x3_t __temp;
-  __temp.val[0] = vcombine_f32 (__val.val[0], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f32 (__val.val[1], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f32 (__val.val[2], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3v2sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st3v2sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_p64 (poly64_t * __a, poly64x1x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  poly64x2x3_t __temp;
-  __temp.val[0] = vcombine_p64 (__val.val[0], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p64 (__val.val[1], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p64 (__val.val[2], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st3di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_s8 (int8_t * __a, int8x16x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st3v16qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_p8 (poly8_t * __a, poly8x16x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st3v16qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_s16 (int16_t * __a, int16x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st3v8hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_p16 (poly16_t * __a, poly16x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st3v8hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_s32 (int32_t * __a, int32x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st3v4si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_s64 (int64_t * __a, int64x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st3v2di ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_u8 (uint8_t * __a, uint8x16x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st3v16qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_u16 (uint16_t * __a, uint16x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st3v8hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_u32 (uint32_t * __a, uint32x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st3v4si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_u64 (uint64_t * __a, uint64x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st3v2di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_f16 (float16_t * __a, float16x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v8hf ((__builtin_aarch64_simd_hf *) __a, __o);
+  __builtin_aarch64_st3v8hf ((__builtin_aarch64_simd_hf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_f32 (float32_t * __a, float32x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v4sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st3v4sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_f64 (float64_t * __a, float64x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v2df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st3v2df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_p64 (poly64_t * __a, poly64x2x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st3v2di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_s64 (int64_t * __a, int64x1x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  int64x2x4_t __temp;
-  __temp.val[0] = vcombine_s64 (__val.val[0], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s64 (__val.val[1], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s64 (__val.val[2], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __temp.val[3] = vcombine_s64 (__val.val[3], vcreate_s64 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st4di ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_u64 (uint64_t * __a, uint64x1x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint64x2x4_t __temp;
-  __temp.val[0] = vcombine_u64 (__val.val[0], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u64 (__val.val[1], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u64 (__val.val[2], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u64 (__val.val[3], vcreate_u64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st4di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_f64 (float64_t * __a, float64x1x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  float64x2x4_t __temp;
-  __temp.val[0] = vcombine_f64 (__val.val[0], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f64 (__val.val[1], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f64 (__val.val[2], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_f64 (__val.val[3], vcreate_f64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st4df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_s8 (int8_t * __a, int8x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  int8x16x4_t __temp;
-  __temp.val[0] = vcombine_s8 (__val.val[0], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s8 (__val.val[1], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s8 (__val.val[2], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __temp.val[3] = vcombine_s8 (__val.val[3], vcreate_s8 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st4v8qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_p8 (poly8_t * __a, poly8x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly8x16x4_t __temp;
-  __temp.val[0] = vcombine_p8 (__val.val[0], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p8 (__val.val[1], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p8 (__val.val[2], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_p8 (__val.val[3], vcreate_p8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st4v8qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_s16 (int16_t * __a, int16x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  int16x8x4_t __temp;
-  __temp.val[0] = vcombine_s16 (__val.val[0], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s16 (__val.val[1], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s16 (__val.val[2], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __temp.val[3] = vcombine_s16 (__val.val[3], vcreate_s16 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st4v4hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_p16 (poly16_t * __a, poly16x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly16x8x4_t __temp;
-  __temp.val[0] = vcombine_p16 (__val.val[0], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p16 (__val.val[1], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p16 (__val.val[2], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_p16 (__val.val[3], vcreate_p16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st4v4hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_s32 (int32_t * __a, int32x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  int32x4x4_t __temp;
-  __temp.val[0] = vcombine_s32 (__val.val[0], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[1] = vcombine_s32 (__val.val[1], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[2] = vcombine_s32 (__val.val[2], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __temp.val[3] = vcombine_s32 (__val.val[3], vcreate_s32 (__AARCH64_INT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st4v2si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_u8 (uint8_t * __a, uint8x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint8x16x4_t __temp;
-  __temp.val[0] = vcombine_u8 (__val.val[0], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u8 (__val.val[1], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u8 (__val.val[2], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u8 (__val.val[3], vcreate_u8 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4v8qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st4v8qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_u16 (uint16_t * __a, uint16x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint16x8x4_t __temp;
-  __temp.val[0] = vcombine_u16 (__val.val[0], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u16 (__val.val[1], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u16 (__val.val[2], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u16 (__val.val[3], vcreate_u16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4v4hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st4v4hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_u32 (uint32_t * __a, uint32x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  uint32x4x4_t __temp;
-  __temp.val[0] = vcombine_u32 (__val.val[0], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_u32 (__val.val[1], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_u32 (__val.val[2], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_u32 (__val.val[3], vcreate_u32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4v2si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st4v2si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_f16 (float16_t * __a, float16x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  float16x8x4_t __temp;
-  __temp.val[0] = vcombine_f16 (__val.val[0], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f16 (__val.val[1], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f16 (__val.val[2], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_f16 (__val.val[3], vcreate_f16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4v4hf ((__builtin_aarch64_simd_hf *) __a, __o);
+  __builtin_aarch64_st4v4hf ((__builtin_aarch64_simd_hf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_f32 (float32_t * __a, float32x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  float32x4x4_t __temp;
-  __temp.val[0] = vcombine_f32 (__val.val[0], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_f32 (__val.val[1], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_f32 (__val.val[2], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_f32 (__val.val[3], vcreate_f32 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4v2sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st4v2sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_p64 (poly64_t * __a, poly64x1x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  poly64x2x4_t __temp;
-  __temp.val[0] = vcombine_p64 (__val.val[0], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_p64 (__val.val[1], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_p64 (__val.val[2], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_p64 (__val.val[3], vcreate_p64 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st4di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_s8 (int8_t * __a, int8x16x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st4v16qi ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_p8 (poly8_t * __a, poly8x16x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st4v16qi_sp ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_s16 (int16_t * __a, int16x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st4v8hi ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_p16 (poly16_t * __a, poly16x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st4v8hi_sp ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_s32 (int32_t * __a, int32x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st4v4si ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_s64 (int64_t * __a, int64x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st4v2di ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_u8 (uint8_t * __a, uint8x16x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v16qi ((__builtin_aarch64_simd_qi *) __a, __o);
+  __builtin_aarch64_st4v16qi_su ((__builtin_aarch64_simd_qi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_u16 (uint16_t * __a, uint16x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v8hi ((__builtin_aarch64_simd_hi *) __a, __o);
+  __builtin_aarch64_st4v8hi_su ((__builtin_aarch64_simd_hi *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_u32 (uint32_t * __a, uint32x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v4si ((__builtin_aarch64_simd_si *) __a, __o);
+  __builtin_aarch64_st4v4si_su ((__builtin_aarch64_simd_si *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_u64 (uint64_t * __a, uint64x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st4v2di_su ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_f16 (float16_t * __a, float16x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v8hf ((__builtin_aarch64_simd_hf *) __a, __o);
+  __builtin_aarch64_st4v8hf ((__builtin_aarch64_simd_hf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_f32 (float32_t * __a, float32x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v4sf ((__builtin_aarch64_simd_sf *) __a, __o);
+  __builtin_aarch64_st4v4sf ((__builtin_aarch64_simd_sf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_f64 (float64_t * __a, float64x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v2df ((__builtin_aarch64_simd_df *) __a, __o);
+  __builtin_aarch64_st4v2df ((__builtin_aarch64_simd_df *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_p64 (poly64_t * __a, poly64x2x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v2di ((__builtin_aarch64_simd_di *) __a, __o);
+  __builtin_aarch64_st4v2di_sp ((__builtin_aarch64_simd_di *) __a, __val);
 }
 
 __extension__ extern __inline void
@@ -30377,11 +26273,9 @@ __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vtbx4_s8 (int8x8_t __r, int8x8x4_t __tab, int8x8_t __idx)
 {
   int8x16x2_t __temp;
-  __builtin_aarch64_simd_oi __o;
   __temp.val[0] = vcombine_s8 (__tab.val[0], __tab.val[1]);
   __temp.val[1] = vcombine_s8 (__tab.val[2], __tab.val[3]);
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  return __builtin_aarch64_qtbx2v8qi (__r, __o, __idx);
+  return __builtin_aarch64_qtbx2v8qi (__r, __temp, __idx);
 }
 
 __extension__ extern __inline uint8x8_t
@@ -30389,12 +26283,9 @@ __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vtbx4_u8 (uint8x8_t __r, uint8x8x4_t __tab, uint8x8_t __idx)
 {
   uint8x16x2_t __temp;
-  __builtin_aarch64_simd_oi __o;
   __temp.val[0] = vcombine_u8 (__tab.val[0], __tab.val[1]);
   __temp.val[1] = vcombine_u8 (__tab.val[2], __tab.val[3]);
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  return (uint8x8_t)__builtin_aarch64_qtbx2v8qi ((int8x8_t)__r, __o,
-						 (int8x8_t)__idx);
+  return __builtin_aarch64_qtbx2v8qi_uuuu (__r, __temp, __idx);
 }
 
 __extension__ extern __inline poly8x8_t
@@ -30402,12 +26293,9 @@ __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vtbx4_p8 (poly8x8_t __r, poly8x8x4_t __tab, uint8x8_t __idx)
 {
   poly8x16x2_t __temp;
-  __builtin_aarch64_simd_oi __o;
   __temp.val[0] = vcombine_p8 (__tab.val[0], __tab.val[1]);
   __temp.val[1] = vcombine_p8 (__tab.val[2], __tab.val[3]);
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  return (poly8x8_t)__builtin_aarch64_qtbx2v8qi ((int8x8_t)__r, __o,
-						 (int8x8_t)__idx);
+  return __builtin_aarch64_qtbx2v8qi_pppu (__r, __temp, __idx);
 }
 
 /* vtrn */
@@ -33088,14 +28976,14 @@ __extension__ extern __inline float16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmax_f16 (float16x4_t __a, float16x4_t __b)
 {
-  return __builtin_aarch64_smax_nanv4hf (__a, __b);
+  return __builtin_aarch64_fmax_nanv4hf (__a, __b);
 }
 
 __extension__ extern __inline float16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmaxq_f16 (float16x8_t __a, float16x8_t __b)
 {
-  return __builtin_aarch64_smax_nanv8hf (__a, __b);
+  return __builtin_aarch64_fmax_nanv8hf (__a, __b);
 }
 
 __extension__ extern __inline float16x4_t
@@ -33116,14 +29004,14 @@ __extension__ extern __inline float16x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vmin_f16 (float16x4_t __a, float16x4_t __b)
 {
-  return __builtin_aarch64_smin_nanv4hf (__a, __b);
+  return __builtin_aarch64_fmin_nanv4hf (__a, __b);
 }
 
 __extension__ extern __inline float16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vminq_f16 (float16x8_t __a, float16x8_t __b)
 {
-  return __builtin_aarch64_smin_nanv8hf (__a, __b);
+  return __builtin_aarch64_fmin_nanv8hf (__a, __b);
 }
 
 __extension__ extern __inline float16x4_t
@@ -34755,7 +30643,7 @@ __extension__ extern __inline bfloat16x8_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vcombine_bf16 (bfloat16x4_t __a, bfloat16x4_t __b)
 {
-  return (bfloat16x8_t)__builtin_aarch64_combinev4bf (__a, __b);
+  return __builtin_aarch64_combinev4bf (__a, __b);
 }
 
 /* vdup */
@@ -34836,69 +30724,42 @@ __extension__ extern __inline bfloat16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_bf16_x2 (const bfloat16_t *__a)
 {
-  bfloat16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v4bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x4_t) __builtin_aarch64_get_dregoiv4bf (__o, 0);
-  ret.val[1] = (bfloat16x4_t) __builtin_aarch64_get_dregoiv4bf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld1x2v4bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_bf16_x2 (const bfloat16_t *__a)
 {
-  bfloat16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld1x2v8bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x8_t) __builtin_aarch64_get_qregoiv8bf (__o, 0);
-  ret.val[1] = (bfloat16x8_t) __builtin_aarch64_get_qregoiv8bf (__o, 1);
-  return ret;
+  return  __builtin_aarch64_ld1x2v8bf (
+				(const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_bf16_x3 (const bfloat16_t *__a)
 {
-  bfloat16x4x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v4bf ((const __builtin_aarch64_simd_bf *) __a);
-  __i.val[0] = (bfloat16x4_t) __builtin_aarch64_get_dregciv4bf  (__o, 0);
-  __i.val[1] = (bfloat16x4_t) __builtin_aarch64_get_dregciv4bf  (__o, 1);
-  __i.val[2] = (bfloat16x4_t) __builtin_aarch64_get_dregciv4bf  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v4bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_bf16_x3 (const bfloat16_t *__a)
 {
-  bfloat16x8x3_t __i;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld1x3v8bf ((const __builtin_aarch64_simd_bf *) __a);
-  __i.val[0] = (bfloat16x8_t) __builtin_aarch64_get_qregciv8bf  (__o, 0);
-  __i.val[1] = (bfloat16x8_t) __builtin_aarch64_get_qregciv8bf  (__o, 1);
-  __i.val[2] = (bfloat16x8_t) __builtin_aarch64_get_qregciv8bf  (__o, 2);
-  return __i;
+  return __builtin_aarch64_ld1x3v8bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 __extension__ extern __inline bfloat16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1_bf16_x4 (const bfloat16_t *__a)
 {
-  union { bfloat16x4x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v4bf ((const __builtin_aarch64_simd_bf *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v4bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld1q_bf16_x4 (const bfloat16_t *__a)
 {
-  union { bfloat16x8x4_t __i; __builtin_aarch64_simd_xi __o; } __au;
-  __au.__o
-    = __builtin_aarch64_ld1x4v8bf ((const __builtin_aarch64_simd_bf *) __a);
-  return __au.__i;
+  return __builtin_aarch64_ld1x4v8bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x4_t
@@ -34933,156 +30794,84 @@ __extension__ extern __inline bfloat16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v4bf (__a);
-  ret.val[0] = (bfloat16x4_t) __builtin_aarch64_get_dregoiv4bf (__o, 0);
-  ret.val[1] = (bfloat16x4_t) __builtin_aarch64_get_dregoiv4bf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v4bf (__a);
 }
 
 __extension__ extern __inline bfloat16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2v8bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x8_t) __builtin_aarch64_get_qregoiv8bf (__o, 0);
-  ret.val[1] = (bfloat16x8_t) __builtin_aarch64_get_qregoiv8bf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2v8bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2_dup_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x4x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv4bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x4_t) __builtin_aarch64_get_dregoiv4bf (__o, 0);
-  ret.val[1] = (bfloat16x4_t) __builtin_aarch64_get_dregoiv4bf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv4bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld2q_dup_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x8x2_t ret;
-  __builtin_aarch64_simd_oi __o;
-  __o = __builtin_aarch64_ld2rv8bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x8_t) __builtin_aarch64_get_qregoiv8bf (__o, 0);
-  ret.val[1] = (bfloat16x8_t) __builtin_aarch64_get_qregoiv8bf (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2rv8bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v4bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x4_t) __builtin_aarch64_get_dregciv4bf (__o, 0);
-  ret.val[1] = (bfloat16x4_t) __builtin_aarch64_get_dregciv4bf (__o, 1);
-  ret.val[2] = (bfloat16x4_t) __builtin_aarch64_get_dregciv4bf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v4bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3v8bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x8_t) __builtin_aarch64_get_qregciv8bf (__o, 0);
-  ret.val[1] = (bfloat16x8_t) __builtin_aarch64_get_qregciv8bf (__o, 1);
-  ret.val[2] = (bfloat16x8_t) __builtin_aarch64_get_qregciv8bf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3v8bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3_dup_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x4x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv4bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x4_t) __builtin_aarch64_get_dregciv4bf (__o, 0);
-  ret.val[1] = (bfloat16x4_t) __builtin_aarch64_get_dregciv4bf (__o, 1);
-  ret.val[2] = (bfloat16x4_t) __builtin_aarch64_get_dregciv4bf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv4bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld3q_dup_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x8x3_t ret;
-  __builtin_aarch64_simd_ci __o;
-  __o = __builtin_aarch64_ld3rv8bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x8_t) __builtin_aarch64_get_qregciv8bf (__o, 0);
-  ret.val[1] = (bfloat16x8_t) __builtin_aarch64_get_qregciv8bf (__o, 1);
-  ret.val[2] = (bfloat16x8_t) __builtin_aarch64_get_qregciv8bf (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3rv8bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v4bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x4_t) __builtin_aarch64_get_dregxiv4bf (__o, 0);
-  ret.val[1] = (bfloat16x4_t) __builtin_aarch64_get_dregxiv4bf (__o, 1);
-  ret.val[2] = (bfloat16x4_t) __builtin_aarch64_get_dregxiv4bf (__o, 2);
-  ret.val[3] = (bfloat16x4_t) __builtin_aarch64_get_dregxiv4bf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v4bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4v8bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv8bf (__o, 0);
-  ret.val[1] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv8bf (__o, 1);
-  ret.val[2] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv8bf (__o, 2);
-  ret.val[3] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv8bf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4v8bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4_dup_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x4x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv4bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x4_t) __builtin_aarch64_get_dregxiv4bf (__o, 0);
-  ret.val[1] = (bfloat16x4_t) __builtin_aarch64_get_dregxiv4bf (__o, 1);
-  ret.val[2] = (bfloat16x4_t) __builtin_aarch64_get_dregxiv4bf (__o, 2);
-  ret.val[3] = (bfloat16x4_t) __builtin_aarch64_get_dregxiv4bf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv4bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 __extension__ extern __inline bfloat16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vld4q_dup_bf16 (const bfloat16_t * __a)
 {
-  bfloat16x8x4_t ret;
-  __builtin_aarch64_simd_xi __o;
-  __o = __builtin_aarch64_ld4rv8bf ((const __builtin_aarch64_simd_bf *) __a);
-  ret.val[0] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv8bf (__o, 0);
-  ret.val[1] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv8bf (__o, 1);
-  ret.val[2] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv8bf (__o, 2);
-  ret.val[3] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv8bf (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4rv8bf ((const __builtin_aarch64_simd_bf *) __a);
 }
 
 /* vst */
@@ -35098,66 +30887,42 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_bf16_x2 (bfloat16_t * __a, bfloat16x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  bfloat16x8x2_t __temp;
-  __temp.val[0] = vcombine_bf16 (__val.val[0], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_bf16 (__val.val[1], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x2v4bf (__a, __o);
+  __builtin_aarch64_st1x2v4bf (__a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_bf16_x2 (bfloat16_t * __a, bfloat16x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x2v8bf (__a, __o);
+  __builtin_aarch64_st1x2v8bf (__a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_bf16_x3 (bfloat16_t * __a, bfloat16x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  bfloat16x8x3_t __temp;
-  __temp.val[0] = vcombine_bf16 (__val.val[0], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_bf16 (__val.val[1], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_bf16 (__val.val[2], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x3v4bf ((__builtin_aarch64_simd_bf *) __a, __o);
+  __builtin_aarch64_st1x3v4bf ((__builtin_aarch64_simd_bf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_bf16_x3 (bfloat16_t * __a, bfloat16x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x3v8bf ((__builtin_aarch64_simd_bf *) __a, __o);
+  __builtin_aarch64_st1x3v8bf ((__builtin_aarch64_simd_bf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1_bf16_x4 (bfloat16_t * __a, bfloat16x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  bfloat16x8x4_t __temp;
-  __temp.val[0] = vcombine_bf16 (__val.val[0], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_bf16 (__val.val[1], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_bf16 (__val.val[2], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_bf16 (__val.val[3], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st1x4v4bf ((__builtin_aarch64_simd_bf *) __a, __o);
+  __builtin_aarch64_st1x4v4bf ((__builtin_aarch64_simd_bf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst1q_bf16_x4 (bfloat16_t * __a, bfloat16x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st1x4v8bf ((__builtin_aarch64_simd_bf *) __a, __o);
+  __builtin_aarch64_st1x4v8bf ((__builtin_aarch64_simd_bf *) __a, __val);
 }
 
 __extension__ extern __inline void
@@ -35185,66 +30950,42 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_bf16 (bfloat16_t * __a, bfloat16x4x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  bfloat16x8x2_t __temp;
-  __temp.val[0] = vcombine_bf16 (__val.val[0], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_bf16 (__val.val[1], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2v4bf (__a, __o);
+  __builtin_aarch64_st2v4bf (__a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_bf16 (bfloat16_t * __a, bfloat16x8x2_t __val)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2v8bf (__a, __o);
+  __builtin_aarch64_st2v8bf (__a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_bf16 (bfloat16_t * __a, bfloat16x4x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  bfloat16x8x3_t __temp;
-  __temp.val[0] = vcombine_bf16 (__val.val[0], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_bf16 (__val.val[1], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_bf16 (__val.val[2], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3v4bf ((__builtin_aarch64_simd_bf *) __a, __o);
+  __builtin_aarch64_st3v4bf ((__builtin_aarch64_simd_bf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_bf16 (bfloat16_t * __a, bfloat16x8x3_t __val)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3v8bf ((__builtin_aarch64_simd_bf *) __a, __o);
+  __builtin_aarch64_st3v8bf ((__builtin_aarch64_simd_bf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_bf16 (bfloat16_t * __a, bfloat16x4x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  bfloat16x8x4_t __temp;
-  __temp.val[0] = vcombine_bf16 (__val.val[0], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_bf16 (__val.val[1], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_bf16 (__val.val[2], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_bf16 (__val.val[3], vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4v4bf ((__builtin_aarch64_simd_bf *) __a, __o);
+  __builtin_aarch64_st4v4bf ((__builtin_aarch64_simd_bf *) __a, __val);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_bf16 (bfloat16_t * __a, bfloat16x8x4_t __val)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4v8bf ((__builtin_aarch64_simd_bf *) __a, __o);
+  __builtin_aarch64_st4v8bf ((__builtin_aarch64_simd_bf *) __a, __val);
 }
 
 /* vreinterpret */
@@ -35851,125 +31592,55 @@ __extension__ extern __inline bfloat16x4x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2_lane_bf16 (const bfloat16_t * __ptr, bfloat16x4x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  bfloat16x8x2_t __temp;
-  __temp.val[0] = vcombine_bf16 (__b.val[0], vcreate_bf16 (0));
-  __temp.val[1] = vcombine_bf16 (__b.val[1], vcreate_bf16 (0));
-  __o = __builtin_aarch64_set_qregoiv8bf (__o, (bfloat16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv8bf (__o, (bfloat16x8_t) __temp.val[1], 1);
-  __o =	__builtin_aarch64_ld2_lanev4bf (
-	  (__builtin_aarch64_simd_bf *) __ptr, __o, __c);
-  __b.val[0] = (bfloat16x4_t) __builtin_aarch64_get_dregoidi (__o, 0);
-  __b.val[1] = (bfloat16x4_t) __builtin_aarch64_get_dregoidi (__o, 1);
-  return __b;
+  return __builtin_aarch64_ld2_lanev4bf (
+	  (__builtin_aarch64_simd_bf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline bfloat16x8x2_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld2q_lane_bf16 (const bfloat16_t * __ptr, bfloat16x8x2_t __b, const int __c)
 {
-  __builtin_aarch64_simd_oi __o;
-  bfloat16x8x2_t ret;
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregoiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_ld2_lanev8bf (
-	(__builtin_aarch64_simd_bf *) __ptr, __o, __c);
-  ret.val[0] = (bfloat16x8_t) __builtin_aarch64_get_qregoiv4si (__o, 0);
-  ret.val[1] = (bfloat16x8_t) __builtin_aarch64_get_qregoiv4si (__o, 1);
-  return ret;
+  return __builtin_aarch64_ld2_lanev8bf (
+	  (__builtin_aarch64_simd_bf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline bfloat16x4x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3_lane_bf16 (const bfloat16_t * __ptr, bfloat16x4x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  bfloat16x8x3_t __temp;
-  __temp.val[0] = vcombine_bf16 (__b.val[0], vcreate_bf16 (0));
-  __temp.val[1] = vcombine_bf16 (__b.val[1], vcreate_bf16 (0));
-  __temp.val[2] = vcombine_bf16 (__b.val[2], vcreate_bf16 (0));
-  __o = __builtin_aarch64_set_qregciv8bf (__o, (bfloat16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv8bf (__o, (bfloat16x8_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv8bf (__o, (bfloat16x8_t) __temp.val[2], 2);
-  __o =	__builtin_aarch64_ld3_lanev4bf (
-	  (__builtin_aarch64_simd_bf *) __ptr, __o, __c);
-  __b.val[0] = (bfloat16x4_t) __builtin_aarch64_get_dregcidi (__o, 0);
-  __b.val[1] = (bfloat16x4_t) __builtin_aarch64_get_dregcidi (__o, 1);
-  __b.val[2] = (bfloat16x4_t) __builtin_aarch64_get_dregcidi (__o, 2);
-  return __b;
+  return __builtin_aarch64_ld3_lanev4bf (
+	  (__builtin_aarch64_simd_bf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline bfloat16x8x3_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld3q_lane_bf16 (const bfloat16_t * __ptr, bfloat16x8x3_t __b, const int __c)
 {
-  __builtin_aarch64_simd_ci __o;
-  bfloat16x8x3_t ret;
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregciv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_ld3_lanev8bf (
-	(__builtin_aarch64_simd_bf *) __ptr, __o, __c);
-  ret.val[0] = (bfloat16x8_t) __builtin_aarch64_get_qregciv4si (__o, 0);
-  ret.val[1] = (bfloat16x8_t) __builtin_aarch64_get_qregciv4si (__o, 1);
-  ret.val[2] = (bfloat16x8_t) __builtin_aarch64_get_qregciv4si (__o, 2);
-  return ret;
+  return __builtin_aarch64_ld3_lanev8bf (
+	  (__builtin_aarch64_simd_bf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline bfloat16x4x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4_lane_bf16 (const bfloat16_t * __ptr, bfloat16x4x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  bfloat16x8x4_t __temp;
-  __temp.val[0] = vcombine_bf16 (__b.val[0], vcreate_bf16 (0));
-  __temp.val[1] = vcombine_bf16 (__b.val[1], vcreate_bf16 (0));
-  __temp.val[2] = vcombine_bf16 (__b.val[2], vcreate_bf16 (0));
-  __temp.val[3] = vcombine_bf16 (__b.val[3], vcreate_bf16 (0));
-  __o = __builtin_aarch64_set_qregxiv8bf (__o, (bfloat16x8_t) __temp.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv8bf (__o, (bfloat16x8_t) __temp.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv8bf (__o, (bfloat16x8_t) __temp.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv8bf (__o, (bfloat16x8_t) __temp.val[3], 3);
-  __o =	__builtin_aarch64_ld4_lanev4bf (
-	  (__builtin_aarch64_simd_bf *) __ptr, __o, __c);
-  __b.val[0] = (bfloat16x4_t) __builtin_aarch64_get_dregxidi (__o, 0);
-  __b.val[1] = (bfloat16x4_t) __builtin_aarch64_get_dregxidi (__o, 1);
-  __b.val[2] = (bfloat16x4_t) __builtin_aarch64_get_dregxidi (__o, 2);
-  __b.val[3] = (bfloat16x4_t) __builtin_aarch64_get_dregxidi (__o, 3);
-  return __b;
+  return __builtin_aarch64_ld4_lanev4bf (
+	  (__builtin_aarch64_simd_bf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline bfloat16x8x4_t
 __attribute__ ((__always_inline__, __gnu_inline__,__artificial__))
 vld4q_lane_bf16 (const bfloat16_t * __ptr, bfloat16x8x4_t __b, const int __c)
 {
-  __builtin_aarch64_simd_xi __o;
-  bfloat16x8x4_t ret;
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[0], 0);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[1], 1);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[2], 2);
-  __o = __builtin_aarch64_set_qregxiv4si (__o, (int32x4_t) __b.val[3], 3);
-  __o = __builtin_aarch64_ld4_lanev8bf (
-	(__builtin_aarch64_simd_bf *) __ptr, __o, __c);
-  ret.val[0] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 0);
-  ret.val[1] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 1);
-  ret.val[2] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 2);
-  ret.val[3] = (bfloat16x8_t) __builtin_aarch64_get_qregxiv4si (__o, 3);
-  return ret;
+  return __builtin_aarch64_ld4_lanev8bf (
+	  (__builtin_aarch64_simd_bf *) __ptr, __b, __c);
 }
 
 __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2_lane_bf16 (bfloat16_t *__ptr, bfloat16x4x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  bfloat16x8x2_t __temp;
-  __temp.val[0] = vcombine_bf16 (__val.val[0],
-				 vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_bf16 (__val.val[1],
-				 vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st2_lanev4bf ((__builtin_aarch64_simd_bf *) __ptr, __o,
+  __builtin_aarch64_st2_lanev4bf ((__builtin_aarch64_simd_bf *) __ptr, __val,
 				  __lane);
 }
 
@@ -35977,9 +31648,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst2q_lane_bf16 (bfloat16_t *__ptr, bfloat16x8x2_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_oi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st2_lanev8bf ((__builtin_aarch64_simd_bf *) __ptr, __o,
+  __builtin_aarch64_st2_lanev8bf ((__builtin_aarch64_simd_bf *) __ptr, __val,
 				  __lane);
 }
 
@@ -35987,16 +31656,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3_lane_bf16 (bfloat16_t *__ptr, bfloat16x4x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  bfloat16x8x3_t __temp;
-  __temp.val[0] = vcombine_bf16 (__val.val[0],
-				 vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_bf16 (__val.val[1],
-				 vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_bf16 (__val.val[2],
-				 vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st3_lanev4bf ((__builtin_aarch64_simd_bf *) __ptr, __o,
+  __builtin_aarch64_st3_lanev4bf ((__builtin_aarch64_simd_bf *) __ptr, __val,
 				  __lane);
 }
 
@@ -36004,9 +31664,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst3q_lane_bf16 (bfloat16_t *__ptr, bfloat16x8x3_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_ci __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st3_lanev8bf ((__builtin_aarch64_simd_bf *) __ptr, __o,
+  __builtin_aarch64_st3_lanev8bf ((__builtin_aarch64_simd_bf *) __ptr, __val,
 				  __lane);
 }
 
@@ -36014,18 +31672,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4_lane_bf16 (bfloat16_t *__ptr, bfloat16x4x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  bfloat16x8x4_t __temp;
-  __temp.val[0] = vcombine_bf16 (__val.val[0],
-				 vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[1] = vcombine_bf16 (__val.val[1],
-				 vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[2] = vcombine_bf16 (__val.val[2],
-				 vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __temp.val[3] = vcombine_bf16 (__val.val[3],
-				 vcreate_bf16 (__AARCH64_UINT64_C (0)));
-  __builtin_memcpy (&__o, &__temp, sizeof (__temp));
-  __builtin_aarch64_st4_lanev4bf ((__builtin_aarch64_simd_bf *) __ptr, __o,
+  __builtin_aarch64_st4_lanev4bf ((__builtin_aarch64_simd_bf *) __ptr, __val,
 				  __lane);
 }
 
@@ -36033,9 +31680,7 @@ __extension__ extern __inline void
 __attribute__ ((__always_inline__, __gnu_inline__, __artificial__))
 vst4q_lane_bf16 (bfloat16_t *__ptr, bfloat16x8x4_t __val, const int __lane)
 {
-  __builtin_aarch64_simd_xi __o;
-  __builtin_memcpy (&__o, &__val, sizeof (__val));
-  __builtin_aarch64_st4_lanev8bf ((__builtin_aarch64_simd_bf *) __ptr, __o,
+  __builtin_aarch64_st4_lanev8bf ((__builtin_aarch64_simd_bf *) __ptr, __val,
 				  __lane);
 }
 
diff --git a/gcc/config/aarch64/constraints.md b/gcc/config/aarch64/constraints.md
index 1863081..b197ec2 100644
--- a/gcc/config/aarch64/constraints.md
+++ b/gcc/config/aarch64/constraints.md
@@ -152,6 +152,14 @@
        (match_test "aarch64_symbolic_address_p (op)")
        (match_test "aarch64_mov_operand_p (op, GET_MODE (op))")))
 
+;; const is needed here to support UNSPEC_SALT_ADDR.
+(define_constraint "Usw"
+  "@internal
+   A constraint that matches a small GOT access."
+  (and (match_code "const,symbol_ref")
+       (match_test "aarch64_classify_symbolic_expression (op)
+		     == SYMBOL_SMALL_GOT_4G")))
+
 (define_constraint "Uss"
   "@internal
   A constraint that matches an immediate shift constant in SImode."
diff --git a/gcc/config/aarch64/geniterators.sh b/gcc/config/aarch64/geniterators.sh
index 5fd8bec9..52e07d4 100644
--- a/gcc/config/aarch64/geniterators.sh
+++ b/gcc/config/aarch64/geniterators.sh
@@ -64,7 +64,7 @@ iterdef {
 	gsub(/ *"[^"]*" *\)/, "", s)
 	gsub(/\( */, "", s)
 
-	if (s !~ /^[A-Za-z0-9_]+ \[[A-Z0-9 ]*\]$/)
+	if (s !~ /^[A-Za-z0-9_]+ \[[A-Za-z0-9 ]*\]$/)
 		next
 	sub(/\[ */, "", s)
 	sub(/ *\]/, "", s)
diff --git a/gcc/config/aarch64/host-aarch64-darwin.c b/gcc/config/aarch64/host-aarch64-darwin.c
new file mode 100644
index 0000000..d70f2df
--- /dev/null
+++ b/gcc/config/aarch64/host-aarch64-darwin.c
@@ -0,0 +1,33 @@
+/* aarch64/arm64-darwin host-specific hook definitions.
+
+Copyright The GNU Toolchain Authors.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#define IN_TARGET_CODE 1
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "hosthooks.h"
+#include "hosthooks-def.h"
+#include "config/host-darwin.h"
+
+/* Darwin doesn't do anything special for arm64/aarch64 hosts; this file
+   exists just to include the generic config/host-darwin.h.  */
+
+const struct host_hooks host_hooks = HOST_HOOKS_INITIALIZER;
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index aee32dc..a9842b3 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -119,6 +119,9 @@
 ;; VQMOV without 2-element modes.
 (define_mode_iterator VQMOV_NO2E [V16QI V8HI V4SI V8HF V8BF V4SF])
 
+;; Double integer vector modes.
+(define_mode_iterator VD_I [V8QI V4HI V2SI DI])
+
 ;; Quad integer vector modes.
 (define_mode_iterator VQ_I [V16QI V8HI V4SI V2DI])
 
@@ -207,6 +210,9 @@
 ;; All Advanced SIMD polynomial modes and DI.
 (define_mode_iterator VALLP [V8QI V16QI V4HI V8HI V2DI DI])
 
+;; All Advanced SIMD polynomial modes.
+(define_mode_iterator VALLP_NO_DI [V8QI V16QI V4HI V8HI V2DI])
+
 ;; Advanced SIMD modes for Integer reduction across lanes.
 (define_mode_iterator VDQV [V8QI V16QI V4HI V8HI V4SI V2DI])
 
@@ -235,6 +241,9 @@
 ;; Double vector modes for combines.
 (define_mode_iterator VDC [V8QI V4HI V4BF V4HF V2SI V2SF DI DF])
 
+;; Polynomial modes for vector combines.
+(define_mode_iterator VDC_P [V8QI V4HI DI])
+
 ;; Advanced SIMD modes except double int.
 (define_mode_iterator VDQIF [V8QI V16QI V4HI V8HI V2SI V4SI V2SF V4SF V2DF])
 (define_mode_iterator VDQIF_F16 [V8QI V16QI V4HI V8HI V2SI V4SI
@@ -299,6 +308,102 @@
 ;; Advanced SIMD opaque structure modes.
 (define_mode_iterator VSTRUCT [OI CI XI])
 
+;; Advanced SIMD 64-bit vector structure modes.
+(define_mode_iterator VSTRUCT_D [V2x8QI V2x4HI V2x2SI V2x1DI
+				 V2x4HF V2x2SF V2x1DF V2x4BF
+				 V3x8QI V3x4HI V3x2SI V3x1DI
+				 V3x4HF V3x2SF V3x1DF V3x4BF
+				 V4x8QI V4x4HI V4x2SI V4x1DI
+				 V4x4HF V4x2SF V4x1DF V4x4BF])
+
+;; Advanced SIMD 64-bit 2-vector structure modes.
+(define_mode_iterator VSTRUCT_2D [V2x8QI V2x4HI V2x2SI V2x1DI
+				  V2x4HF V2x2SF V2x1DF V2x4BF])
+
+;; Advanced SIMD 64-bit 3-vector structure modes.
+(define_mode_iterator VSTRUCT_3D [V3x8QI V3x4HI V3x2SI V3x1DI
+				  V3x4HF V3x2SF V3x1DF V3x4BF])
+
+;; Advanced SIMD 64-bit 4-vector structure modes.
+(define_mode_iterator VSTRUCT_4D [V4x8QI V4x4HI V4x2SI V4x1DI
+				  V4x4HF V4x2SF V4x1DF V4x4BF])
+
+;; Advanced SIMD 64-bit 2-vector structure modes minus V2x1DI and V2x1DF.
+(define_mode_iterator VSTRUCT_2DNX [V2x8QI V2x4HI V2x2SI V2x4HF
+				    V2x2SF V2x4BF])
+
+;; Advanced SIMD 64-bit 3-vector structure modes minus V3x1DI and V3x1DF.
+(define_mode_iterator VSTRUCT_3DNX [V3x8QI V3x4HI V3x2SI V3x4HF
+				    V3x2SF V3x4BF])
+
+;; Advanced SIMD 64-bit 4-vector structure modes minus V4x1DI and V4x1DF.
+(define_mode_iterator VSTRUCT_4DNX [V4x8QI V4x4HI V4x2SI V4x4HF
+				    V4x2SF V4x4BF])
+
+;; Advanced SIMD 64-bit structure modes with 64-bit elements.
+(define_mode_iterator VSTRUCT_DX [V2x1DI V2x1DF V3x1DI V3x1DF V4x1DI V4x1DF])
+
+;; Advanced SIMD 64-bit 2-vector structure modes with 64-bit elements.
+(define_mode_iterator VSTRUCT_2DX [V2x1DI V2x1DF])
+
+;; Advanced SIMD 64-bit 3-vector structure modes with 64-bit elements.
+(define_mode_iterator VSTRUCT_3DX [V3x1DI V3x1DF])
+
+;; Advanced SIMD 64-bit 4-vector structure modes with 64-bit elements.
+(define_mode_iterator VSTRUCT_4DX [V4x1DI V4x1DF])
+
+;; Advanced SIMD 128-bit vector structure modes.
+(define_mode_iterator VSTRUCT_Q [V2x16QI V2x8HI V2x4SI V2x2DI
+				 V2x8HF V2x4SF V2x2DF V2x8BF
+				 V3x16QI V3x8HI V3x4SI V3x2DI
+				 V3x8HF V3x4SF V3x2DF V3x8BF
+				 V4x16QI V4x8HI V4x4SI V4x2DI
+				 V4x8HF V4x4SF V4x2DF V4x8BF])
+
+;; Advanced SIMD 128-bit 2-vector structure modes.
+(define_mode_iterator VSTRUCT_2Q [V2x16QI V2x8HI V2x4SI V2x2DI
+				  V2x8HF V2x4SF V2x2DF V2x8BF])
+
+;; Advanced SIMD 128-bit 3-vector structure modes.
+(define_mode_iterator VSTRUCT_3Q [V3x16QI V3x8HI V3x4SI V3x2DI
+				  V3x8HF V3x4SF V3x2DF V3x8BF])
+
+;; Advanced SIMD 128-bit 4-vector structure modes.
+(define_mode_iterator VSTRUCT_4Q [V4x16QI V4x8HI V4x4SI V4x2DI
+				  V4x8HF V4x4SF V4x2DF V4x8BF])
+
+;; Advanced SIMD 2-vector structure modes.
+(define_mode_iterator VSTRUCT_2QD [V2x8QI V2x4HI V2x2SI V2x1DI
+				   V2x4HF V2x2SF V2x1DF V2x4BF
+				   V2x16QI V2x8HI V2x4SI V2x2DI
+				   V2x8HF V2x4SF V2x2DF V2x8BF])
+
+;; Advanced SIMD 3-vector structure modes.
+(define_mode_iterator VSTRUCT_3QD [V3x8QI V3x4HI V3x2SI V3x1DI
+				   V3x4HF V3x2SF V3x1DF V3x4BF
+				   V3x16QI V3x8HI V3x4SI V3x2DI
+				   V3x8HF V3x4SF V3x2DF V3x8BF])
+
+;; Advanced SIMD 4-vector structure modes.
+(define_mode_iterator VSTRUCT_4QD [V4x8QI V4x4HI V4x2SI V4x1DI
+				   V4x4HF V4x2SF V4x1DF V4x4BF
+				   V4x16QI V4x8HI V4x4SI V4x2DI
+				   V4x8HF V4x4SF V4x2DF V4x8BF])
+
+;; Advanced SIMD vector structure modes.
+(define_mode_iterator VSTRUCT_QD [V2x8QI V2x4HI V2x2SI V2x1DI
+				  V2x4HF V2x2SF V2x1DF V2x4BF
+				  V3x8QI V3x4HI V3x2SI V3x1DI
+				  V3x4HF V3x2SF V3x1DF V3x4BF
+				  V4x8QI V4x4HI V4x2SI V4x1DI
+				  V4x4HF V4x2SF V4x1DF V4x4BF
+				  V2x16QI V2x8HI V2x4SI V2x2DI
+				  V2x8HF V2x4SF V2x2DF V2x8BF
+				  V3x16QI V3x8HI V3x4SI V3x2DI
+				  V3x8HF V3x4SF V3x2DF V3x8BF
+				  V4x16QI V4x8HI V4x4SI V4x2DI
+				  V4x8HF V4x4SF V4x2DF V4x8BF])
+
 ;; Double scalar modes
 (define_mode_iterator DX [DI DF])
 
@@ -1021,7 +1126,31 @@
                          (DI   "1d") (DF    "1d")
                          (V2DI "2d") (V2SF "2s")
 			 (V4SF "4s") (V2DF "2d")
-			 (V4HF "4h") (V8HF "8h")])
+			 (V4HF "4h") (V8HF "8h")
+			 (V2x8QI "8b") (V2x4HI "4h")
+			 (V2x2SI "2s") (V2x1DI  "1d")
+			 (V2x4HF "4h") (V2x2SF "2s")
+			 (V2x1DF "1d") (V2x4BF "4h")
+			 (V2x16QI "16b") (V2x8HI "8h")
+			 (V2x4SI "4s") (V2x2DI "2d")
+			 (V2x8HF "8h") (V2x4SF "4s")
+			 (V2x2DF "2d") (V2x8BF "8h")
+			 (V3x8QI "8b") (V3x4HI "4h")
+			 (V3x2SI "2s") (V3x1DI "1d")
+			 (V3x4HF "4h") (V3x2SF "2s")
+			 (V3x1DF "1d") (V3x4BF "4h")
+			 (V3x16QI "16b") (V3x8HI "8h")
+			 (V3x4SI "4s") (V3x2DI "2d")
+			 (V3x8HF "8h") (V3x4SF "4s")
+			 (V3x2DF "2d") (V3x8BF "8h")
+			 (V4x8QI "8b") (V4x4HI "4h")
+			 (V4x2SI "2s") (V4x1DI "1d")
+			 (V4x4HF "4h") (V4x2SF "2s")
+			 (V4x1DF "1d") (V4x4BF "4h")
+			 (V4x16QI "16b") (V4x8HI "8h")
+			 (V4x4SI "4s") (V4x2DI "2d")
+			 (V4x8HF "8h") (V4x4SF "4s")
+			 (V4x2DF "2d") (V4x8BF "8h")])
 
 ;; Map mode to type used in widening multiplies.
 (define_mode_attr Vcondtype [(V4HI "4h") (V8HI "4h") (V2SI "2s") (V4SI "2s")])
@@ -1059,6 +1188,30 @@
 			  (V4HF "h") (V8HF  "h")
 			  (V2SF "s") (V4SF  "s")
 			  (V2DF "d")
+			  (V2x8QI "b") (V2x4HI "h")
+			  (V2x2SI "s") (V2x1DI "d")
+			  (V2x4HF "h") (V2x2SF "s")
+			  (V2x1DF "d") (V2x4BF "h")
+			  (V2x16QI "b") (V2x8HI "h")
+			  (V2x4SI "s") (V2x2DI "d")
+			  (V2x8HF "h") (V2x4SF "s")
+			  (V2x2DF "d") (V2x8BF "h")
+			  (V3x8QI "b") (V3x4HI "h")
+			  (V3x2SI "s") (V3x1DI "d")
+			  (V3x4HF "h") (V3x2SF "s")
+			  (V3x1DF "d") (V3x4BF "h")
+			  (V3x16QI "b") (V3x8HI "h")
+			  (V3x4SI "s") (V3x2DI "d")
+			  (V3x8HF "h") (V3x4SF "s")
+			  (V3x2DF "d") (V3x8BF "h")
+			  (V4x8QI "b") (V4x4HI "h")
+			  (V4x2SI "s") (V4x1DI "d")
+			  (V4x4HF "h") (V4x2SF "s")
+			  (V4x1DF "d") (V4x4BF "h")
+			  (V4x16QI "b") (V4x8HI "h")
+			  (V4x4SI "s") (V4x2DI "d")
+			  (V4x8HF "h") (V4x4SF "s")
+			  (V4x2DF "d") (V4x8BF "h")
 			  (VNx16BI "b") (VNx8BI "h") (VNx4BI "s") (VNx2BI "d")
 			  (VNx16QI "b") (VNx8QI "b") (VNx4QI "b") (VNx2QI "b")
 			  (VNx8HI "h") (VNx4HI "h") (VNx2HI "h")
@@ -1138,6 +1291,58 @@
 			  (SI   "8b")  (SF    "8b")
 			  (V4BF "8b")  (V8BF  "16b")])
 
+;; Advanced SIMD vector structure to element modes.
+(define_mode_attr VSTRUCT_ELT [(V2x8QI "V8QI") (V2x4HI "V4HI")
+			       (V2x2SI "V2SI") (V2x1DI "DI")
+			       (V2x4HF "V4HF") (V2x2SF "V2SF")
+			       (V2x1DF "DF") (V2x4BF "V4BF")
+			       (V3x8QI "V8QI") (V3x4HI "V4HI")
+			       (V3x2SI "V2SI") (V3x1DI "DI")
+			       (V3x4HF "V4HF") (V3x2SF "V2SF")
+			       (V3x1DF "DF") (V3x4BF "V4BF")
+			       (V4x8QI "V8QI") (V4x4HI "V4HI")
+			       (V4x2SI "V2SI") (V4x1DI "DI")
+			       (V4x4HF "V4HF") (V4x2SF "V2SF")
+			       (V4x1DF "DF") (V4x4BF "V4BF")
+			       (V2x16QI "V16QI") (V2x8HI "V8HI")
+			       (V2x4SI "V4SI") (V2x2DI "V2DI")
+			       (V2x8HF "V8HF") (V2x4SF "V4SF")
+			       (V2x2DF "V2DF") (V2x8BF "V8BF")
+			       (V3x16QI "V16QI") (V3x8HI "V8HI")
+			       (V3x4SI "V4SI") (V3x2DI "V2DI")
+			       (V3x8HF "V8HF") (V3x4SF "V4SF")
+			       (V3x2DF "V2DF") (V3x8BF "V8BF")
+			       (V4x16QI "V16QI") (V4x8HI "V8HI")
+			       (V4x4SI "V4SI") (V4x2DI "V2DI")
+			       (V4x8HF "V8HF") (V4x4SF "V4SF")
+			       (V4x2DF "V2DF") (V4x8BF "V8BF")])
+
+;; Advanced SIMD vector structure to element modes in lower case.
+(define_mode_attr vstruct_elt [(V2x8QI "v8qi") (V2x4HI "v4hi")
+			       (V2x2SI "v2si") (V2x1DI "di")
+			       (V2x4HF "v4hf") (V2x2SF "v2sf")
+			       (V2x1DF "df") (V2x4BF "v4bf")
+			       (V3x8QI "v8qi") (V3x4HI "v4hi")
+			       (V3x2SI "v2si") (V3x1DI "di")
+			       (V3x4HF "v4hf") (V3x2SF "v2sf")
+			       (V3x1DF "df") (V3x4BF "v4bf")
+			       (V4x8QI "v8qi") (V4x4HI "v4hi")
+			       (V4x2SI "v2si") (V4x1DI "di")
+			       (V4x4HF "v4hf") (V4x2SF "v2sf")
+			       (V4x1DF "df") (V4x4BF "v4bf")
+			       (V2x16QI "v16qi") (V2x8HI "v8hi")
+			       (V2x4SI "v4si") (V2x2DI "v2di")
+			       (V2x8HF "v8hf") (V2x4SF "v4sf")
+			       (V2x2DF "v2df") (V2x8BF "v8bf")
+			       (V3x16QI "v16qi") (V3x8HI "v8hi")
+			       (V3x4SI "v4si") (V3x2DI "v2di")
+			       (V3x8HF "v8hf") (V3x4SF "v4sf")
+			       (V3x2DF "v2df") (V3x8BF "v8bf")
+			       (V4x16QI "v16qi") (V4x8HI "v8hi")
+			       (V4x4SI "v4si") (V4x2DI "v2di")
+			       (V4x8HF "v8hf") (V4x4SF "v4sf")
+			       (V4x2DF "v2df") (V4x8BF "v8bf")])
+
 ;; Define element mode for each vector mode.
 (define_mode_attr VEL [(V8QI  "QI") (V16QI "QI")
 		       (V4HI "HI") (V8HI  "HI")
@@ -1492,12 +1697,60 @@
 (define_mode_attr vwx [(V4HI "x") (V8HI "x") (HI "x")
 		       (V2SI "w") (V4SI "w") (SI "w")])
 
-(define_mode_attr Vendreg [(OI "T") (CI "U") (XI "V")])
+(define_mode_attr Vendreg [(OI "T") (CI "U") (XI "V")
+			   (V2x8QI "T") (V2x16QI "T")
+			   (V2x4HI "T") (V2x8HI "T")
+			   (V2x2SI "T") (V2x4SI "T")
+			   (V2x1DI "T") (V2x2DI "T")
+			   (V2x4HF "T") (V2x8HF "T")
+			   (V2x2SF "T") (V2x4SF "T")
+			   (V2x1DF "T") (V2x2DF "T")
+			   (V2x4BF "T") (V2x8BF "T")
+			   (V3x8QI "U") (V3x16QI "U")
+			   (V3x4HI "U") (V3x8HI "U")
+			   (V3x2SI "U") (V3x4SI "U")
+			   (V3x1DI "U") (V3x2DI "U")
+			   (V3x4HF "U") (V3x8HF "U")
+			   (V3x2SF "U") (V3x4SF "U")
+			   (V3x1DF "U") (V3x2DF "U")
+			   (V3x4BF "U") (V3x8BF "U")
+			   (V4x8QI "V") (V4x16QI "V")
+			   (V4x4HI "V") (V4x8HI "V")
+			   (V4x2SI "V") (V4x4SI "V")
+			   (V4x1DI "V") (V4x2DI "V")
+			   (V4x4HF "V") (V4x8HF "V")
+			   (V4x2SF "V") (V4x4SF "V")
+			   (V4x1DF "V") (V4x2DF "V")
+			   (V4x4BF "V") (V4x8BF "V")])
 
 ;; This is both the number of Q-Registers needed to hold the corresponding
 ;; opaque large integer mode, and the number of elements touched by the
 ;; ld..._lane and st..._lane operations.
-(define_mode_attr nregs [(OI "2") (CI "3") (XI "4")])
+(define_mode_attr nregs [(OI "2") (CI "3") (XI "4")
+			 (V2x8QI "2") (V2x16QI "2")
+			 (V2x4HI "2") (V2x8HI "2")
+			 (V2x2SI "2") (V2x4SI "2")
+			 (V2x1DI "2") (V2x2DI "2")
+			 (V2x4HF "2") (V2x8HF "2")
+			 (V2x2SF "2") (V2x4SF "2")
+			 (V2x1DF "2") (V2x2DF "2")
+			 (V2x4BF "2") (V2x8BF "2")
+			 (V3x8QI "3") (V3x16QI "3")
+			 (V3x4HI "3") (V3x8HI "3")
+			 (V3x2SI "3") (V3x4SI "3")
+			 (V3x1DI "3") (V3x2DI "3")
+			 (V3x4HF "3") (V3x8HF "3")
+			 (V3x2SF "3") (V3x4SF "3")
+			 (V3x1DF "3") (V3x2DF "3")
+			 (V3x4BF "3") (V3x8BF "3")
+			 (V4x8QI "4") (V4x16QI "4")
+			 (V4x4HI "4") (V4x8HI "4")
+			 (V4x2SI "4") (V4x4SI "4")
+			 (V4x1DI "4") (V4x2DI "4")
+			 (V4x4HF "4") (V4x8HF "4")
+			 (V4x2SF "4") (V4x4SF "4")
+			 (V4x1DF "4") (V4x2DF "4")
+			 (V4x4BF "4") (V4x8BF "4")])
 
 ;; Mode for atomic operation suffixes
 (define_mode_attr atomic_sfx
@@ -1575,7 +1828,31 @@
 		     (V4BF "") (V8BF "_q")
 		     (V2SF "") (V4SF  "_q")
 			       (V2DF  "_q")
-		     (QI "") (HI "") (SI "") (DI "") (HF "") (SF "") (DF "")])
+		     (QI "") (HI "") (SI "") (DI "") (HF "") (SF "") (DF "")
+		     (V2x8QI "") (V2x16QI "_q")
+		     (V2x4HI "") (V2x8HI "_q")
+		     (V2x2SI "") (V2x4SI "_q")
+		     (V2x1DI "") (V2x2DI "_q")
+		     (V2x4HF "") (V2x8HF "_q")
+		     (V2x2SF "") (V2x4SF "_q")
+		     (V2x1DF "") (V2x2DF "_q")
+		     (V2x4BF "") (V2x8BF "_q")
+		     (V3x8QI "") (V3x16QI "_q")
+		     (V3x4HI "") (V3x8HI "_q")
+		     (V3x2SI "") (V3x4SI "_q")
+		     (V3x1DI "") (V3x2DI "_q")
+		     (V3x4HF "") (V3x8HF "_q")
+		     (V3x2SF "") (V3x4SF "_q")
+		     (V3x1DF "") (V3x2DF "_q")
+		     (V3x4BF "") (V3x8BF "_q")
+		     (V4x8QI "") (V4x16QI "_q")
+		     (V4x4HI "") (V4x8HI "_q")
+		     (V4x2SI "") (V4x4SI "_q")
+		     (V4x1DI "") (V4x2DI "_q")
+		     (V4x4HF "") (V4x8HF "_q")
+		     (V4x2SF "") (V4x4SF "_q")
+		     (V4x1DF "") (V4x2DF "_q")
+		     (V4x4BF "") (V4x8BF "_q")])
 
 (define_mode_attr vp [(V8QI "v") (V16QI "v")
 		      (V4HI "v") (V8HI  "v")
@@ -1597,7 +1874,31 @@
 (define_mode_attr Vbfdottype [(V2SF "4h") (V4SF "8h")])
 
 ;; Sum of lengths of instructions needed to move vector registers of a mode.
-(define_mode_attr insn_count [(OI "8") (CI "12") (XI "16")])
+(define_mode_attr insn_count [(OI "8") (CI "12") (XI "16")
+			      (V2x8QI "8") (V2x16QI "8")
+			      (V2x4HI "8") (V2x8HI "8")
+			      (V2x2SI "8") (V2x4SI "8")
+			      (V2x1DI "8") (V2x2DI "8")
+			      (V2x4HF "8") (V2x8HF "8")
+			      (V2x2SF "8") (V2x4SF "8")
+			      (V2x1DF "8") (V2x2DF "8")
+			      (V2x4BF "8") (V2x8BF "8")
+			      (V3x8QI "12") (V3x16QI "12")
+			      (V3x4HI "12") (V3x8HI "12")
+			      (V3x2SI "12") (V3x4SI "12")
+			      (V3x1DI "12") (V3x2DI "12")
+			      (V3x4HF "12") (V3x8HF "12")
+			      (V3x2SF "12") (V3x4SF "12")
+			      (V3x1DF "12") (V3x2DF "12")
+			      (V3x4BF "12") (V3x8BF "12")
+			      (V4x8QI "16") (V4x16QI "16")
+			      (V4x4HI "16") (V4x8HI "16")
+			      (V4x2SI "16") (V4x4SI "16")
+			      (V4x1DI "16") (V4x2DI "16")
+			      (V4x4HF "16") (V4x8HF "16")
+			      (V4x2SF "16") (V4x4SF "16")
+			      (V4x1DF "16") (V4x2DF "16")
+			      (V4x4BF "16") (V4x8BF "16")])
 
 ;; -fpic small model GOT reloc modifers: gotpage_lo15/lo14 for ILP64/32.
 ;; No need of iterator for -fPIC as it use got_lo12 for both modes.
@@ -2897,9 +3198,9 @@
 			(UNSPEC_COND_FCVTZS "fix_trunc")
 			(UNSPEC_COND_FCVTZU "fixuns_trunc")
 			(UNSPEC_COND_FDIV "div")
-			(UNSPEC_COND_FMAX "smax_nan")
+			(UNSPEC_COND_FMAX "fmax_nan")
 			(UNSPEC_COND_FMAXNM "smax")
-			(UNSPEC_COND_FMIN "smin_nan")
+			(UNSPEC_COND_FMIN "fmin_nan")
 			(UNSPEC_COND_FMINNM "smin")
 			(UNSPEC_COND_FMLA "fma")
 			(UNSPEC_COND_FMLS "fnma")
@@ -2922,22 +3223,12 @@
 			(UNSPEC_COND_SCVTF "float")
 			(UNSPEC_COND_UCVTF "floatuns")])
 
-(define_int_attr  maxmin_uns [(UNSPEC_UMAXV "umax")
-			      (UNSPEC_UMINV "umin")
-			      (UNSPEC_SMAXV "smax")
-			      (UNSPEC_SMINV "smin")
-			      (UNSPEC_FMAX  "smax_nan")
-			      (UNSPEC_FMAXNMV "smax")
-			      (UNSPEC_FMAXV "smax_nan")
-			      (UNSPEC_FMIN "smin_nan")
-			      (UNSPEC_FMINNMV "smin")
-			      (UNSPEC_FMINV "smin_nan")
-			      (UNSPEC_FMAXNM "fmax")
-			      (UNSPEC_FMINNM "fmin")
-			      (UNSPEC_COND_FMAX "fmax_nan")
-			      (UNSPEC_COND_FMAXNM "fmax")
-			      (UNSPEC_COND_FMIN "fmin_nan")
-			      (UNSPEC_COND_FMINNM "fmin")])
+(define_int_attr fmaxmin [(UNSPEC_FMAX "fmax_nan")
+			  (UNSPEC_FMAXNM "fmax")
+			  (UNSPEC_FMIN "fmin_nan")
+			  (UNSPEC_FMINNM "fmin")
+			  (UNSPEC_COND_FMAXNM "fmax")
+			  (UNSPEC_COND_FMINNM "fmin")])
 
 (define_int_attr  maxmin_uns_op [(UNSPEC_UMAXV "umax")
 				 (UNSPEC_UMINV "umin")
diff --git a/gcc/config/aarch64/x-darwin b/gcc/config/aarch64/x-darwin
new file mode 100644
index 0000000..6d788d5
--- /dev/null
+++ b/gcc/config/aarch64/x-darwin
@@ -0,0 +1,3 @@
+host-aarch64-darwin.o : $(srcdir)/config/aarch64/host-aarch64-darwin.c
+	$(COMPILE) $<
+	$(POSTCOMPILE)
diff --git a/gcc/config/arc/arc.md b/gcc/config/arc/arc.md
index 4919d27..74ec38f 100644
--- a/gcc/config/arc/arc.md
+++ b/gcc/config/arc/arc.md
@@ -6023,26 +6023,26 @@ core_3, archs4x, archs4xd, archs4xd_slow"
 (define_expand "maddhisi4"
   [(match_operand:SI 0 "register_operand" "")
    (match_operand:HI 1 "register_operand" "")
-   (match_operand:HI 2 "extend_operand"   "")
+   (match_operand:HI 2 "register_operand" "")
    (match_operand:SI 3 "register_operand" "")]
   "TARGET_PLUS_MACD"
   "{
-   rtx acc_reg = gen_rtx_REG (SImode, ACC_REG_FIRST);
+   rtx acc_reg = gen_rtx_REG (SImode, ACCL_REGNO);
 
    emit_move_insn (acc_reg, operands[3]);
-   emit_insn (gen_machi (operands[1], operands[2]));
-   emit_move_insn (operands[0], acc_reg);
+   emit_insn (gen_machi (operands[0], operands[1], operands[2], acc_reg));
    DONE;
   }")
 
 (define_insn "machi"
-  [(set (reg:SI ARCV2_ACC)
+  [(set (match_operand:SI 0 "register_operand" "=Ral,r")
 	(plus:SI
-	 (mult:SI (sign_extend:SI (match_operand:HI 0 "register_operand" "%r"))
-		  (sign_extend:SI (match_operand:HI 1 "register_operand" "r")))
-	 (reg:SI ARCV2_ACC)))]
+	 (mult:SI (sign_extend:SI (match_operand:HI 1 "register_operand" "%r,r"))
+		  (sign_extend:SI (match_operand:HI 2 "register_operand" "r,r")))
+	 (match_operand:SI 3 "accl_operand" "")))
+   (clobber (reg:DI ARCV2_ACC))]
   "TARGET_PLUS_MACD"
-  "vmac2h\\t0,%0,%1"
+  "dmach\\t%0,%1,%2"
   [(set_attr "length" "4")
    (set_attr "type" "multi")
    (set_attr "predicable" "no")
@@ -6056,22 +6056,22 @@ core_3, archs4x, archs4xd, archs4xd_slow"
    (match_operand:SI 3 "register_operand" "")]
   "TARGET_PLUS_MACD"
   "{
-   rtx acc_reg = gen_rtx_REG (SImode, ACC_REG_FIRST);
+   rtx acc_reg = gen_rtx_REG (SImode, ACCL_REGNO);
 
    emit_move_insn (acc_reg, operands[3]);
-   emit_insn (gen_umachi (operands[1], operands[2]));
-   emit_move_insn (operands[0], acc_reg);
+   emit_insn (gen_umachi (operands[0], operands[1], operands[2], acc_reg));
    DONE;
   }")
 
 (define_insn "umachi"
-  [(set (reg:SI ARCV2_ACC)
+  [(set (match_operand:SI 0 "register_operand" "=Ral,r")
 	(plus:SI
-	 (mult:SI (zero_extend:SI (match_operand:HI 0 "register_operand" "%r"))
-		  (zero_extend:SI (match_operand:HI 1 "register_operand" "r")))
-	 (reg:SI ARCV2_ACC)))]
+	 (mult:SI (zero_extend:SI (match_operand:HI 1 "register_operand" "%r,r"))
+		  (zero_extend:SI (match_operand:HI 2 "register_operand" "r,r")))
+	 (match_operand:SI 3 "accl_operand" "")))
+   (clobber (reg:DI ARCV2_ACC))]
   "TARGET_PLUS_MACD"
-  "vmac2hu\\t0,%0,%1"
+  "dmachu\\t%0,%1,%2"
   [(set_attr "length" "4")
    (set_attr "type" "multi")
    (set_attr "predicable" "no")
diff --git a/gcc/config/arm/aarch-common-protos.h b/gcc/config/arm/aarch-common-protos.h
index 6be5fb1..55a470d 100644
--- a/gcc/config/arm/aarch-common-protos.h
+++ b/gcc/config/arm/aarch-common-protos.h
@@ -133,6 +133,9 @@ struct vector_cost_table
 {
   const int alu;
   const int mult;
+  const int movi;
+  const int dup;
+  const int extract;
 };
 
 struct cpu_cost_table
diff --git a/gcc/config/arm/aarch-cost-tables.h b/gcc/config/arm/aarch-cost-tables.h
index 25ff702..0e6a626 100644
--- a/gcc/config/arm/aarch-cost-tables.h
+++ b/gcc/config/arm/aarch-cost-tables.h
@@ -122,7 +122,10 @@ const struct cpu_cost_table generic_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* alu.  */
-    COSTS_N_INSNS (4)   /* mult.  */
+    COSTS_N_INSNS (4),  /* mult.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (2),  /* dup.  */
+    COSTS_N_INSNS (2)   /* extract.  */
   }
 };
 
@@ -226,7 +229,10 @@ const struct cpu_cost_table cortexa53_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* alu.  */
-    COSTS_N_INSNS (4)   /* mult.  */
+    COSTS_N_INSNS (4),  /* mult.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (2),  /* dup.  */
+    COSTS_N_INSNS (2)   /* extract.  */
   }
 };
 
@@ -330,7 +336,10 @@ const struct cpu_cost_table cortexa57_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),  /* alu.  */
-    COSTS_N_INSNS (4)   /* mult.  */
+    COSTS_N_INSNS (4),  /* mult.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (2),  /* dup.  */
+    COSTS_N_INSNS (2)   /* extract.  */
   }
 };
 
@@ -434,7 +443,10 @@ const struct cpu_cost_table cortexa76_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),  /* alu.  */
-    COSTS_N_INSNS (4)   /* mult.  */
+    COSTS_N_INSNS (4),  /* mult.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (2),  /* dup.  */
+    COSTS_N_INSNS (2)   /* extract.  */
   }
 };
 
@@ -538,7 +550,10 @@ const struct cpu_cost_table exynosm1_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (0),  /* alu.  */
-    COSTS_N_INSNS (4)   /* mult.  */
+    COSTS_N_INSNS (4),  /* mult.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (2),  /* dup.  */
+    COSTS_N_INSNS (2)   /* extract.  */
   }
 };
 
@@ -642,7 +657,10 @@ const struct cpu_cost_table xgene1_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (2),  /* alu.  */
-    COSTS_N_INSNS (8)   /* mult.  */
+    COSTS_N_INSNS (8),  /* mult.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (2),  /* dup.  */
+    COSTS_N_INSNS (2)   /* extract.  */
   }
 };
 
diff --git a/gcc/config/arm/arm-cpus.in b/gcc/config/arm/arm-cpus.in
index d0d0d0f..a6a8e43 100644
--- a/gcc/config/arm/arm-cpus.in
+++ b/gcc/config/arm/arm-cpus.in
@@ -132,6 +132,9 @@ define feature cmse
 # Architecture rel 8.1-M.
 define feature armv8_1m_main
 
+# Architecture rel 9.0.
+define feature armv9
+
 # Floating point and Neon extensions.
 # VFPv1 is not supported in GCC.
 
@@ -293,6 +296,7 @@ define fgroup ARMv8m_base ARMv6m armv8 cmse tdiv
 define fgroup ARMv8m_main ARMv7m armv8 cmse
 define fgroup ARMv8r      ARMv8a
 define fgroup ARMv8_1m_main ARMv8m_main armv8_1m_main
+define fgroup ARMv9a      ARMv8_5a armv9
 
 # Useful combinations.
 define fgroup VFPv2	vfpv2
@@ -751,6 +755,21 @@ begin arch armv8.1-m.main
  option cdecp7 add cdecp7
 end arch armv8.1-m.main
 
+begin arch armv9-a
+ tune for cortex-a53
+ tune flags CO_PROC
+ base 9A
+ profile A
+ isa ARMv9a
+ option simd add FP_ARMv8 DOTPROD
+ option fp16 add fp16 fp16fml FP_ARMv8 DOTPROD
+ option crypto add FP_ARMv8 CRYPTO DOTPROD
+ option nocrypto remove ALL_CRYPTO
+ option nofp remove ALL_FP
+ option i8mm add i8mm FP_ARMv8 DOTPROD
+ option bf16 add bf16 FP_ARMv8 DOTPROD
+end arch armv9-a
+
 begin arch iwmmxt
  tune for iwmmxt
  tune flags LDSCHED STRONG XSCALE
@@ -1494,6 +1513,17 @@ begin cpu cortex-a78c
  part d4b
 end cpu cortex-a78c
 
+begin cpu cortex-a710
+ cname cortexa710
+ tune for cortex-a57
+ tune flags LDSCHED
+ architecture armv9-a+fp16+bf16+i8mm
+ option crypto add FP_ARMv8 CRYPTO
+ costs cortex_a57
+ vendor 41
+ part d47
+end cpu cortex-a710
+
 begin cpu cortex-x1
  cname cortexx1
  tune for cortex-a57
diff --git a/gcc/config/arm/arm-tables.opt b/gcc/config/arm/arm-tables.opt
index 8bb0c9f..6e457fb 100644
--- a/gcc/config/arm/arm-tables.opt
+++ b/gcc/config/arm/arm-tables.opt
@@ -250,6 +250,9 @@ EnumValue
 Enum(processor_type) String(cortex-a78c) Value( TARGET_CPU_cortexa78c)
 
 EnumValue
+Enum(processor_type) String(cortex-a710) Value( TARGET_CPU_cortexa710)
+
+EnumValue
 Enum(processor_type) String(cortex-x1) Value( TARGET_CPU_cortexx1)
 
 EnumValue
@@ -383,10 +386,13 @@ EnumValue
 Enum(arm_arch) String(armv8.1-m.main) Value(30)
 
 EnumValue
-Enum(arm_arch) String(iwmmxt) Value(31)
+Enum(arm_arch) String(armv9-a) Value(31)
+
+EnumValue
+Enum(arm_arch) String(iwmmxt) Value(32)
 
 EnumValue
-Enum(arm_arch) String(iwmmxt2) Value(32)
+Enum(arm_arch) String(iwmmxt2) Value(33)
 
 Enum
 Name(arm_fpu) Type(enum fpu_type)
diff --git a/gcc/config/arm/arm-tune.md b/gcc/config/arm/arm-tune.md
index 6482833..54e701f 100644
--- a/gcc/config/arm/arm-tune.md
+++ b/gcc/config/arm/arm-tune.md
@@ -46,8 +46,9 @@
 	cortexa73cortexa53,cortexa55,cortexa75,
 	cortexa76,cortexa76ae,cortexa77,
 	cortexa78,cortexa78ae,cortexa78c,
-	cortexx1,neoversen1,cortexa75cortexa55,
-	cortexa76cortexa55,neoversev1,neoversen2,
-	cortexm23,cortexm33,cortexm35p,
-	cortexm55,cortexr52,cortexr52plus"
+	cortexa710,cortexx1,neoversen1,
+	cortexa75cortexa55,cortexa76cortexa55,neoversev1,
+	neoversen2,cortexm23,cortexm33,
+	cortexm35p,cortexm55,cortexr52,
+	cortexr52plus"
 	(const (symbol_ref "((enum attr_tune) arm_tune)")))
diff --git a/gcc/config/arm/arm.c b/gcc/config/arm/arm.c
index c4ff06b..a5b403e 100644
--- a/gcc/config/arm/arm.c
+++ b/gcc/config/arm/arm.c
@@ -1189,7 +1189,10 @@ const struct cpu_cost_table cortexa9_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* alu.  */
-    COSTS_N_INSNS (4)	/* mult.  */
+    COSTS_N_INSNS (4),	/* mult.  */
+    COSTS_N_INSNS (1),	/* movi.  */
+    COSTS_N_INSNS (2),	/* dup.  */
+    COSTS_N_INSNS (2)	/* extract.  */
   }
 };
 
@@ -1293,7 +1296,10 @@ const struct cpu_cost_table cortexa8_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* alu.  */
-    COSTS_N_INSNS (4)	/* mult.  */
+    COSTS_N_INSNS (4),	/* mult.  */
+    COSTS_N_INSNS (1),	/* movi.  */
+    COSTS_N_INSNS (2),	/* dup.  */
+    COSTS_N_INSNS (2)	/* extract.  */
   }
 };
 
@@ -1398,7 +1404,10 @@ const struct cpu_cost_table cortexa5_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* alu.  */
-    COSTS_N_INSNS (4)	/* mult.  */
+    COSTS_N_INSNS (4),	/* mult.  */
+    COSTS_N_INSNS (1),	/* movi.  */
+    COSTS_N_INSNS (2),	/* dup.  */
+    COSTS_N_INSNS (2)	/* extract.  */
   }
 };
 
@@ -1504,7 +1513,10 @@ const struct cpu_cost_table cortexa7_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* alu.  */
-    COSTS_N_INSNS (4)	/* mult.  */
+    COSTS_N_INSNS (4),	/* mult.  */
+    COSTS_N_INSNS (1),	/* movi.  */
+    COSTS_N_INSNS (2),	/* dup.  */
+    COSTS_N_INSNS (2)	/* extract.  */
   }
 };
 
@@ -1608,7 +1620,10 @@ const struct cpu_cost_table cortexa12_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* alu.  */
-    COSTS_N_INSNS (4)	/* mult.  */
+    COSTS_N_INSNS (4),	/* mult.  */
+    COSTS_N_INSNS (1),	/* movi.  */
+    COSTS_N_INSNS (2),	/* dup.  */
+    COSTS_N_INSNS (2)	/* extract.  */
   }
 };
 
@@ -1712,7 +1727,10 @@ const struct cpu_cost_table cortexa15_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* alu.  */
-    COSTS_N_INSNS (4)	/* mult.  */
+    COSTS_N_INSNS (4),	/* mult.  */
+    COSTS_N_INSNS (1),	/* movi.  */
+    COSTS_N_INSNS (2),	/* dup.  */
+    COSTS_N_INSNS (2)	/* extract.  */
   }
 };
 
@@ -1816,7 +1834,10 @@ const struct cpu_cost_table v7m_extra_costs =
   /* Vector */
   {
     COSTS_N_INSNS (1),	/* alu.  */
-    COSTS_N_INSNS (4)	/* mult.  */
+    COSTS_N_INSNS (4),	/* mult.  */
+    COSTS_N_INSNS (1),	/* movi.  */
+    COSTS_N_INSNS (2),	/* dup.  */
+    COSTS_N_INSNS (2)	/* extract.  */
   }
 };
 
diff --git a/gcc/config/arm/arm.h b/gcc/config/arm/arm.h
index 015299c..3a8d223 100644
--- a/gcc/config/arm/arm.h
+++ b/gcc/config/arm/arm.h
@@ -452,7 +452,8 @@ enum base_architecture
   BASE_ARCH_8A = 8,
   BASE_ARCH_8M_BASE = 8,
   BASE_ARCH_8M_MAIN = 8,
-  BASE_ARCH_8R = 8
+  BASE_ARCH_8R = 8,
+  BASE_ARCH_9A = 9
 };
 
 /* The major revision number of the ARM Architecture implemented by the target.  */
diff --git a/gcc/config/arm/t-aprofile b/gcc/config/arm/t-aprofile
index 8574ac3..68e2251 100644
--- a/gcc/config/arm/t-aprofile
+++ b/gcc/config/arm/t-aprofile
@@ -26,8 +26,8 @@
 
 # Arch and FPU variants to build libraries with
 
-MULTI_ARCH_OPTS_A       = march=armv7-a/march=armv7-a+fp/march=armv7-a+simd/march=armv7ve+simd/march=armv8-a/march=armv8-a+simd
-MULTI_ARCH_DIRS_A       = v7-a v7-a+fp v7-a+simd v7ve+simd v8-a v8-a+simd
+MULTI_ARCH_OPTS_A       = march=armv7-a/march=armv7-a+fp/march=armv7-a+simd/march=armv7ve+simd/march=armv8-a/march=armv8-a+simd/march=armv9-a/march=armv9-a+simd
+MULTI_ARCH_DIRS_A       = v7-a v7-a+fp v7-a+simd v7ve+simd v8-a v8-a+simd v9-a v9-a+simd
 
 # ARMv7-A - build nofp, fp-d16 and SIMD variants
 
@@ -46,6 +46,11 @@ MULTILIB_REQUIRED	+= mthumb/march=armv8-a/mfloat-abi=soft
 MULTILIB_REQUIRED	+= mthumb/march=armv8-a+simd/mfloat-abi=hard
 MULTILIB_REQUIRED	+= mthumb/march=armv8-a+simd/mfloat-abi=softfp
 
+# Armv9-A - build nofp and SIMD variants.
+MULTILIB_REQUIRED	+= mthumb/march=armv9-a/mfloat-abi=soft
+MULTILIB_REQUIRED	+= mthumb/march=armv9-a+simd/mfloat-abi=hard
+MULTILIB_REQUIRED	+= mthumb/march=armv9-a+simd/mfloat-abi=softfp
+
 # Matches
 
 # Arch Matches
@@ -129,17 +134,29 @@ MULTILIB_MATCHES	+= march?armv8-a=march?armv8.6-a
 MULTILIB_MATCHES	+= $(foreach ARCH, $(v8_6_a_simd_variants), \
 			     march?armv8-a+simd=march?armv8.6-a$(ARCH))
 
+# Armv9 without SIMD: map down to base architecture
+MULTILIB_MATCHES	+= $(foreach ARCH, $(v9_a_nosimd_variants), \
+			     march?armv9-a=march?armv9-a$(ARCH))
+
+# Armv9 with SIMD: map down to base arch + simd
+MULTILIB_MATCHES	+= march?armv9-a+simd=march?armv9-a+crc+simd \
+			   $(foreach ARCH, $(filter-out +simd, $(v9_a_simd_variants)), \
+			     march?armv9-a+simd=march?armv9-a$(ARCH) \
+			     march?armv9-a+simd=march?armv9-a+crc$(ARCH))
+
 # Use Thumb libraries for everything.
 
 MULTILIB_REUSE		+= mthumb/march.armv7-a/mfloat-abi.soft=marm/march.armv7-a/mfloat-abi.soft
 
 MULTILIB_REUSE		+= mthumb/march.armv8-a/mfloat-abi.soft=marm/march.armv8-a/mfloat-abi.soft
 
+MULTILIB_REUSE		+= mthumb/march.armv9-a/mfloat-abi.soft=marm/march.armv9-a/mfloat-abi.soft
+
 MULTILIB_REUSE		+= $(foreach ABI, hard softfp, \
-			     $(foreach ARCH, armv7-a+fp armv7-a+simd armv7ve+simd armv8-a+simd, \
+			     $(foreach ARCH, armv7-a+fp armv7-a+simd armv7ve+simd armv8-a+simd armv9-a+simd, \
 			       mthumb/march.$(ARCH)/mfloat-abi.$(ABI)=marm/march.$(ARCH)/mfloat-abi.$(ABI)))
 
 # Softfp but no FP, use the soft-float libraries.
 MULTILIB_REUSE		+= $(foreach MODE, arm thumb, \
-			     $(foreach ARCH, armv7-a armv8-a, \
+			     $(foreach ARCH, armv7-a armv8-a armv9-a, \
 			       mthumb/march.$(ARCH)/mfloat-abi.soft=m$(MODE)/march.$(ARCH)/mfloat-abi.softfp))
diff --git a/gcc/config/arm/t-arm-elf b/gcc/config/arm/t-arm-elf
index d68def3..b3a900e 100644
--- a/gcc/config/arm/t-arm-elf
+++ b/gcc/config/arm/t-arm-elf
@@ -38,6 +38,8 @@ v7ve_fps	:= vfpv3-d16 vfpv3 vfpv3-d16-fp16 vfpv3-fp16 vfpv4 neon \
 # it seems to work ok.
 v8_fps		:= simd fp16 crypto fp16+crypto dotprod fp16fml
 
+v9_fps		:= simd fp16 crypto fp16+crypto dotprod fp16fml
+
 # We don't do anything special with these.  Pre-v4t probably doesn't work.
 all_early_nofp	:= armv4 armv4t armv5t
 
@@ -49,6 +51,8 @@ all_v7_a_r	:= armv7-a armv7ve armv7-r
 all_v8_archs	:= armv8-a armv8-a+crc armv8.1-a armv8.2-a armv8.3-a armv8.4-a \
 		   armv8.5-a armv8.6-a
 
+all_v9_archs	:= armv9-a
+
 # No floating point variants, require thumb1 softfp
 all_nofp_t	:= armv6-m armv6s-m armv8-m.base
 
@@ -110,6 +114,11 @@ MULTILIB_MATCHES     += $(foreach ARCH, $(all_v8_archs), \
 			  $(foreach FPARCH, $(v8_fps), \
 			    march?armv7+fp=march?$(ARCH)+$(FPARCH)))
 
+MULTILIB_MATCHES     += $(foreach ARCH, $(all_v9_archs), \
+			  march?armv7+fp=march?$(ARCH) \
+			  $(foreach FPARCH, $(v9_fps), \
+			    march?armv7+fp=march?$(ARCH)+$(FPARCH)))
+
 MULTILIB_MATCHES     += $(foreach ARCH, armv7e-m armv8-m.mainline, \
 			  march?armv7+fp=march?$(ARCH)+fp.dp)
 
diff --git a/gcc/config/arm/t-multilib b/gcc/config/arm/t-multilib
index ddc5033..d789b86 100644
--- a/gcc/config/arm/t-multilib
+++ b/gcc/config/arm/t-multilib
@@ -78,6 +78,8 @@ v8_4_a_simd_variants	:= $(call all_feat_combs, simd fp16 crypto i8mm bf16)
 v8_5_a_simd_variants	:= $(call all_feat_combs, simd fp16 crypto i8mm bf16)
 v8_6_a_simd_variants	:= $(call all_feat_combs, simd fp16 crypto i8mm bf16)
 v8_r_nosimd_variants	:= +crc
+v9_a_nosimd_variants	:= +crc
+v9_a_simd_variants	:= $(call all_feat_combs, simd fp16 crypto i8mm bf16)
 
 ifneq (,$(HAS_APROFILE))
 include $(srcdir)/config/arm/t-aprofile
@@ -202,6 +204,16 @@ MULTILIB_MATCHES	+= march?armv7=march?armv8.6-a
 MULTILIB_MATCHES	+= $(foreach ARCH, $(v8_6_a_simd_variants), \
 			     march?armv7+fp=march?armv8.6-a$(ARCH))
 
+# Armv9
+MULTILIB_MATCHES	+= march?armv7=march?armv9-a
+MULTILIB_MATCHES	+= $(foreach ARCH, $(v9_a_nosimd_variants), \
+			     march?armv7=march?armv9-a$(ARCH))
+
+# Armv9 with SIMD
+MULTILIB_MATCHES	+= march?armv7+fp=march?armv9-a+crc+simd \
+			   $(foreach ARCH, $(v9_a_simd_variants), \
+			     march?armv7+fp=march?armv9-a$(ARCH) \
+			     march?armv7+fp=march?armv9-a+crc$(ARCH))
 endif		# Not APROFILE.
 
 # Use Thumb libraries for everything.
diff --git a/gcc/config/darwin-c.c b/gcc/config/darwin-c.c
index 951a998..62d28fc 100644
--- a/gcc/config/darwin-c.c
+++ b/gcc/config/darwin-c.c
@@ -691,7 +691,7 @@ macosx_version_as_macro (void)
   if (!version_array)
     goto fail;
 
-  if (version_array[MAJOR] < 10 || version_array[MAJOR] > 11)
+  if (version_array[MAJOR] < 10 || version_array[MAJOR] > 12)
     goto fail;
 
   if (version_array[MAJOR] == 10 && version_array[MINOR] < 10)
diff --git a/gcc/config/darwin-driver.c b/gcc/config/darwin-driver.c
index a036e09..4f0c6ba 100644
--- a/gcc/config/darwin-driver.c
+++ b/gcc/config/darwin-driver.c
@@ -64,17 +64,17 @@ validate_macosx_version_min (const char *version_str)
 
   major = strtoul (version_str, &end, 10);
 
-  if (major < 10 || major > 11 ) /* MacOS 10 and 11 are known. */
+  if (major < 10 || major > 12 ) /* macOS 10, 11, and 12 are known. */
     return NULL;
 
   /* Skip a separating period, if there's one.  */
   version_str = end + ((*end == '.') ? 1 : 0);
 
-  if (major == 11 && *end != '\0' && !ISDIGIT (version_str[0]))
-     /* For MacOS 11, we allow just the major number, but if the minor is
+  if (major > 10 && *end != '\0' && !ISDIGIT (version_str[0]))
+     /* For macOS 11+, we allow just the major number, but if the minor is
 	there it must be numeric.  */
     return NULL;
-  else if (major == 11 && *end == '\0')
+  else if (major > 10 && *end == '\0')
     /* We will rewrite 11 =>  11.0.0.  */
     need_rewrite = true;
   else if (major == 10 && (*end == '\0' || !ISDIGIT (version_str[0])))
@@ -172,7 +172,7 @@ darwin_find_version_from_kernel (void)
       if (minor_vers > 0)
 	minor_vers -= 1; /* Kernel 20.3 => macOS 11.2.  */
       /* It's not yet clear whether patch level will be considered.  */
-      asprintf (&new_flag, "11.%02d.00", minor_vers);
+      asprintf (&new_flag, "%d.%02d.00", major_vers - 9, minor_vers);
     }
   else if (major_vers - 4 <= 4)
     /* On 10.4 and earlier, the old linker is used which does not
diff --git a/gcc/config/darwin.h b/gcc/config/darwin.h
index 27cb3e4..7ed01ef 100644
--- a/gcc/config/darwin.h
+++ b/gcc/config/darwin.h
@@ -54,6 +54,11 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 
 #define DO_GLOBAL_DTORS_BODY
 
+/* Register static destructors to run from __cxa_atexit instead of putting
+   them into a .mod_term_funcs section.  */
+
+#define TARGET_DTORS_FROM_CXA_ATEXIT true
+
 /* The string value for __SIZE_TYPE__.  */
 
 #ifndef SIZE_TYPE
@@ -921,6 +926,8 @@ extern GTY(()) section * darwin_sections[NUM_DARWIN_SECTIONS];
       sprintf (LABEL, "*%s%ld", "lubsan_type", (long)(NUM));\
     else if (strcmp ("LASAN", PREFIX) == 0)	\
       sprintf (LABEL, "*%s%ld", "lASAN", (long)(NUM));\
+    else if (strcmp ("LTRAMP", PREFIX) == 0)	\
+      sprintf (LABEL, "*%s%ld", "lTRAMP", (long)(NUM));\
     else						\
       sprintf (LABEL, "*%s%ld", PREFIX, (long)(NUM));	\
   } while (0)
diff --git a/gcc/config/i386/amxtileintrin.h b/gcc/config/i386/amxtileintrin.h
index 75d784a..3a0a6b4 100644
--- a/gcc/config/i386/amxtileintrin.h
+++ b/gcc/config/i386/amxtileintrin.h
@@ -62,7 +62,7 @@ _tile_release (void)
 #define _tile_loadd_internal(dst,base,stride)				\
   __asm__ volatile							\
   ("{tileloadd\t(%0,%1,1), %%tmm"#dst"|tileloadd\t%%tmm"#dst", [%0+%1*1]}" \
-   :: "r" ((const void*) base), "r" ((long) stride))
+   :: "r" ((const void*) (base)), "r" ((long) (stride)))
 
 #define _tile_stream_loadd(dst,base,stride)		\
   _tile_stream_loadd_internal (dst, base, stride)
@@ -70,7 +70,7 @@ _tile_release (void)
 #define _tile_stream_loadd_internal(dst,base,stride)			\
   __asm__ volatile							\
   ("{tileloaddt1\t(%0,%1,1), %%tmm"#dst"|tileloaddt1\t%%tmm"#dst", [%0+%1*1]}" \
-   :: "r" ((const void*) base), "r" ((long) stride))
+   :: "r" ((const void*) (base)), "r" ((long) (stride)))
 
 #define _tile_stored(dst,base,stride)		\
   _tile_stored_internal (dst, base, stride)
@@ -78,7 +78,7 @@ _tile_release (void)
 #define _tile_stored_internal(src,base,stride)				\
   __asm__ volatile							\
   ("{tilestored\t%%tmm"#src", (%0,%1,1)|tilestored\t[%0+%1*1], %%tmm"#src"}" \
-   :: "r" ((void*) base), "r" ((long) stride) \
+   :: "r" ((void*) (base)), "r" ((long) (stride)) \
    : "memory")
 
 #define _tile_zero(dst)				\
diff --git a/gcc/config/i386/i386-builtin-types.def b/gcc/config/i386/i386-builtin-types.def
index 4c355c5..e33f06a 100644
--- a/gcc/config/i386/i386-builtin-types.def
+++ b/gcc/config/i386/i386-builtin-types.def
@@ -1380,3 +1380,10 @@ DEF_FUNCTION_TYPE (USI, V32HF, V32HF, INT, USI, INT)
 DEF_FUNCTION_TYPE (V32HF, V32HF, V32HF, V32HF, UHI, INT)
 DEF_FUNCTION_TYPE (V32HF, V32HF, V32HF, V32HF, USI, INT)
 DEF_FUNCTION_TYPE (V32HF, V32HF, INT, V32HF, USI, INT)
+
+DEF_FUNCTION_TYPE (V8HF, V8HF)
+DEF_FUNCTION_TYPE (V16HF, V16HF)
+DEF_FUNCTION_TYPE (V32HF, V32HF)
+DEF_FUNCTION_TYPE_ALIAS (V8HF_FTYPE_V8HF, ROUND)
+DEF_FUNCTION_TYPE_ALIAS (V16HF_FTYPE_V16HF, ROUND)
+DEF_FUNCTION_TYPE_ALIAS (V32HF_FTYPE_V32HF, ROUND)
diff --git a/gcc/config/i386/i386-builtin.def b/gcc/config/i386/i386-builtin.def
index 99217d0..d9eee3f 100644
--- a/gcc/config/i386/i386-builtin.def
+++ b/gcc/config/i386/i386-builtin.def
@@ -958,6 +958,10 @@ BDESC (OPTION_MASK_ISA_SSE4_1, 0, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__buil
 BDESC (OPTION_MASK_ISA_SSE4_1, 0, CODE_FOR_roundv2df2, "__builtin_ia32_roundpd_az", IX86_BUILTIN_ROUNDPD_AZ, UNKNOWN, (int) V2DF_FTYPE_V2DF)
 BDESC (OPTION_MASK_ISA_SSE4_1, 0, CODE_FOR_roundv2df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF)
 
+BDESC (OPTION_MASK_ISA_AVX512VL, OPTION_MASK_ISA2_AVX512FP16, CODE_FOR_avx512fp16_rndscalev8hf, "__builtin_ia32_floorph", IX86_BUILTIN_FLOORPH, (enum rtx_code) ROUND_FLOOR, (int) V8HF_FTYPE_V8HF_ROUND)
+BDESC (OPTION_MASK_ISA_AVX512VL, OPTION_MASK_ISA2_AVX512FP16, CODE_FOR_avx512fp16_rndscalev8hf, "__builtin_ia32_ceilph", IX86_BUILTIN_CEILPH, (enum rtx_code) ROUND_CEIL, (int) V8HF_FTYPE_V8HF_ROUND)
+BDESC (OPTION_MASK_ISA_AVX512VL, OPTION_MASK_ISA2_AVX512FP16, CODE_FOR_avx512fp16_rndscalev8hf, "__builtin_ia32_truncph", IX86_BUILTIN_TRUNCPH, (enum rtx_code) ROUND_TRUNC, (int) V8HF_FTYPE_V8HF_ROUND)
+
 BDESC (OPTION_MASK_ISA_SSE4_1, 0, CODE_FOR_sse4_1_roundps, "__builtin_ia32_floorps", IX86_BUILTIN_FLOORPS, (enum rtx_code) ROUND_FLOOR, (int) V4SF_FTYPE_V4SF_ROUND)
 BDESC (OPTION_MASK_ISA_SSE4_1, 0, CODE_FOR_sse4_1_roundps, "__builtin_ia32_ceilps", IX86_BUILTIN_CEILPS, (enum rtx_code) ROUND_CEIL, (int) V4SF_FTYPE_V4SF_ROUND)
 BDESC (OPTION_MASK_ISA_SSE4_1, 0, CODE_FOR_sse4_1_roundps, "__builtin_ia32_truncps", IX86_BUILTIN_TRUNCPS, (enum rtx_code) ROUND_TRUNC, (int) V4SF_FTYPE_V4SF_ROUND)
@@ -1090,6 +1094,10 @@ BDESC (OPTION_MASK_ISA_AVX, 0, CODE_FOR_roundv4df2_vec_pack_sfix, "__builtin_ia3
 BDESC (OPTION_MASK_ISA_AVX, 0, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_floorpd_vec_pack_sfix256", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V4DF_V4DF_ROUND)
 BDESC (OPTION_MASK_ISA_AVX, 0, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_ceilpd_vec_pack_sfix256", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V4DF_V4DF_ROUND)
 
+BDESC (OPTION_MASK_ISA_AVX512VL, OPTION_MASK_ISA2_AVX512FP16, CODE_FOR_avx512vl_rndscalev16hf, "__builtin_ia32_floorph256", IX86_BUILTIN_FLOORPH256, (enum rtx_code) ROUND_FLOOR, (int) V16HF_FTYPE_V16HF_ROUND)
+BDESC (OPTION_MASK_ISA_AVX512VL, OPTION_MASK_ISA2_AVX512FP16, CODE_FOR_avx512vl_rndscalev16hf, "__builtin_ia32_ceilph256", IX86_BUILTIN_CEILPH256, (enum rtx_code) ROUND_CEIL, (int) V16HF_FTYPE_V16HF_ROUND)
+BDESC (OPTION_MASK_ISA_AVX512VL, OPTION_MASK_ISA2_AVX512FP16, CODE_FOR_avx512vl_rndscalev16hf, "__builtin_ia32_truncph256", IX86_BUILTIN_TRUNCPH256, (enum rtx_code) ROUND_TRUNC, (int) V16HF_FTYPE_V16HF_ROUND)
+
 BDESC (OPTION_MASK_ISA_AVX, 0, CODE_FOR_avx_roundps256, "__builtin_ia32_floorps256", IX86_BUILTIN_FLOORPS256, (enum rtx_code) ROUND_FLOOR, (int) V8SF_FTYPE_V8SF_ROUND)
 BDESC (OPTION_MASK_ISA_AVX, 0, CODE_FOR_avx_roundps256, "__builtin_ia32_ceilps256", IX86_BUILTIN_CEILPS256, (enum rtx_code) ROUND_CEIL, (int) V8SF_FTYPE_V8SF_ROUND)
 BDESC (OPTION_MASK_ISA_AVX, 0, CODE_FOR_avx_roundps256, "__builtin_ia32_truncps256", IX86_BUILTIN_TRUNCPS256, (enum rtx_code) ROUND_TRUNC, (int) V8SF_FTYPE_V8SF_ROUND)
@@ -1528,6 +1536,9 @@ BDESC (OPTION_MASK_ISA_AVX512F, 0, CODE_FOR_copysignv8df3,  "__builtin_ia32_copy
 BDESC (OPTION_MASK_ISA_AVX512F, 0, CODE_FOR_avx512f_sqrtv8df2, "__builtin_ia32_sqrtpd512", IX86_BUILTIN_SQRTPD512, UNKNOWN, (int) V8DF_FTYPE_V8DF)
 BDESC (OPTION_MASK_ISA_AVX512F, 0, CODE_FOR_sqrtv16sf2, "__builtin_ia32_sqrtps512", IX86_BUILTIN_SQRTPS_NR512, UNKNOWN, (int) V16SF_FTYPE_V16SF)
 BDESC (OPTION_MASK_ISA_AVX512ER, 0, CODE_FOR_avx512er_exp2v16sf, "__builtin_ia32_exp2ps", IX86_BUILTIN_EXP2PS, UNKNOWN, (int) V16SF_FTYPE_V16SF)
+BDESC (0, OPTION_MASK_ISA2_AVX512FP16, CODE_FOR_avx512bw_rndscalev32hf, "__builtin_ia32_floorph512", IX86_BUILTIN_FLOORPH512, (enum rtx_code) ROUND_FLOOR, (int) V32HF_FTYPE_V32HF_ROUND)
+BDESC (0, OPTION_MASK_ISA2_AVX512FP16, CODE_FOR_avx512bw_rndscalev32hf, "__builtin_ia32_ceilph512", IX86_BUILTIN_CEILPH512, (enum rtx_code) ROUND_CEIL, (int) V32HF_FTYPE_V32HF_ROUND)
+BDESC (0, OPTION_MASK_ISA2_AVX512FP16, CODE_FOR_avx512bw_rndscalev32hf, "__builtin_ia32_truncph512", IX86_BUILTIN_TRUNCPH512, (enum rtx_code) ROUND_TRUNC, (int) V32HF_FTYPE_V32HF_ROUND)
 BDESC (OPTION_MASK_ISA_AVX512F, 0, CODE_FOR_avx512f_roundps512, "__builtin_ia32_floorps512", IX86_BUILTIN_FLOORPS512, (enum rtx_code) ROUND_FLOOR, (int) V16SF_FTYPE_V16SF_ROUND)
 BDESC (OPTION_MASK_ISA_AVX512F, 0, CODE_FOR_avx512f_roundps512, "__builtin_ia32_ceilps512", IX86_BUILTIN_CEILPS512, (enum rtx_code) ROUND_CEIL, (int) V16SF_FTYPE_V16SF_ROUND)
 BDESC (OPTION_MASK_ISA_AVX512F, 0, CODE_FOR_avx512f_roundps512, "__builtin_ia32_truncps512", IX86_BUILTIN_TRUNCPS512, (enum rtx_code) ROUND_TRUNC, (int) V16SF_FTYPE_V16SF_ROUND)
diff --git a/gcc/config/i386/i386-builtins.c b/gcc/config/i386/i386-builtins.c
index 11ce58b..0fb14b5 100644
--- a/gcc/config/i386/i386-builtins.c
+++ b/gcc/config/i386/i386-builtins.c
@@ -1652,6 +1652,20 @@ ix86_builtin_vectorized_function (unsigned int fn, tree type_out,
 	  else if (out_n == 16 && in_n == 16)
 	    return ix86_get_builtin (IX86_BUILTIN_FLOORPS512);
 	}
+      if (out_mode == HFmode && in_mode == HFmode)
+	{
+	  /* V8HF/V16HF is supported in ix86_vector_mode_supported_p
+	     under TARGET_AVX512FP16, TARGET_AVX512VL is needed here.  */
+	  if (out_n < 32 && !TARGET_AVX512VL)
+	    break;
+
+	  if (out_n == 8 && in_n == 8)
+	    return ix86_get_builtin (IX86_BUILTIN_FLOORPH);
+	  else if (out_n == 16 && in_n == 16)
+	    return ix86_get_builtin (IX86_BUILTIN_FLOORPH256);
+	  else if (out_n == 32 && in_n == 32)
+	    return ix86_get_builtin (IX86_BUILTIN_FLOORPH512);
+	}
       break;
 
     CASE_CFN_CEIL:
@@ -1677,6 +1691,20 @@ ix86_builtin_vectorized_function (unsigned int fn, tree type_out,
 	  else if (out_n == 16 && in_n == 16)
 	    return ix86_get_builtin (IX86_BUILTIN_CEILPS512);
 	}
+      if (out_mode == HFmode && in_mode == HFmode)
+	{
+	  /* V8HF/V16HF is supported in ix86_vector_mode_supported_p
+	     under TARGET_AVX512FP16, TARGET_AVX512VL is needed here.  */
+	  if (out_n < 32 && !TARGET_AVX512VL)
+	    break;
+
+	  if (out_n == 8 && in_n == 8)
+	    return ix86_get_builtin (IX86_BUILTIN_CEILPH);
+	  else if (out_n == 16 && in_n == 16)
+	    return ix86_get_builtin (IX86_BUILTIN_CEILPH256);
+	  else if (out_n == 32 && in_n == 32)
+	    return ix86_get_builtin (IX86_BUILTIN_CEILPH512);
+	}
       break;
 
     CASE_CFN_TRUNC:
@@ -1702,6 +1730,20 @@ ix86_builtin_vectorized_function (unsigned int fn, tree type_out,
 	  else if (out_n == 16 && in_n == 16)
 	    return ix86_get_builtin (IX86_BUILTIN_TRUNCPS512);
 	}
+      if (out_mode == HFmode && in_mode == HFmode)
+	{
+	  /* V8HF/V16HF is supported in ix86_vector_mode_supported_p
+	     under TARGET_AVX512FP16, TARGET_AVX512VL is needed here.  */
+	  if (out_n < 32 && !TARGET_AVX512VL)
+	    break;
+
+	  if (out_n == 8 && in_n == 8)
+	    return ix86_get_builtin (IX86_BUILTIN_TRUNCPH);
+	  else if (out_n == 16 && in_n == 16)
+	    return ix86_get_builtin (IX86_BUILTIN_TRUNCPH256);
+	  else if (out_n == 32 && in_n == 32)
+	    return ix86_get_builtin (IX86_BUILTIN_TRUNCPH512);
+	}
       break;
 
     CASE_CFN_FMA:
diff --git a/gcc/config/i386/i386-expand.c b/gcc/config/i386/i386-expand.c
index 56dd99b..3e4de64 100644
--- a/gcc/config/i386/i386-expand.c
+++ b/gcc/config/i386/i386-expand.c
@@ -6157,6 +6157,687 @@ ix86_split_lshr (rtx *operands, rtx scratch, machine_mode mode)
     }
 }
 
+/* Expand move of V1TI mode register X to a new TI mode register.  */
+static rtx
+ix86_expand_v1ti_to_ti (rtx x)
+{
+  rtx result = gen_reg_rtx (TImode);
+  emit_move_insn (result, gen_lowpart (TImode, x));
+  return result;
+}
+
+/* Expand move of TI mode register X to a new V1TI mode register.  */
+static rtx
+ix86_expand_ti_to_v1ti (rtx x)
+{
+  rtx result = gen_reg_rtx (V1TImode);
+  if (TARGET_SSE2)
+    {
+      rtx lo = gen_lowpart (DImode, x);
+      rtx hi = gen_highpart (DImode, x);
+      rtx tmp = gen_reg_rtx (V2DImode);
+      emit_insn (gen_vec_concatv2di (tmp, lo, hi));
+      emit_move_insn (result, gen_lowpart (V1TImode, tmp));
+    }
+  else
+    emit_move_insn (result, gen_lowpart (V1TImode, x));
+  return result;
+}
+
+/* Expand V1TI mode shift (of rtx_code CODE) by constant.  */
+void
+ix86_expand_v1ti_shift (enum rtx_code code, rtx operands[])
+{
+  rtx op1 = force_reg (V1TImode, operands[1]);
+
+  if (!CONST_INT_P (operands[2]))
+    {
+      rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
+      rtx tmp2 = gen_reg_rtx (TImode);
+      rtx (*shift) (rtx, rtx, rtx)
+	    = (code == ASHIFT) ? gen_ashlti3 : gen_lshrti3;
+      emit_insn (shift (tmp2, tmp1, operands[2]));
+      rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
+      emit_move_insn (operands[0], tmp3);
+      return;
+    }
+
+  HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
+
+  if (bits == 0)
+    {
+      emit_move_insn (operands[0], op1);
+      return;
+    }
+
+  if ((bits & 7) == 0)
+    {
+      rtx tmp = gen_reg_rtx (V1TImode);
+      if (code == ASHIFT)
+	emit_insn (gen_sse2_ashlv1ti3 (tmp, op1, GEN_INT (bits)));
+      else
+	emit_insn (gen_sse2_lshrv1ti3 (tmp, op1, GEN_INT (bits)));
+      emit_move_insn (operands[0], tmp);
+      return;
+    }
+
+  rtx tmp1 = gen_reg_rtx (V1TImode);
+  if (code == ASHIFT)
+    emit_insn (gen_sse2_ashlv1ti3 (tmp1, op1, GEN_INT (64)));
+  else
+    emit_insn (gen_sse2_lshrv1ti3 (tmp1, op1, GEN_INT (64)));
+
+  /* tmp2 is operands[1] shifted by 64, in V2DImode.  */
+  rtx tmp2 = gen_reg_rtx (V2DImode);
+  emit_move_insn (tmp2, gen_lowpart (V2DImode, tmp1));
+
+  /* tmp3 will be the V2DImode result.  */
+  rtx tmp3 = gen_reg_rtx (V2DImode);
+
+  if (bits > 64)
+    {
+      if (code == ASHIFT)
+	emit_insn (gen_ashlv2di3 (tmp3, tmp2, GEN_INT (bits - 64)));
+      else
+	emit_insn (gen_lshrv2di3 (tmp3, tmp2, GEN_INT (bits - 64)));
+    }
+  else
+    {
+      /* tmp4 is operands[1], in V2DImode.  */
+      rtx tmp4 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp4, gen_lowpart (V2DImode, op1));
+
+      rtx tmp5 = gen_reg_rtx (V2DImode);
+      if (code == ASHIFT)
+	emit_insn (gen_ashlv2di3 (tmp5, tmp4, GEN_INT (bits)));
+      else
+	emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
+
+      rtx tmp6 = gen_reg_rtx (V2DImode);
+      if (code == ASHIFT)
+	emit_insn (gen_lshrv2di3 (tmp6, tmp2, GEN_INT (64 - bits)));
+      else
+	emit_insn (gen_ashlv2di3 (tmp6, tmp2, GEN_INT (64 - bits)));
+
+      emit_insn (gen_iorv2di3 (tmp3, tmp5, tmp6));
+    }
+
+  /* Convert the result back to V1TImode and store in operands[0].  */
+  rtx tmp7 = gen_reg_rtx (V1TImode);
+  emit_move_insn (tmp7, gen_lowpart (V1TImode, tmp3));
+  emit_move_insn (operands[0], tmp7);
+}
+
+/* Expand V1TI mode rotate (of rtx_code CODE) by constant.  */
+void
+ix86_expand_v1ti_rotate (enum rtx_code code, rtx operands[])
+{
+  rtx op1 = force_reg (V1TImode, operands[1]);
+
+  if (!CONST_INT_P (operands[2]))
+    {
+      rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
+      rtx tmp2 = gen_reg_rtx (TImode);
+      rtx (*rotate) (rtx, rtx, rtx)
+	    = (code == ROTATE) ? gen_rotlti3 : gen_rotrti3;
+      emit_insn (rotate (tmp2, tmp1, operands[2]));
+      rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
+      emit_move_insn (operands[0], tmp3);
+      return;
+    }
+
+  HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
+
+  if (bits == 0)
+    {
+      emit_move_insn (operands[0], op1);
+      return;
+    }
+
+  if (code == ROTATERT)
+    bits = 128 - bits;
+
+  if ((bits & 31) == 0)
+    {
+      rtx tmp1 = gen_reg_rtx (V4SImode);
+      rtx tmp2 = gen_reg_rtx (V4SImode);
+      rtx tmp3 = gen_reg_rtx (V1TImode);
+
+      emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+      if (bits == 32)
+	emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x93)));
+      else if (bits == 64)
+	emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x4e)));
+      else
+	emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0x39)));
+      emit_move_insn (tmp3, gen_lowpart (V1TImode, tmp2));
+      emit_move_insn (operands[0], tmp3);
+      return;
+    }
+
+  if ((bits & 7) == 0)
+    {
+      rtx tmp1 = gen_reg_rtx (V1TImode);
+      rtx tmp2 = gen_reg_rtx (V1TImode);
+      rtx tmp3 = gen_reg_rtx (V1TImode);
+
+      emit_insn (gen_sse2_ashlv1ti3 (tmp1, op1, GEN_INT (bits)));
+      emit_insn (gen_sse2_lshrv1ti3 (tmp2, op1, GEN_INT (128 - bits)));
+      emit_insn (gen_iorv1ti3 (tmp3, tmp1, tmp2));
+      emit_move_insn (operands[0], tmp3);
+      return;
+    }
+
+  rtx op1_v4si = gen_reg_rtx (V4SImode);
+  emit_move_insn (op1_v4si, gen_lowpart (V4SImode, op1));
+
+  rtx lobits;
+  rtx hibits;
+
+  switch (bits >> 5)
+    {
+    case 0:
+      lobits = op1_v4si;
+      hibits = gen_reg_rtx (V4SImode);
+      emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x93)));
+      break;
+
+    case 1:
+      lobits = gen_reg_rtx (V4SImode);
+      hibits = gen_reg_rtx (V4SImode);
+      emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x93)));
+      emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x4e)));
+      break;
+
+    case 2:
+      lobits = gen_reg_rtx (V4SImode);
+      hibits = gen_reg_rtx (V4SImode);
+      emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x4e)));
+      emit_insn (gen_sse2_pshufd (hibits, op1_v4si, GEN_INT (0x39)));
+      break;
+
+    default:
+      lobits = gen_reg_rtx (V4SImode);
+      emit_insn (gen_sse2_pshufd (lobits, op1_v4si, GEN_INT (0x39)));
+      hibits = op1_v4si;
+      break;
+    }
+
+  rtx tmp1 = gen_reg_rtx (V4SImode);
+  rtx tmp2 = gen_reg_rtx (V4SImode);
+  rtx tmp3 = gen_reg_rtx (V4SImode);
+  rtx tmp4 = gen_reg_rtx (V1TImode);
+
+  emit_insn (gen_ashlv4si3 (tmp1, lobits, GEN_INT (bits & 31)));
+  emit_insn (gen_lshrv4si3 (tmp2, hibits, GEN_INT (32 - (bits & 31))));
+  emit_insn (gen_iorv4si3 (tmp3, tmp1, tmp2));
+  emit_move_insn (tmp4, gen_lowpart (V1TImode, tmp3));
+  emit_move_insn (operands[0], tmp4);
+}
+
+/* Expand V1TI mode ashiftrt by constant.  */
+void
+ix86_expand_v1ti_ashiftrt (rtx operands[])
+{
+  rtx op1 = force_reg (V1TImode, operands[1]);
+
+  if (!CONST_INT_P (operands[2]))
+    {
+      rtx tmp1 = ix86_expand_v1ti_to_ti (op1);
+      rtx tmp2 = gen_reg_rtx (TImode);
+      emit_insn (gen_ashrti3 (tmp2, tmp1, operands[2]));
+      rtx tmp3 = ix86_expand_ti_to_v1ti (tmp2);
+      emit_move_insn (operands[0], tmp3);
+      return;
+    }
+
+  HOST_WIDE_INT bits = INTVAL (operands[2]) & 127;
+
+  if (bits == 0)
+    {
+      emit_move_insn (operands[0], op1);
+      return;
+    }
+
+  if (bits == 127)
+    {
+      /* Two operations.  */
+      rtx tmp1 = gen_reg_rtx (V4SImode);
+      rtx tmp2 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+      emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
+
+      rtx tmp3 = gen_reg_rtx (V4SImode);
+      emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
+
+      rtx tmp4 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp4, gen_lowpart (V1TImode, tmp3));
+      emit_move_insn (operands[0], tmp4);
+      return;
+    }
+
+  if (bits == 64)
+    {
+      /* Three operations.  */
+      rtx tmp1 = gen_reg_rtx (V4SImode);
+      rtx tmp2 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+      emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
+
+      rtx tmp3 = gen_reg_rtx (V4SImode);
+      emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
+
+      rtx tmp4 = gen_reg_rtx (V2DImode);
+      rtx tmp5 = gen_reg_rtx (V2DImode);
+      rtx tmp6 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp4, gen_lowpart (V2DImode, tmp1));
+      emit_move_insn (tmp5, gen_lowpart (V2DImode, tmp3));
+      emit_insn (gen_vec_interleave_highv2di (tmp6, tmp4, tmp5));
+
+      rtx tmp7 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp7, gen_lowpart (V1TImode, tmp6));
+      emit_move_insn (operands[0], tmp7);
+      return;
+    }
+
+  if (bits == 96)
+    {
+      /* Three operations.  */
+      rtx tmp3 = gen_reg_rtx (V2DImode);
+      rtx tmp1 = gen_reg_rtx (V4SImode);
+      rtx tmp2 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+      emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (31)));
+
+      rtx tmp4 = gen_reg_rtx (V2DImode);
+      rtx tmp5 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp3, gen_lowpart (V2DImode, tmp1));
+      emit_move_insn (tmp4, gen_lowpart (V2DImode, tmp2));
+      emit_insn (gen_vec_interleave_highv2di (tmp5, tmp3, tmp4));
+
+      rtx tmp6 = gen_reg_rtx (V4SImode);
+      rtx tmp7 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp6, gen_lowpart (V4SImode, tmp5));
+      emit_insn (gen_sse2_pshufd (tmp7, tmp6, GEN_INT (0xfd)));
+
+      rtx tmp8 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp8, gen_lowpart (V1TImode, tmp7));
+      emit_move_insn (operands[0], tmp8);
+      return;
+    }
+
+  if (TARGET_AVX2 || TARGET_SSE4_1)
+    {
+      /* Three operations.  */
+      if (bits == 32)
+	{
+	  rtx tmp1 = gen_reg_rtx (V4SImode);
+	  rtx tmp2 = gen_reg_rtx (V4SImode);
+	  emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+	  emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (31)));
+
+	  rtx tmp3 = gen_reg_rtx (V1TImode);
+	  emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (32)));
+
+	  if (TARGET_AVX2)
+	    {
+	      rtx tmp4 = gen_reg_rtx (V4SImode);
+	      rtx tmp5 = gen_reg_rtx (V4SImode);
+	      emit_move_insn (tmp4, gen_lowpart (V4SImode, tmp3));
+	      emit_insn (gen_avx2_pblenddv4si (tmp5, tmp2, tmp4,
+					       GEN_INT (7)));
+
+	      rtx tmp6 = gen_reg_rtx (V1TImode);
+	      emit_move_insn (tmp6, gen_lowpart (V1TImode, tmp5));
+	      emit_move_insn (operands[0], tmp6);
+	    }
+	  else
+	    {
+	      rtx tmp4 = gen_reg_rtx (V8HImode);
+	      rtx tmp5 = gen_reg_rtx (V8HImode);
+	      rtx tmp6 = gen_reg_rtx (V8HImode);
+	      emit_move_insn (tmp4, gen_lowpart (V8HImode, tmp2));
+	      emit_move_insn (tmp5, gen_lowpart (V8HImode, tmp3));
+	      emit_insn (gen_sse4_1_pblendw (tmp6, tmp4, tmp5,
+					     GEN_INT (0x3f)));
+
+	      rtx tmp7 = gen_reg_rtx (V1TImode);
+	      emit_move_insn (tmp7, gen_lowpart (V1TImode, tmp6));
+	      emit_move_insn (operands[0], tmp7);
+	    }
+	  return;
+	}
+
+      /* Three operations.  */
+      if (bits == 8 || bits == 16 || bits == 24)
+	{
+	  rtx tmp1 = gen_reg_rtx (V4SImode);
+	  rtx tmp2 = gen_reg_rtx (V4SImode);
+	  emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+	  emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
+
+	  rtx tmp3 = gen_reg_rtx (V1TImode);
+	  emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (bits)));
+
+	  if (TARGET_AVX2)
+	    {
+	      rtx tmp4 = gen_reg_rtx (V4SImode);
+	      rtx tmp5 = gen_reg_rtx (V4SImode);
+	      emit_move_insn (tmp4, gen_lowpart (V4SImode, tmp3));
+	      emit_insn (gen_avx2_pblenddv4si (tmp5, tmp2, tmp4,
+					       GEN_INT (7)));
+
+	      rtx tmp6 = gen_reg_rtx (V1TImode);
+	      emit_move_insn (tmp6, gen_lowpart (V1TImode, tmp5));
+	      emit_move_insn (operands[0], tmp6);
+	    }
+	  else
+	    {
+	      rtx tmp4 = gen_reg_rtx (V8HImode);
+	      rtx tmp5 = gen_reg_rtx (V8HImode);
+	      rtx tmp6 = gen_reg_rtx (V8HImode);
+	      emit_move_insn (tmp4, gen_lowpart (V8HImode, tmp2));
+	      emit_move_insn (tmp5, gen_lowpart (V8HImode, tmp3));
+	      emit_insn (gen_sse4_1_pblendw (tmp6, tmp4, tmp5,
+					     GEN_INT (0x3f)));
+
+	      rtx tmp7 = gen_reg_rtx (V1TImode);
+	      emit_move_insn (tmp7, gen_lowpart (V1TImode, tmp6));
+	      emit_move_insn (operands[0], tmp7);
+	    }
+	  return;
+	}
+    }
+
+  if (bits > 96)
+    {
+      /* Four operations.  */
+      rtx tmp1 = gen_reg_rtx (V4SImode);
+      rtx tmp2 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+      emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits - 96)));
+
+      rtx tmp3 = gen_reg_rtx (V4SImode);
+      emit_insn (gen_ashrv4si3 (tmp3, tmp1, GEN_INT (31)));
+
+      rtx tmp4 = gen_reg_rtx (V2DImode);
+      rtx tmp5 = gen_reg_rtx (V2DImode);
+      rtx tmp6 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp4, gen_lowpart (V2DImode, tmp2));
+      emit_move_insn (tmp5, gen_lowpart (V2DImode, tmp3));
+      emit_insn (gen_vec_interleave_highv2di (tmp6, tmp4, tmp5));
+
+      rtx tmp7 = gen_reg_rtx (V4SImode);
+      rtx tmp8 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp7, gen_lowpart (V4SImode, tmp6));
+      emit_insn (gen_sse2_pshufd (tmp8, tmp7, GEN_INT (0xfd)));
+
+      rtx tmp9 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp9, gen_lowpart (V1TImode, tmp8));
+      emit_move_insn (operands[0], tmp9);
+      return;
+    }
+
+  if (TARGET_SSE4_1 && (bits == 48 || bits == 80))
+    {
+      /* Four operations.  */
+      rtx tmp1 = gen_reg_rtx (V4SImode);
+      rtx tmp2 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+      emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
+
+      rtx tmp3 = gen_reg_rtx (V4SImode);
+      emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
+
+      rtx tmp4 = gen_reg_rtx (V1TImode);
+      emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (bits)));
+
+      rtx tmp5 = gen_reg_rtx (V8HImode);
+      rtx tmp6 = gen_reg_rtx (V8HImode);
+      rtx tmp7 = gen_reg_rtx (V8HImode);
+      emit_move_insn (tmp5, gen_lowpart (V8HImode, tmp3));
+      emit_move_insn (tmp6, gen_lowpart (V8HImode, tmp4));
+      emit_insn (gen_sse4_1_pblendw (tmp7, tmp5, tmp6,
+				     GEN_INT (bits == 48 ? 0x1f : 0x07)));
+
+      rtx tmp8 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp8, gen_lowpart (V1TImode, tmp7));
+      emit_move_insn (operands[0], tmp8);
+      return;
+    }
+
+  if ((bits & 7) == 0)
+    {
+      /* Five operations.  */
+      rtx tmp1 = gen_reg_rtx (V4SImode);
+      rtx tmp2 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+      emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
+
+      rtx tmp3 = gen_reg_rtx (V4SImode);
+      emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
+
+      rtx tmp4 = gen_reg_rtx (V1TImode);
+      emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (bits)));
+
+      rtx tmp5 = gen_reg_rtx (V1TImode);
+      rtx tmp6 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp5, gen_lowpart (V1TImode, tmp3));
+      emit_insn (gen_sse2_ashlv1ti3 (tmp6, tmp5, GEN_INT (128 - bits)));
+
+      rtx tmp7 = gen_reg_rtx (V2DImode);
+      rtx tmp8 = gen_reg_rtx (V2DImode);
+      rtx tmp9 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp7, gen_lowpart (V2DImode, tmp4));
+      emit_move_insn (tmp8, gen_lowpart (V2DImode, tmp6));
+      emit_insn (gen_iorv2di3 (tmp9, tmp7, tmp8));
+
+      rtx tmp10 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp10, gen_lowpart (V1TImode, tmp9));
+      emit_move_insn (operands[0], tmp10);
+      return;
+    }
+
+  if (TARGET_AVX2 && bits < 32)
+    {
+      /* Six operations.  */
+      rtx tmp1 = gen_reg_rtx (V4SImode);
+      rtx tmp2 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+      emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
+
+      rtx tmp3 = gen_reg_rtx (V1TImode);
+      emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (64)));
+
+      rtx tmp4 = gen_reg_rtx (V2DImode);
+      rtx tmp5 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp4, gen_lowpart (V2DImode, op1));
+      emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
+
+      rtx tmp6 = gen_reg_rtx (V2DImode);
+      rtx tmp7 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp6, gen_lowpart (V2DImode, tmp3));
+      emit_insn (gen_ashlv2di3 (tmp7, tmp6, GEN_INT (64 - bits)));
+
+      rtx tmp8 = gen_reg_rtx (V2DImode);
+      emit_insn (gen_iorv2di3 (tmp8, tmp5, tmp7));
+
+      rtx tmp9 = gen_reg_rtx (V4SImode);
+      rtx tmp10 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp9, gen_lowpart (V4SImode, tmp8));
+      emit_insn (gen_avx2_pblenddv4si (tmp10, tmp2, tmp9, GEN_INT (7)));
+
+      rtx tmp11 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp11, gen_lowpart (V1TImode, tmp10));
+      emit_move_insn (operands[0], tmp11);
+      return;
+    }
+
+  if (TARGET_SSE4_1 && bits < 15)
+    {
+      /* Six operations.  */
+      rtx tmp1 = gen_reg_rtx (V4SImode);
+      rtx tmp2 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+      emit_insn (gen_ashrv4si3 (tmp2, tmp1, GEN_INT (bits)));
+
+      rtx tmp3 = gen_reg_rtx (V1TImode);
+      emit_insn (gen_sse2_lshrv1ti3 (tmp3, op1, GEN_INT (64)));
+
+      rtx tmp4 = gen_reg_rtx (V2DImode);
+      rtx tmp5 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp4, gen_lowpart (V2DImode, op1));
+      emit_insn (gen_lshrv2di3 (tmp5, tmp4, GEN_INT (bits)));
+
+      rtx tmp6 = gen_reg_rtx (V2DImode);
+      rtx tmp7 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp6, gen_lowpart (V2DImode, tmp3));
+      emit_insn (gen_ashlv2di3 (tmp7, tmp6, GEN_INT (64 - bits)));
+
+      rtx tmp8 = gen_reg_rtx (V2DImode);
+      emit_insn (gen_iorv2di3 (tmp8, tmp5, tmp7));
+
+      rtx tmp9 = gen_reg_rtx (V8HImode);
+      rtx tmp10 = gen_reg_rtx (V8HImode);
+      rtx tmp11 = gen_reg_rtx (V8HImode);
+      emit_move_insn (tmp9, gen_lowpart (V8HImode, tmp2));
+      emit_move_insn (tmp10, gen_lowpart (V8HImode, tmp8));
+      emit_insn (gen_sse4_1_pblendw (tmp11, tmp9, tmp10, GEN_INT (0x3f)));
+
+      rtx tmp12 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp12, gen_lowpart (V1TImode, tmp11));
+      emit_move_insn (operands[0], tmp12);
+      return;
+    }
+
+  if (bits == 1)
+    {
+      /* Eight operations.  */
+      rtx tmp1 = gen_reg_rtx (V1TImode);
+      emit_insn (gen_sse2_lshrv1ti3 (tmp1, op1, GEN_INT (64)));
+
+      rtx tmp2 = gen_reg_rtx (V2DImode);
+      rtx tmp3 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp2, gen_lowpart (V2DImode, op1));
+      emit_insn (gen_lshrv2di3 (tmp3, tmp2, GEN_INT (1)));
+
+      rtx tmp4 = gen_reg_rtx (V2DImode);
+      rtx tmp5 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp4, gen_lowpart (V2DImode, tmp1));
+      emit_insn (gen_ashlv2di3 (tmp5, tmp4, GEN_INT (63)));
+
+      rtx tmp6 = gen_reg_rtx (V2DImode);
+      emit_insn (gen_iorv2di3 (tmp6, tmp3, tmp5));
+
+      rtx tmp7 = gen_reg_rtx (V2DImode);
+      emit_insn (gen_lshrv2di3 (tmp7, tmp2, GEN_INT (63)));
+
+      rtx tmp8 = gen_reg_rtx (V4SImode);
+      rtx tmp9 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp8, gen_lowpart (V4SImode, tmp7));
+      emit_insn (gen_sse2_pshufd (tmp9, tmp8, GEN_INT (0xbf)));
+
+      rtx tmp10 = gen_reg_rtx (V2DImode);
+      rtx tmp11 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp10, gen_lowpart (V2DImode, tmp9));
+      emit_insn (gen_ashlv2di3 (tmp11, tmp10, GEN_INT (31)));
+
+      rtx tmp12 = gen_reg_rtx (V2DImode);
+      emit_insn (gen_iorv2di3 (tmp12, tmp6, tmp11));
+
+      rtx tmp13 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp13, gen_lowpart (V1TImode, tmp12));
+      emit_move_insn (operands[0], tmp13);
+      return;
+    }
+
+  if (bits > 64)
+    {
+      /* Eight operations.  */
+      rtx tmp1 = gen_reg_rtx (V4SImode);
+      rtx tmp2 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+      emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
+
+      rtx tmp3 = gen_reg_rtx (V4SImode);
+      emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
+
+      rtx tmp4 = gen_reg_rtx (V1TImode);
+      emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (64)));
+
+      rtx tmp5 = gen_reg_rtx (V2DImode);
+      rtx tmp6 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp5, gen_lowpart (V2DImode, tmp4));
+      emit_insn (gen_lshrv2di3 (tmp6, tmp5, GEN_INT (bits - 64)));
+
+      rtx tmp7 = gen_reg_rtx (V1TImode);
+      rtx tmp8 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp7, gen_lowpart (V1TImode, tmp3));
+      emit_insn (gen_sse2_ashlv1ti3 (tmp8, tmp7, GEN_INT (64)));
+ 
+      rtx tmp9 = gen_reg_rtx (V2DImode);
+      rtx tmp10 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp9, gen_lowpart (V2DImode, tmp3));
+      emit_insn (gen_ashlv2di3 (tmp10, tmp9, GEN_INT (128 - bits)));
+
+      rtx tmp11 = gen_reg_rtx (V2DImode);
+      rtx tmp12 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp11, gen_lowpart (V2DImode, tmp8));
+      emit_insn (gen_iorv2di3 (tmp12, tmp10, tmp11));
+
+      rtx tmp13 = gen_reg_rtx (V2DImode);
+      emit_insn (gen_iorv2di3 (tmp13, tmp6, tmp12));
+
+      rtx tmp14 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp14, gen_lowpart (V1TImode, tmp13));
+      emit_move_insn (operands[0], tmp14);
+    }
+  else
+    {
+      /* Nine operations.  */
+      rtx tmp1 = gen_reg_rtx (V4SImode);
+      rtx tmp2 = gen_reg_rtx (V4SImode);
+      emit_move_insn (tmp1, gen_lowpart (V4SImode, op1));
+      emit_insn (gen_sse2_pshufd (tmp2, tmp1, GEN_INT (0xff)));
+
+      rtx tmp3 = gen_reg_rtx (V4SImode);
+      emit_insn (gen_ashrv4si3 (tmp3, tmp2, GEN_INT (31)));
+
+      rtx tmp4 = gen_reg_rtx (V1TImode);
+      emit_insn (gen_sse2_lshrv1ti3 (tmp4, op1, GEN_INT (64)));
+
+      rtx tmp5 = gen_reg_rtx (V2DImode);
+      rtx tmp6 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp5, gen_lowpart (V2DImode, op1));
+      emit_insn (gen_lshrv2di3 (tmp6, tmp5, GEN_INT (bits)));
+
+      rtx tmp7 = gen_reg_rtx (V2DImode);
+      rtx tmp8 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp7, gen_lowpart (V2DImode, tmp4));
+      emit_insn (gen_ashlv2di3 (tmp8, tmp7, GEN_INT (64 - bits)));
+
+      rtx tmp9 = gen_reg_rtx (V2DImode);
+      emit_insn (gen_iorv2di3 (tmp9, tmp6, tmp8));
+
+      rtx tmp10 = gen_reg_rtx (V1TImode);
+      rtx tmp11 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp10, gen_lowpart (V1TImode, tmp3));
+      emit_insn (gen_sse2_ashlv1ti3 (tmp11, tmp10, GEN_INT (64)));
+
+      rtx tmp12 = gen_reg_rtx (V2DImode);
+      rtx tmp13 = gen_reg_rtx (V2DImode);
+      emit_move_insn (tmp12, gen_lowpart (V2DImode, tmp11));
+      emit_insn (gen_ashlv2di3 (tmp13, tmp12, GEN_INT (64 - bits)));
+
+      rtx tmp14 = gen_reg_rtx (V2DImode);
+      emit_insn (gen_iorv2di3 (tmp14, tmp9, tmp13));
+
+      rtx tmp15 = gen_reg_rtx (V1TImode);
+      emit_move_insn (tmp15, gen_lowpart (V1TImode, tmp14));
+      emit_move_insn (operands[0], tmp15);
+    }
+}
+
 /* Return mode for the memcpy/memset loop counter.  Prefer SImode over
    DImode for constant loop counts.  */
 
@@ -9423,6 +10104,9 @@ ix86_expand_args_builtin (const struct builtin_description *d,
     case V4SF_FTYPE_V4SF_ROUND:
     case V8SF_FTYPE_V8SF_ROUND:
     case V16SF_FTYPE_V16SF_ROUND:
+    case V8HF_FTYPE_V8HF_ROUND:
+    case V16HF_FTYPE_V16HF_ROUND:
+    case V32HF_FTYPE_V32HF_ROUND:
     case V4SI_FTYPE_V4SF_ROUND:
     case V8SI_FTYPE_V8SF_ROUND:
     case V16SI_FTYPE_V16SF_ROUND:
@@ -22454,4 +23138,80 @@ ix86_expand_divmod_libfunc (rtx libfunc, machine_mode mode,
   *rem_p = rem;
 }
 
+void ix86_expand_atomic_fetch_op_loop (rtx target, rtx mem, rtx val,
+				       enum rtx_code code, bool after,
+				       bool doubleword)
+{
+  rtx old_reg, new_reg, old_mem, success, oldval, new_mem;
+  rtx_code_label *loop_label, *pause_label;
+  machine_mode mode = GET_MODE (target);
+
+  old_reg = gen_reg_rtx (mode);
+  new_reg = old_reg;
+  loop_label = gen_label_rtx ();
+  pause_label = gen_label_rtx ();
+  old_mem = copy_to_reg (mem);
+  emit_label (loop_label);
+  emit_move_insn (old_reg, old_mem);
+
+  /* return value for atomic_fetch_op.  */
+  if (!after)
+    emit_move_insn (target, old_reg);
+
+  if (code == NOT)
+    {
+      new_reg = expand_simple_binop (mode, AND, new_reg, val, NULL_RTX,
+				     true, OPTAB_LIB_WIDEN);
+      new_reg = expand_simple_unop (mode, code, new_reg, NULL_RTX, true);
+    }
+  else
+    new_reg = expand_simple_binop (mode, code, new_reg, val, NULL_RTX,
+				   true, OPTAB_LIB_WIDEN);
+
+  /* return value for atomic_op_fetch.  */
+  if (after)
+    emit_move_insn (target, new_reg);
+
+  /* Load memory again inside loop.  */
+  new_mem = copy_to_reg (mem);
+  /* Compare mem value with expected value.  */
+
+  if (doubleword)
+    {
+      machine_mode half_mode = (mode == DImode)? SImode : DImode;
+      rtx low_new_mem = gen_lowpart (half_mode, new_mem);
+      rtx low_old_mem = gen_lowpart (half_mode, old_mem);
+      rtx high_new_mem = gen_highpart (half_mode, new_mem);
+      rtx high_old_mem = gen_highpart (half_mode, old_mem);
+      emit_cmp_and_jump_insns (low_new_mem, low_old_mem, NE, NULL_RTX,
+			       half_mode, 1, pause_label,
+			       profile_probability::guessed_never ());
+      emit_cmp_and_jump_insns (high_new_mem, high_old_mem, NE, NULL_RTX,
+			       half_mode, 1, pause_label,
+			       profile_probability::guessed_never ());
+    }
+  else
+    emit_cmp_and_jump_insns (new_mem, old_mem, NE, NULL_RTX,
+			     GET_MODE (old_mem), 1, pause_label,
+			     profile_probability::guessed_never ());
+
+  success = NULL_RTX;
+  oldval = old_mem;
+  expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
+				  new_reg, false, MEMMODEL_SYNC_SEQ_CST,
+				  MEMMODEL_RELAXED);
+  if (oldval != old_mem)
+    emit_move_insn (old_mem, oldval);
+
+  emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
+			   GET_MODE (success), 1, loop_label,
+			   profile_probability::guessed_never ());
+
+  /* If mem is not expected, pause and loop back.  */
+  emit_label (pause_label);
+  emit_insn (gen_pause ());
+  emit_jump_insn (gen_jump (loop_label));
+  emit_barrier ();
+}
+
 #include "gt-i386-expand.h"
diff --git a/gcc/config/i386/i386-options.c b/gcc/config/i386/i386-options.c
index e7a3bd4..feff258 100644
--- a/gcc/config/i386/i386-options.c
+++ b/gcc/config/i386/i386-options.c
@@ -131,7 +131,7 @@ along with GCC; see the file COPYING3.  If not see
 		       | m_ICELAKE_CLIENT | m_ICELAKE_SERVER | m_CASCADELAKE \
 		       | m_TIGERLAKE | m_COOPERLAKE | m_SAPPHIRERAPIDS \
 		       | m_ROCKETLAKE)
-#define m_CORE_AVX2 (m_HASWELL | m_SKYLAKE | m_ALDERLAKE | m_CORE_AVX512)
+#define m_CORE_AVX2 (m_HASWELL | m_SKYLAKE | m_CORE_AVX512)
 #define m_CORE_ALL (m_CORE2 | m_NEHALEM  | m_SANDYBRIDGE | m_CORE_AVX2)
 #define m_GOLDMONT (HOST_WIDE_INT_1U<<PROCESSOR_GOLDMONT)
 #define m_GOLDMONT_PLUS (HOST_WIDE_INT_1U<<PROCESSOR_GOLDMONT_PLUS)
@@ -397,7 +397,8 @@ ix86_target_string (HOST_WIDE_INT isa, HOST_WIDE_INT isa2,
     { "-mstv",				MASK_STV },
     { "-mavx256-split-unaligned-load",	MASK_AVX256_SPLIT_UNALIGNED_LOAD },
     { "-mavx256-split-unaligned-store",	MASK_AVX256_SPLIT_UNALIGNED_STORE },
-    { "-mcall-ms2sysv-xlogues",		MASK_CALL_MS2SYSV_XLOGUES }
+    { "-mcall-ms2sysv-xlogues",		MASK_CALL_MS2SYSV_XLOGUES },
+    { "-mrelax-cmpxchg-loop",		MASK_RELAX_CMPXCHG_LOOP }
   };
 
   /* Additional flag options.  */
@@ -736,7 +737,7 @@ static const struct processor_costs *processor_cost_table[] =
   &icelake_cost,
   &skylake_cost,
   &icelake_cost,
-  &icelake_cost,
+  &alderlake_cost,
   &icelake_cost,
   &intel_cost,
   &geode_cost,
@@ -1092,6 +1093,10 @@ ix86_valid_target_attribute_inner_p (tree fndecl, tree args, char *p_strings[],
     IX86_ATTR_IX86_YES ("general-regs-only",
 			OPT_mgeneral_regs_only,
 			OPTION_MASK_GENERAL_REGS_ONLY),
+
+    IX86_ATTR_YES ("relax-cmpxchg-loop",
+		   OPT_mrelax_cmpxchg_loop,
+		   MASK_RELAX_CMPXCHG_LOOP),
   };
 
   location_t loc
diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
index 708834a..7e05510 100644
--- a/gcc/config/i386/i386-protos.h
+++ b/gcc/config/i386/i386-protos.h
@@ -159,6 +159,9 @@ extern void ix86_split_long_move (rtx[]);
 extern void ix86_split_ashl (rtx *, rtx, machine_mode);
 extern void ix86_split_ashr (rtx *, rtx, machine_mode);
 extern void ix86_split_lshr (rtx *, rtx, machine_mode);
+extern void ix86_expand_v1ti_shift (enum rtx_code, rtx[]);
+extern void ix86_expand_v1ti_rotate (enum rtx_code, rtx[]);
+extern void ix86_expand_v1ti_ashiftrt (rtx[]);
 extern rtx ix86_find_base_term (rtx);
 extern bool ix86_check_movabs (rtx, int);
 extern bool ix86_check_no_addr_space (rtx);
@@ -214,6 +217,8 @@ extern void ix86_move_vector_high_sse_to_mmx (rtx);
 extern void ix86_split_mmx_pack (rtx[], enum rtx_code);
 extern void ix86_split_mmx_punpck (rtx[], bool);
 extern void ix86_expand_avx_vzeroupper (void);
+extern void ix86_expand_atomic_fetch_op_loop (rtx, rtx, rtx, enum rtx_code,
+					      bool, bool);
 
 #ifdef TREE_CODE
 extern void init_cumulative_args (CUMULATIVE_ARGS *, tree, rtx, tree, int);
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index 299e1ab..73c4d51 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -11161,7 +11161,7 @@ legitimize_pic_address (rtx orig, rtx reg)
 	new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
     }
   else if ((GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_TLS_MODEL (addr) == 0)
-	   /* We can't use @GOTOFF for text labels
+	   /* We can't always use @GOTOFF for text labels
 	      on VxWorks, see gotoff_operand.  */
 	   || (TARGET_VXWORKS_RTP && GET_CODE (addr) == LABEL_REF))
     {
@@ -11190,9 +11190,19 @@ legitimize_pic_address (rtx orig, rtx reg)
 	     from the Global Offset Table (@GOT).  */
 	  new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOT);
 	  new_rtx = gen_rtx_CONST (Pmode, new_rtx);
+
 	  if (TARGET_64BIT)
-	    new_rtx = force_reg (Pmode, new_rtx);
-	  new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
+	    new_rtx = copy_to_suggested_reg (new_rtx, reg, Pmode);
+
+	  if (reg != 0)
+	    {
+	      gcc_assert (REG_P (reg));
+	      new_rtx = expand_simple_binop (Pmode, PLUS, pic_offset_table_rtx,
+					     new_rtx, reg, 1, OPTAB_DIRECT);
+	    }
+	  else
+	    new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
+
 	  new_rtx = gen_const_mem (Pmode, new_rtx);
 	  set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
 	}
@@ -17004,6 +17014,7 @@ ix86_sched_init_global (FILE *, int, int)
     case PROCESSOR_SANDYBRIDGE:
     case PROCESSOR_HASWELL:
     case PROCESSOR_TREMONT:
+    case PROCESSOR_ALDERLAKE:
     case PROCESSOR_GENERIC:
       /* Do not perform multipass scheduling for pre-reload schedule
          to save compile time.  */
@@ -18905,9 +18916,10 @@ ix86_vectorize_builtin_scatter (const_tree vectype,
    1.0/sqrt.  */
 
 static bool
-use_rsqrt_p ()
+use_rsqrt_p (machine_mode mode)
 {
-  return (TARGET_SSE && TARGET_SSE_MATH
+  return ((mode == HFmode
+	   || (TARGET_SSE && TARGET_SSE_MATH))
 	  && flag_finite_math_only
 	  && !flag_trapping_math
 	  && flag_unsafe_math_optimizations);
@@ -21988,6 +22000,10 @@ ix86_libgcc_floating_mode_supported_p (scalar_float_mode mode)
 static bool
 ix86_vector_mode_supported_p (machine_mode mode)
 {
+  /* For ia32, scalar TImode isn't supported and so V1TImode shouldn't be
+     either.  */
+  if (!TARGET_64BIT && GET_MODE_INNER (mode) == TImode)
+    return false;
   if (TARGET_SSE && VALID_SSE_REG_MODE (mode))
     return true;
   if (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
@@ -22863,26 +22879,30 @@ ix86_noce_conversion_profitable_p (rtx_insn *seq, struct noce_if_info *if_info)
   return default_noce_conversion_profitable_p (seq, if_info);
 }
 
-/* Implement targetm.vectorize.init_cost.  */
+/* x86-specific vector costs.  */
+class ix86_vector_costs : public vector_costs
+{
+  using vector_costs::vector_costs;
 
-static void *
-ix86_init_cost (class loop *, bool)
+  unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
+			      stmt_vec_info stmt_info, tree vectype,
+			      int misalign,
+			      vect_cost_model_location where) override;
+};
+
+/* Implement targetm.vectorize.create_costs.  */
+
+static vector_costs *
+ix86_vectorize_create_costs (vec_info *vinfo, bool costing_for_scalar)
 {
-  unsigned *cost = XNEWVEC (unsigned, 3);
-  cost[vect_prologue] = cost[vect_body] = cost[vect_epilogue] = 0;
-  return cost;
+  return new ix86_vector_costs (vinfo, costing_for_scalar);
 }
 
-/* Implement targetm.vectorize.add_stmt_cost.  */
-
-static unsigned
-ix86_add_stmt_cost (class vec_info *vinfo, void *data, int count,
-		    enum vect_cost_for_stmt kind,
-		    class _stmt_vec_info *stmt_info, tree vectype,
-		    int misalign,
-		    enum vect_cost_model_location where)
+unsigned
+ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
+				  stmt_vec_info stmt_info, tree vectype,
+				  int misalign, vect_cost_model_location where)
 {
-  unsigned *cost = (unsigned *) data;
   unsigned retval = 0;
   bool scalar_p
     = (kind == scalar_stmt || kind == scalar_load || kind == scalar_store);
@@ -23053,15 +23073,7 @@ ix86_add_stmt_cost (class vec_info *vinfo, void *data, int count,
   /* Statements in an inner loop relative to the loop being
      vectorized are weighted more heavily.  The value here is
      arbitrary and could potentially be improved with analysis.  */
-  if (where == vect_body && stmt_info
-      && stmt_in_inner_loop_p (vinfo, stmt_info))
-    {
-      loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
-      gcc_assert (loop_vinfo);
-      count *= LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo); /* FIXME.  */
-    }
-
-  retval = (unsigned) (count * stmt_cost);
+  retval = adjust_cost_for_freq (stmt_info, where, count * stmt_cost);
 
   /* We need to multiply all vector stmt cost by 1.7 (estimated cost)
      for Silvermont as it has out of order integer pipeline and can execute
@@ -23076,31 +23088,11 @@ ix86_add_stmt_cost (class vec_info *vinfo, void *data, int count,
 	retval = (retval * 17) / 10;
     }
 
-  cost[where] += retval;
+  m_costs[where] += retval;
 
   return retval;
 }
 
-/* Implement targetm.vectorize.finish_cost.  */
-
-static void
-ix86_finish_cost (void *data, unsigned *prologue_cost,
-		  unsigned *body_cost, unsigned *epilogue_cost)
-{
-  unsigned *cost = (unsigned *) data;
-  *prologue_cost = cost[vect_prologue];
-  *body_cost     = cost[vect_body];
-  *epilogue_cost = cost[vect_epilogue];
-}
-
-/* Implement targetm.vectorize.destroy_cost_data.  */
-
-static void
-ix86_destroy_cost_data (void *data)
-{
-  free (data);
-}
-
 /* Validate target specific memory model bits in VAL. */
 
 static unsigned HOST_WIDE_INT
@@ -23603,29 +23595,27 @@ ix86_optab_supported_p (int op, machine_mode mode1, machine_mode,
       return opt_type == OPTIMIZE_FOR_SPEED;
 
     case rint_optab:
-      if (mode1 == HFmode)
-	return true;
-      else if (SSE_FLOAT_MODE_P (mode1)
-	       && TARGET_SSE_MATH
-	       && !flag_trapping_math
-	       && !TARGET_SSE4_1)
+      if (SSE_FLOAT_MODE_P (mode1)
+	  && TARGET_SSE_MATH
+	  && !flag_trapping_math
+	  && !TARGET_SSE4_1
+	  && mode1 != HFmode)
 	return opt_type == OPTIMIZE_FOR_SPEED;
       return true;
 
     case floor_optab:
     case ceil_optab:
     case btrunc_optab:
-      if (mode1 == HFmode)
-	return true;
-      else if (SSE_FLOAT_MODE_P (mode1)
-	       && TARGET_SSE_MATH
-	       && !flag_trapping_math
-	       && TARGET_SSE4_1)
+      if (((SSE_FLOAT_MODE_P (mode1)
+	    && TARGET_SSE_MATH
+	    && TARGET_SSE4_1)
+	   || mode1 == HFmode)
+	  && !flag_trapping_math)
 	return true;
       return opt_type == OPTIMIZE_FOR_SPEED;
 
     case rsqrt_optab:
-      return opt_type == OPTIMIZE_FOR_SPEED && use_rsqrt_p ();
+      return opt_type == OPTIMIZE_FOR_SPEED && use_rsqrt_p (mode1);
 
     default:
       return true;
@@ -24384,14 +24374,8 @@ ix86_libgcc_floating_mode_supported_p
   ix86_autovectorize_vector_modes
 #undef TARGET_VECTORIZE_GET_MASK_MODE
 #define TARGET_VECTORIZE_GET_MASK_MODE ix86_get_mask_mode
-#undef TARGET_VECTORIZE_INIT_COST
-#define TARGET_VECTORIZE_INIT_COST ix86_init_cost
-#undef TARGET_VECTORIZE_ADD_STMT_COST
-#define TARGET_VECTORIZE_ADD_STMT_COST ix86_add_stmt_cost
-#undef TARGET_VECTORIZE_FINISH_COST
-#define TARGET_VECTORIZE_FINISH_COST ix86_finish_cost
-#undef TARGET_VECTORIZE_DESTROY_COST_DATA
-#define TARGET_VECTORIZE_DESTROY_COST_DATA ix86_destroy_cost_data
+#undef TARGET_VECTORIZE_CREATE_COSTS
+#define TARGET_VECTORIZE_CREATE_COSTS ix86_vectorize_create_costs
 
 #undef TARGET_SET_CURRENT_FUNCTION
 #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index cba6d83..e35c79c 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -78,8 +78,9 @@ struct stringop_algs
        this issue.  Since this header is used by code compiled with the C
        compiler we must guard the addition.  */
 #ifdef __cplusplus
-    stringop_strategy(int _max = -1, enum stringop_alg _alg = libcall,
-		      int _noalign = false)
+    constexpr
+    stringop_strategy (int _max = -1, enum stringop_alg _alg = libcall,
+		       int _noalign = false)
       : max (_max), alg (_alg), noalign (_noalign) {}
 #endif
     const int max;
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index e733a40..7394906 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -8417,11 +8417,27 @@
 		(match_operand:XF 2 "register_operand")))]
   "TARGET_80387")
 
+/* There is no more precision loss than Newton-Rhapson approximation
+  when using HFmode rcp/rsqrt, so do the transformation directly under
+  TARGET_RECIP_DIV and fast-math.  */
 (define_expand "divhf3"
   [(set (match_operand:HF 0 "register_operand")
 	(div:HF (match_operand:HF 1 "register_operand")
 		   (match_operand:HF 2 "nonimmediate_operand")))]
-  "TARGET_AVX512FP16")
+  "TARGET_AVX512FP16"
+{
+  if (TARGET_RECIP_DIV
+      && optimize_insn_for_speed_p ()
+      && flag_finite_math_only && !flag_trapping_math
+      && flag_unsafe_math_optimizations)
+    {
+      rtx op = gen_reg_rtx (HFmode);
+      operands[2] = force_reg (HFmode, operands[2]);
+      emit_insn (gen_rcphf2 (op, operands[2]));
+      emit_insn (gen_mulhf3 (operands[0], operands[1], op));
+      DONE;
+    }
+})
 
 (define_expand "div<mode>3"
   [(set (match_operand:MODEF 0 "register_operand")
@@ -12573,8 +12589,24 @@
     emit_insn (gen_ix86_<insn>ti3_doubleword
 		(operands[0], operands[1], operands[2]));
   else
-    FAIL;
-
+    {
+      rtx amount = force_reg (QImode, operands[2]);
+      rtx src_lo = gen_lowpart (DImode, operands[1]);
+      rtx src_hi = gen_highpart (DImode, operands[1]);
+      rtx tmp_lo = gen_reg_rtx (DImode);
+      rtx tmp_hi = gen_reg_rtx (DImode);
+      emit_move_insn (tmp_lo, src_lo);
+      emit_move_insn (tmp_hi, src_hi);
+      rtx (*shiftd) (rtx, rtx, rtx)
+	    = (<CODE> == ROTATE) ? gen_x86_64_shld : gen_x86_64_shrd;
+      emit_insn (shiftd (tmp_lo, src_hi, amount));
+      emit_insn (shiftd (tmp_hi, src_lo, amount));
+      rtx dst_lo = gen_lowpart (DImode, operands[0]);
+      rtx dst_hi = gen_highpart (DImode, operands[0]);
+      emit_move_insn (dst_lo, tmp_lo);
+      emit_move_insn (dst_hi, tmp_hi);
+      emit_insn (gen_x86_shiftdi_adj_1 (dst_lo, dst_hi, amount, tmp_lo));
+    }
   DONE;
 })
 
@@ -12743,7 +12775,7 @@
 	(rotatert:SWI48
 	  (match_operand:SWI48 1 "nonimmediate_operand" "rm")
 	  (match_operand:QI 2 "<rorx_immediate_operand>" "<S>")))]
-  "TARGET_BMI2"
+  "TARGET_BMI2 && !optimize_function_for_size_p (cfun)"
   "rorx\t{%2, %1, %0|%0, %1, %2}"
   [(set_attr "type" "rotatex")
    (set_attr "mode" "<MODE>")])
@@ -12771,6 +12803,10 @@
 }
   [(set_attr "isa" "*,bmi2")
    (set_attr "type" "rotate,rotatex")
+   (set (attr "preferred_for_size")
+     (cond [(eq_attr "alternative" "0")
+	      (symbol_ref "true")]
+	   (symbol_ref "false")))
    (set (attr "length_immediate")
      (if_then_else
        (and (eq_attr "type" "rotate")
@@ -12787,7 +12823,7 @@
 	(rotate:SWI48 (match_operand:SWI48 1 "nonimmediate_operand")
 		      (match_operand:QI 2 "const_int_operand")))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_BMI2 && reload_completed"
+  "TARGET_BMI2 && reload_completed && !optimize_function_for_size_p (cfun)"
   [(set (match_dup 0)
 	(rotatert:SWI48 (match_dup 1) (match_dup 2)))]
 {
@@ -12801,7 +12837,7 @@
 	(rotatert:SWI48 (match_operand:SWI48 1 "nonimmediate_operand")
 			(match_operand:QI 2 "const_int_operand")))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_BMI2 && reload_completed"
+  "TARGET_BMI2 && reload_completed && !optimize_function_for_size_p (cfun)"
   [(set (match_dup 0)
 	(rotatert:SWI48 (match_dup 1) (match_dup 2)))])
 
@@ -12810,7 +12846,7 @@
 	(zero_extend:DI
 	  (rotatert:SI (match_operand:SI 1 "nonimmediate_operand" "rm")
 		       (match_operand:QI 2 "const_0_to_31_operand" "I"))))]
-  "TARGET_64BIT && TARGET_BMI2"
+  "TARGET_64BIT && TARGET_BMI2 && !optimize_function_for_size_p (cfun)"
   "rorx\t{%2, %1, %k0|%k0, %1, %2}"
   [(set_attr "type" "rotatex")
    (set_attr "mode" "SI")])
@@ -12838,6 +12874,10 @@
 }
   [(set_attr "isa" "*,bmi2")
    (set_attr "type" "rotate,rotatex")
+   (set (attr "preferred_for_size")
+     (cond [(eq_attr "alternative" "0")
+	      (symbol_ref "true")]
+	   (symbol_ref "false")))
    (set (attr "length_immediate")
      (if_then_else
        (and (eq_attr "type" "rotate")
@@ -12855,7 +12895,8 @@
 	  (rotate:SI (match_operand:SI 1 "nonimmediate_operand")
 		     (match_operand:QI 2 "const_int_operand"))))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && TARGET_BMI2 && reload_completed"
+  "TARGET_64BIT && TARGET_BMI2 && reload_completed
+   && !optimize_function_for_size_p (cfun)"
   [(set (match_dup 0)
 	(zero_extend:DI (rotatert:SI (match_dup 1) (match_dup 2))))]
 {
@@ -12870,7 +12911,8 @@
 	  (rotatert:SI (match_operand:SI 1 "nonimmediate_operand")
 		       (match_operand:QI 2 "const_int_operand"))))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && TARGET_BMI2 && reload_completed"
+  "TARGET_64BIT && TARGET_BMI2 && reload_completed
+   && !optimize_function_for_size_p (cfun)"
   [(set (match_dup 0)
 	(zero_extend:DI (rotatert:SI (match_dup 1) (match_dup 2))))])
 
@@ -16973,6 +17015,19 @@
 	    ]
 	    (symbol_ref "true")))])
 
+(define_insn "rcphf2"
+  [(set (match_operand:HF 0 "register_operand" "=v,v")
+	(unspec:HF [(match_operand:HF 1 "nonimmediate_operand" "v,m")]
+		   UNSPEC_RCP))]
+  "TARGET_AVX512FP16"
+  "@
+   vrcpsh\t{%d1, %0|%0, %d1}
+   vrcpsh\t{%1, %d0|%d0, %1}"
+  [(set_attr "type" "sse")
+   (set_attr "prefix" "evex")
+   (set_attr "mode" "HF")
+   (set_attr "avx_partial_xmm_update" "false,true")])
+
 (define_insn "*fop_xf_1_i387"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
 	(match_operator:XF 3 "binary_fp_operator"
@@ -17230,6 +17285,19 @@
   DONE;
 })
 
+(define_insn "rsqrthf2"
+  [(set (match_operand:HF 0 "register_operand" "=v,v")
+	(unspec:HF [(match_operand:HF 1 "nonimmediate_operand" "v,m")]
+		   UNSPEC_RSQRT))]
+  "TARGET_AVX512FP16"
+  "@
+   vrsqrtsh\t{%d1, %0|%0, %d1}
+   vrsqrtsh\t{%1, %d0|%d0, %1}"
+  [(set_attr "type" "sse")
+   (set_attr "prefix" "evex")
+   (set_attr "avx_partial_xmm_update" "false,true")
+   (set_attr "mode" "HF")])
+
 (define_insn "sqrthf2"
   [(set (match_operand:HF 0 "register_operand" "=v,v")
 	(sqrt:HF
diff --git a/gcc/config/i386/i386.opt b/gcc/config/i386/i386.opt
index ad36697..46fad3c 100644
--- a/gcc/config/i386/i386.opt
+++ b/gcc/config/i386/i386.opt
@@ -404,6 +404,10 @@ momit-leaf-frame-pointer
 Target Mask(OMIT_LEAF_FRAME_POINTER) Save
 Omit the frame pointer in leaf functions.
 
+mrelax-cmpxchg-loop
+Target Mask(RELAX_CMPXCHG_LOOP) Save
+Relax cmpxchg loop for atomic_fetch_{or,xor,and,nand} by adding load and cmp before cmpxchg, execute pause and loop back to load and compare if load value is not expected.
+
 mpc32
 Target RejectNegative
 Set 80387 floating-point precision to 32-bit.
diff --git a/gcc/config/i386/predicates.md b/gcc/config/i386/predicates.md
index df5acb4..4ccbe11 100644
--- a/gcc/config/i386/predicates.md
+++ b/gcc/config/i386/predicates.md
@@ -617,9 +617,11 @@
 ;; use @GOTOFF unless we are absolutely sure that the symbol is in the
 ;; same segment as the GOT.  Unfortunately, the flexibility of linker
 ;; scripts means that we can't be sure of that in general, so assume
-;; that @GOTOFF is never valid on VxWorks.
+;; @GOTOFF is not valid on VxWorks, except with the large code model.
 (define_predicate "gotoff_operand"
-  (and (not (match_test "TARGET_VXWORKS_RTP"))
+  (and (ior (not (match_test "TARGET_VXWORKS_RTP"))
+            (match_test "ix86_cmodel == CM_LARGE")
+            (match_test "ix86_cmodel == CM_LARGE_PIC"))
        (match_operand 0 "local_symbolic_operand")))
 
 ;; Test for various thread-local symbols.
@@ -1046,10 +1048,10 @@
 
 ;; True for registers, or (not: registers).  Used to optimize 3-operand
 ;; bitwise operation.
-(define_predicate "reg_or_notreg_operand"
-  (ior (match_operand 0 "register_operand")
+(define_predicate "regmem_or_bitnot_regmem_operand"
+  (ior (match_operand 0 "nonimmediate_operand")
        (and (match_code "not")
-	    (match_test "register_operand (XEXP (op, 0), mode)"))))
+	    (match_test "nonimmediate_operand (XEXP (op, 0), mode)"))))
 
 ;; True if OP is acceptable as operand of DImode shift expander.
 (define_predicate "shiftdi_operand"
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index fbf056b..b109c2a 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -193,7 +193,9 @@
 
   ;; For AVX512FP16 suppport
   UNSPEC_COMPLEX_FMA
+  UNSPEC_COMPLEX_FMA_PAIR
   UNSPEC_COMPLEX_FCMA
+  UNSPEC_COMPLEX_FCMA_PAIR
   UNSPEC_COMPLEX_FMUL
   UNSPEC_COMPLEX_FCMUL
   UNSPEC_COMPLEX_MASK
@@ -302,7 +304,7 @@
 
 ;; All 128bit and 256bit vector modes
 (define_mode_iterator V_128_256
-  [V32QI V16QI V16HI V8HI V8SI V4SI V4DI V2DI V8SF V4SF V4DF V2DF])
+  [V32QI V16QI V16HI V8HI V8SI V4SI V4DI V2DI V16HF V8HF V8SF V4SF V4DF V2DF])
 
 ;; All 512bit vector modes
 (define_mode_iterator V_512 [V64QI V32HI V16SI V8DI V16SF V8DF])
@@ -500,6 +502,9 @@
 (define_mode_iterator VI1_AVX512F
   [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI])
 
+(define_mode_iterator VI1_AVX512VNNI
+  [(V64QI "TARGET_AVX512VNNI") (V32QI "TARGET_AVX2") V16QI])
+
 (define_mode_iterator VI12_256_512_AVX512VL
   [V64QI (V32QI "TARGET_AVX512VL")
    V32HI (V16HI "TARGET_AVX512VL")])
@@ -510,6 +515,10 @@
 (define_mode_iterator VI2_AVX512F
   [(V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX2") V8HI])
 
+(define_mode_iterator VI2_AVX512VNNIBW
+  [(V32HI "TARGET_AVX512BW || TARGET_AVX512VNNI")
+   (V16HI "TARGET_AVX2") V8HI])
+
 (define_mode_iterator VI4_AVX
   [(V8SI "TARGET_AVX") V4SI])
 
@@ -896,9 +905,9 @@
   [(V64QI "512") (V32HI "512") (V16SI "512") (V8DI "512")
    (V32QI "256") (V16HI "256") (V8SI "256") (V4DI "256")
    (V16QI "") (V8HI "") (V4SI "") (V2DI "")
-   (V16SF "512") (V8DF "512")
-   (V8SF "256") (V4DF "256")
-   (V4SF "") (V2DF "")])
+   (V32HF "512") (V16SF "512") (V8DF "512")
+   (V16HF "256") (V8SF "256") (V4DF "256")
+   (V8HF "") (V4SF "") (V2DF "")])
 
 ;; SSE instruction mode
 (define_mode_attr sseinsnmode
@@ -2306,12 +2315,34 @@
    (set_attr "mode" "<ssescalarmode>")])
 
 (define_expand "div<mode>3"
-  [(set (match_operand:VF2H 0 "register_operand")
-	(div:VF2H (match_operand:VF2H 1 "register_operand")
-		  (match_operand:VF2H 2 "vector_operand")))]
+  [(set (match_operand:VF2 0 "register_operand")
+	(div:VF2 (match_operand:VF2 1 "register_operand")
+		  (match_operand:VF2 2 "vector_operand")))]
   "TARGET_SSE2")
 
 (define_expand "div<mode>3"
+  [(set (match_operand:VF_AVX512FP16VL 0 "register_operand")
+	(div:VF_AVX512FP16VL
+	  (match_operand:VF_AVX512FP16VL 1 "register_operand")
+	  (match_operand:VF_AVX512FP16VL 2 "vector_operand")))]
+  "TARGET_AVX512FP16"
+{
+  /* Transform HF vector div to vector mul/rcp.  */
+  if (GET_MODE_INNER (<MODE>mode) == HFmode
+      && TARGET_RECIP_VEC_DIV
+      && optimize_insn_for_speed_p ()
+      && flag_finite_math_only && !flag_trapping_math
+      && flag_unsafe_math_optimizations)
+    {
+      rtx op = gen_reg_rtx (<MODE>mode);
+      operands[2] = force_reg (<MODE>mode, operands[2]);
+      emit_insn (gen_avx512fp16_rcp<mode>2 (op, operands[2]));
+      emit_insn (gen_mul<mode>3 (operands[0], operands[1], op));
+      DONE;
+    }
+})
+
+(define_expand "div<mode>3"
   [(set (match_operand:VF1 0 "register_operand")
 	(div:VF1 (match_operand:VF1 1 "register_operand")
 		 (match_operand:VF1 2 "vector_operand")))]
@@ -2433,6 +2464,20 @@
    (set_attr "prefix" "evex")
    (set_attr "mode" "HF")])
 
+(define_insn "*avx512fp16_vmrcpv8hf2"
+  [(set (match_operand:V8HF 0 "register_operand" "=v")
+	(vec_merge:V8HF
+	  (vec_duplicate:V8HF
+	    (unspec:HF [(match_operand:HF 1 "nonimmediate_operand" "vm")]
+			 UNSPEC_RCP))
+	  (match_operand:V8HF 2 "register_operand" "v")
+	  (const_int 1)))]
+  "TARGET_AVX512FP16"
+  "vrcpsh\t{%1, %2, %0|%0, %2, %w1}"
+  [(set_attr "type" "sse")
+   (set_attr "prefix" "evex")
+   (set_attr "mode" "HF")])
+
 (define_insn "<mask_codefor>rcp14<mode><mask_name>"
   [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
 	(unspec:VF_AVX512VL
@@ -2558,6 +2603,13 @@
   DONE;
 })
 
+(define_expand "rsqrt<mode>2"
+  [(set (match_operand:VF_AVX512FP16VL 0 "register_operand")
+	(unspec:VF_AVX512FP16VL
+	  [(match_operand:VF_AVX512FP16VL 1 "vector_operand")]
+	  UNSPEC_RSQRT))]
+  "TARGET_AVX512FP16")
+
 (define_insn "<sse>_rsqrt<mode>2"
   [(set (match_operand:VF1_128_256 0 "register_operand" "=x")
 	(unspec:VF1_128_256
@@ -2666,6 +2718,20 @@
    (set_attr "prefix" "evex")
    (set_attr "mode" "HF")])
 
+(define_insn "*avx512fp16_vmrsqrtv8hf2"
+  [(set (match_operand:V8HF 0 "register_operand" "=v")
+	(vec_merge:V8HF
+	  (vec_duplicate:V8HF
+	    (unspec:HF [(match_operand:HF 1 "nonimmediate_operand" "vm")]
+		       UNSPEC_RSQRT))
+	  (match_operand:V8HF 2 "register_operand" "v")
+	  (const_int 1)))]
+  "TARGET_AVX512FP16"
+  "vrsqrtsh\t{%1, %2, %0<mask_scalar_operand3>|%0<mask_scalar_operand3>, %2, %w1}"
+  [(set_attr "type" "sse")
+   (set_attr "prefix" "evex")
+   (set_attr "mode" "HF")])
+
 (define_expand "cond_<code><mode>"
   [(set (match_operand:VFH 0 "register_operand")
 	(vec_merge:VFH
@@ -5856,6 +5922,9 @@
 (define_int_iterator UNSPEC_COMPLEX_F_C_MA
 	[UNSPEC_COMPLEX_FMA UNSPEC_COMPLEX_FCMA])
 
+(define_int_iterator UNSPEC_COMPLEX_F_C_MA_PAIR
+	[UNSPEC_COMPLEX_FMA_PAIR UNSPEC_COMPLEX_FCMA_PAIR])
+
 (define_int_iterator UNSPEC_COMPLEX_F_C_MUL
 	[UNSPEC_COMPLEX_FMUL UNSPEC_COMPLEX_FCMUL])
 
@@ -5865,6 +5934,16 @@
 	 (UNSPEC_COMPLEX_FMUL "fmulc")
 	 (UNSPEC_COMPLEX_FCMUL "fcmulc")])
 
+(define_int_attr complexpairopname
+	[(UNSPEC_COMPLEX_FMA_PAIR "fmaddc")
+	 (UNSPEC_COMPLEX_FCMA_PAIR "fcmaddc")])
+
+(define_int_attr conj_op
+	[(UNSPEC_COMPLEX_FMA "")
+	 (UNSPEC_COMPLEX_FCMA "_conj")
+	 (UNSPEC_COMPLEX_FMUL "")
+	 (UNSPEC_COMPLEX_FCMUL "_conj")])
+
 (define_mode_attr complexmove
   [(V32HF "avx512f_loadv16sf")
    (V16HF "avx512vl_loadv8sf")
@@ -5946,6 +6025,15 @@
   DONE;
 })
 
+(define_expand "cmla<conj_op><mode>4"
+  [(set (match_operand:VF_AVX512FP16VL 0 "register_operand")
+	(unspec:VF_AVX512FP16VL
+	    [(match_operand:VF_AVX512FP16VL 1 "vector_operand")
+	     (match_operand:VF_AVX512FP16VL 2 "vector_operand")
+	     (match_operand:VF_AVX512FP16VL 3 "vector_operand")]
+	     UNSPEC_COMPLEX_F_C_MA))]
+  "TARGET_AVX512FP16")
+
 (define_insn "fma_<complexopname>_<mode><sdc_maskz_name><round_name>"
   [(set (match_operand:VF_AVX512FP16VL 0 "register_operand" "=&v")
 	(unspec:VF_AVX512FP16VL
@@ -5958,6 +6046,111 @@
   [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
+(define_insn_and_split "fma_<mode>_fadd_fmul"
+  [(set (match_operand:VF_AVX512FP16VL 0 "register_operand")
+	(plus:VF_AVX512FP16VL
+	  (unspec:VF_AVX512FP16VL
+		[(match_operand:VF_AVX512FP16VL 1 "vector_operand")
+		 (match_operand:VF_AVX512FP16VL 2 "vector_operand")]
+		 UNSPEC_COMPLEX_FMUL)
+	  (match_operand:VF_AVX512FP16VL 3 "vector_operand")))]
+  "TARGET_AVX512FP16 && flag_unsafe_math_optimizations
+  && ix86_pre_reload_split ()"
+  "#"
+  "&& 1"
+  [(set (match_dup 0)
+	(unspec:VF_AVX512FP16VL
+	  [(match_dup 1) (match_dup 2) (match_dup 3)]
+	   UNSPEC_COMPLEX_FMA))])
+
+(define_insn_and_split "fma_<mode>_fadd_fcmul"
+  [(set (match_operand:VF_AVX512FP16VL 0 "register_operand")
+	(plus:VF_AVX512FP16VL
+	  (unspec:VF_AVX512FP16VL
+		[(match_operand:VF_AVX512FP16VL 1 "vector_operand")
+		 (match_operand:VF_AVX512FP16VL 2 "vector_operand")]
+		 UNSPEC_COMPLEX_FCMUL)
+	  (match_operand:VF_AVX512FP16VL 3 "vector_operand")))]
+  "TARGET_AVX512FP16 && flag_unsafe_math_optimizations
+  && ix86_pre_reload_split ()"
+  "#"
+  "&& 1"
+  [(set (match_dup 0)
+	(unspec:VF_AVX512FP16VL
+	  [(match_dup 1) (match_dup 2) (match_dup 3)]
+	   UNSPEC_COMPLEX_FCMA))])
+
+(define_insn_and_split "fma_<complexopname>_<mode>_fma_zero"
+  [(set (match_operand:VF_AVX512FP16VL 0 "register_operand")
+	(plus:VF_AVX512FP16VL
+	  (unspec:VF_AVX512FP16VL
+		[(match_operand:VF_AVX512FP16VL 1 "vector_operand")
+		 (match_operand:VF_AVX512FP16VL 2 "vector_operand")
+		 (match_operand:VF_AVX512FP16VL 3 "const0_operand")]
+		 UNSPEC_COMPLEX_F_C_MA)
+	  (match_operand:VF_AVX512FP16VL 4 "vector_operand")))]
+  "TARGET_AVX512FP16 && flag_unsafe_math_optimizations
+  && ix86_pre_reload_split ()"
+  "#"
+  "&& 1"
+  [(set (match_dup 0)
+	(unspec:VF_AVX512FP16VL
+	  [(match_dup 1) (match_dup 2) (match_dup 4)]
+	   UNSPEC_COMPLEX_F_C_MA))])
+
+(define_insn "fma_<complexpairopname>_<mode>_pair"
+ [(set (match_operand:VF1_AVX512VL 0 "register_operand" "=&v")
+       (unspec:VF1_AVX512VL
+	 [(match_operand:VF1_AVX512VL 1 "vector_operand" "%v")
+	  (match_operand:VF1_AVX512VL 2 "bcst_vector_operand" "vmBr")
+	  (match_operand:VF1_AVX512VL 3 "vector_operand" "0")]
+	  UNSPEC_COMPLEX_F_C_MA_PAIR))]
+ "TARGET_AVX512FP16"
+ "v<complexpairopname>ph\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "ssemuladd")])
+
+(define_insn_and_split "fma_<mode>_fmaddc_bcst"
+  [(set (match_operand:VF_AVX512FP16VL 0 "register_operand")
+	(unspec:VF_AVX512FP16VL
+	  [(match_operand:VF_AVX512FP16VL 1 "vector_operand")
+	   (subreg:VF_AVX512FP16VL
+	     (match_operand:<ssePSmode> 2 "bcst_vector_operand") 0)
+	   (match_operand:VF_AVX512FP16VL 3 "vector_operand")]
+	   UNSPEC_COMPLEX_FMA))]
+  "TARGET_AVX512FP16"
+  "#"
+  "&& 1"
+  [(set (match_dup 0)
+	(unspec:<ssePSmode>
+	  [(match_dup 1) (match_dup 2) (match_dup 3)]
+	   UNSPEC_COMPLEX_FMA_PAIR))]
+  {
+    operands[0] = lowpart_subreg (<ssePSmode>mode, operands[0], <MODE>mode);
+    operands[1] = lowpart_subreg (<ssePSmode>mode, operands[1], <MODE>mode);
+    operands[3] = lowpart_subreg (<ssePSmode>mode, operands[3], <MODE>mode);
+  })
+
+(define_insn_and_split "fma_<mode>_fcmaddc_bcst"
+  [(set (match_operand:VF_AVX512FP16VL 0 "register_operand")
+	(unspec:VF_AVX512FP16VL
+	  [(match_operand:VF_AVX512FP16VL 1 "vector_operand")
+	   (subreg:VF_AVX512FP16VL
+	     (match_operand:<ssePSmode> 2 "bcst_vector_operand") 0)
+	   (match_operand:VF_AVX512FP16VL 3 "vector_operand")]
+	   UNSPEC_COMPLEX_FCMA))]
+  "TARGET_AVX512FP16"
+  "#"
+  "&& 1"
+  [(set (match_dup 0)
+	(unspec:<ssePSmode>
+	  [(match_dup 1) (match_dup 2) (match_dup 3)]
+	   UNSPEC_COMPLEX_FCMA_PAIR))]
+  {
+    operands[0] = lowpart_subreg (<ssePSmode>mode, operands[0], <MODE>mode);
+    operands[1] = lowpart_subreg (<ssePSmode>mode, operands[1], <MODE>mode);
+    operands[3] = lowpart_subreg (<ssePSmode>mode, operands[3], <MODE>mode);
+  })
+
 (define_insn "<avx512>_<complexopname>_<mode>_mask<round_name>"
   [(set (match_operand:VF_AVX512FP16VL 0 "register_operand" "=&v")
 	(vec_merge:VF_AVX512FP16VL
@@ -5975,6 +6168,14 @@
   [(set_attr "type" "ssemuladd")
    (set_attr "mode" "<MODE>")])
 
+(define_expand "cmul<conj_op><mode>3"
+  [(set (match_operand:VF_AVX512FP16VL 0 "register_operand")
+	(unspec:VF_AVX512FP16VL
+	  [(match_operand:VF_AVX512FP16VL 1 "vector_operand")
+	   (match_operand:VF_AVX512FP16VL 2 "vector_operand")]
+	   UNSPEC_COMPLEX_F_C_MUL))]
+  "TARGET_AVX512FP16")
+
 (define_insn "<avx512>_<complexopname>_<mode><maskc_name><round_name>"
   [(set (match_operand:VF_AVX512FP16VL 0 "register_operand" "=&v")
 	  (unspec:VF_AVX512FP16VL
@@ -11546,7 +11747,11 @@
 	   (match_operand:V 3 "bcst_vector_operand" "vmBr")
 	   (match_operand:SI 4 "const_0_to_255_operand")]
 	  UNSPEC_VTERNLOG))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F
+/* Disallow embeded broadcast for vector HFmode since
+   it's not real AVX512FP16 instruction.  */
+  && (GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) >= 4
+     || GET_CODE (operands[3]) != VEC_DUPLICATE)"
   "vpternlog<ternlogsuffix>\t{%4, %3, %2, %0|%0, %2, %3, %4}"
   [(set_attr "type" "sselog")
    (set_attr "prefix" "evex")
@@ -11574,11 +11779,11 @@
   [(set (match_operand:V 0 "register_operand")
 	(any_logic:V
 	  (any_logic1:V
-	    (match_operand:V 1 "reg_or_notreg_operand")
-	    (match_operand:V 2 "reg_or_notreg_operand"))
+	    (match_operand:V 1 "regmem_or_bitnot_regmem_operand")
+	    (match_operand:V 2 "regmem_or_bitnot_regmem_operand"))
 	  (any_logic2:V
-	    (match_operand:V 3 "reg_or_notreg_operand")
-	    (match_operand:V 4 "reg_or_notreg_operand"))))]
+	    (match_operand:V 3 "regmem_or_bitnot_regmem_operand")
+	    (match_operand:V 4 "regmem_or_bitnot_regmem_operand"))))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL)
    && ix86_pre_reload_split ()
    && (rtx_equal_p (STRIP_UNARY (operands[1]),
@@ -11647,6 +11852,10 @@
   operands[1] = STRIP_UNARY (operands[1]);
   operands[2] = STRIP_UNARY (operands[2]);
   operands[6] = STRIP_UNARY (operands[6]);
+  if (!register_operand (operands[2], <MODE>mode))
+     operands[2] = force_reg (<MODE>mode, operands[2]);
+  if (!register_operand (operands[6], <MODE>mode))
+     operands[6] = force_reg (<MODE>mode, operands[6]);
   operands[5] = GEN_INT (reg_mask);
 })
 
@@ -11655,10 +11864,10 @@
 	(any_logic:V
 	  (any_logic1:V
 	    (any_logic2:V
-	      (match_operand:V 1 "reg_or_notreg_operand")
-	      (match_operand:V 2 "reg_or_notreg_operand"))
-	    (match_operand:V 3 "reg_or_notreg_operand"))
-	  (match_operand:V 4 "reg_or_notreg_operand")))]
+	      (match_operand:V 1 "regmem_or_bitnot_regmem_operand")
+	      (match_operand:V 2 "regmem_or_bitnot_regmem_operand"))
+	    (match_operand:V 3 "regmem_or_bitnot_regmem_operand"))
+	  (match_operand:V 4 "regmem_or_bitnot_regmem_operand")))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL)
    && ix86_pre_reload_split ()
    && (rtx_equal_p (STRIP_UNARY (operands[1]),
@@ -11728,15 +11937,20 @@
   operands[2] = STRIP_UNARY (operands[2]);
   operands[6] = STRIP_UNARY (operands[6]);
   operands[5] = GEN_INT (reg_mask);
+  if (!register_operand (operands[2], <MODE>mode))
+    operands[2] = force_reg (<MODE>mode, operands[2]);
+  if (!register_operand (operands[6], <MODE>mode))
+    operands[6] = force_reg (<MODE>mode, operands[6]);
+
 })
 
 (define_insn_and_split "*<avx512>_vpternlog<mode>_3"
   [(set (match_operand:V 0 "register_operand")
 	(any_logic:V
 	  (any_logic1:V
-	    (match_operand:V 1 "reg_or_notreg_operand")
-	    (match_operand:V 2 "reg_or_notreg_operand"))
-	  (match_operand:V 3 "reg_or_notreg_operand")))]
+	    (match_operand:V 1 "regmem_or_bitnot_regmem_operand")
+	    (match_operand:V 2 "regmem_or_bitnot_regmem_operand"))
+	  (match_operand:V 3 "regmem_or_bitnot_regmem_operand")))]
   "(<MODE_SIZE> == 64 || TARGET_AVX512VL)
    && ix86_pre_reload_split ()"
   "#"
@@ -11767,6 +11981,10 @@
   operands[2] = STRIP_UNARY (operands[2]);
   operands[3] = STRIP_UNARY (operands[3]);
   operands[4] = GEN_INT (reg_mask);
+  if (!register_operand (operands[2], <MODE>mode))
+    operands[2] = force_reg (<MODE>mode, operands[2]);
+  if (!register_operand (operands[3], <MODE>mode))
+    operands[3] = force_reg (<MODE>mode, operands[3]);
 })
 
 
@@ -14689,19 +14907,37 @@
 (define_mode_attr SDOT_PMADD_SUF
   [(V32HI "512v32hi") (V16HI "") (V8HI "")])
 
+(define_mode_attr SDOT_VPDP_SUF
+  [(V32HI "v16si") (V16HI "v8si") (V8HI "v4si")])
+
 (define_expand "sdot_prod<mode>"
   [(match_operand:<sseunpackmode> 0 "register_operand")
-   (match_operand:VI2_AVX2 1 "register_operand")
-   (match_operand:VI2_AVX2 2 "register_operand")
+   (match_operand:VI2_AVX512VNNIBW 1 "register_operand")
+   (match_operand:VI2_AVX512VNNIBW 2 "register_operand")
    (match_operand:<sseunpackmode> 3 "register_operand")]
   "TARGET_SSE2"
 {
-  rtx t = gen_reg_rtx (<sseunpackmode>mode);
-  emit_insn (gen_<sse2_avx2>_pmaddwd<SDOT_PMADD_SUF> (t, operands[1], operands[2]));
-  emit_insn (gen_rtx_SET (operands[0],
-			  gen_rtx_PLUS (<sseunpackmode>mode,
-					operands[3], t)));
-  DONE;
+  /* Try with vnni instructions.  */
+  if ((<MODE_SIZE> == 64 && TARGET_AVX512VNNI)
+      || (<MODE_SIZE> < 64
+	  && ((TARGET_AVX512VNNI && TARGET_AVX512VL) || TARGET_AVXVNNI)))
+    {
+      operands[1] = lowpart_subreg (<sseunpackmode>mode, operands[1], <MODE>mode);
+      operands[2] = lowpart_subreg (<sseunpackmode>mode, operands[2], <MODE>mode);
+      emit_insn (gen_rtx_SET (operands[0], operands[3]));
+      emit_insn (gen_vpdpwssd_<SDOT_VPDP_SUF> (operands[0], operands[3],
+					       operands[1], operands[2]));
+    }
+    /* Otherwise use pmaddwd + paddd.  */
+    else
+    {
+      rtx t = gen_reg_rtx (<sseunpackmode>mode);
+      emit_insn (gen_<sse2_avx2>_pmaddwd<SDOT_PMADD_SUF> (t, operands[1], operands[2]));
+      emit_insn (gen_rtx_SET (operands[0],
+			      gen_rtx_PLUS (<sseunpackmode>mode,
+					    operands[3], t)));
+    }
+    DONE;
 })
 
 ;; Normally we use widen_mul_even/odd, but combine can't quite get it all
@@ -15023,6 +15259,61 @@
   operands[4] = gen_lowpart (<MODE>mode, operands[3]);
 })
 
+(define_expand "ashlv1ti3"
+  [(set (match_operand:V1TI 0 "register_operand")
+	(ashift:V1TI
+	 (match_operand:V1TI 1 "register_operand")
+	 (match_operand:QI 2 "general_operand")))]
+  "TARGET_SSE2 && TARGET_64BIT"
+{
+  ix86_expand_v1ti_shift (ASHIFT, operands);
+  DONE;
+})
+
+(define_expand "lshrv1ti3"
+  [(set (match_operand:V1TI 0 "register_operand")
+	(lshiftrt:V1TI
+	 (match_operand:V1TI 1 "register_operand")
+	 (match_operand:QI 2 "general_operand")))]
+  "TARGET_SSE2 && TARGET_64BIT"
+{
+  ix86_expand_v1ti_shift (LSHIFTRT, operands);
+  DONE;
+})
+
+(define_expand "ashrv1ti3"
+  [(set (match_operand:V1TI 0 "register_operand")
+	(ashiftrt:V1TI
+	 (match_operand:V1TI 1 "register_operand")
+	 (match_operand:QI 2 "general_operand")))]
+  "TARGET_SSE2 && TARGET_64BIT"
+{
+  ix86_expand_v1ti_ashiftrt (operands);
+  DONE;
+})
+
+(define_expand "rotlv1ti3"
+  [(set (match_operand:V1TI 0 "register_operand")
+	(rotate:V1TI
+	 (match_operand:V1TI 1 "register_operand")
+	 (match_operand:QI 2 "const_int_operand")))]
+  "TARGET_SSE2 && TARGET_64BIT"
+{
+  ix86_expand_v1ti_rotate (ROTATE, operands);
+  DONE;
+})
+
+(define_expand "rotrv1ti3"
+  [(set (match_operand:V1TI 0 "register_operand")
+	(rotatert:V1TI
+	 (match_operand:V1TI 1 "register_operand")
+	 (match_operand:QI 2 "const_int_operand")))]
+  "TARGET_SSE2 && TARGET_64BIT"
+{
+  ix86_expand_v1ti_rotate (ROTATERT, operands);
+  DONE;
+})
+
 (define_insn "avx512bw_<insn><mode>3"
   [(set (match_operand:VIMAX_AVX512VL 0 "register_operand" "=v")
 	(any_lshift:VIMAX_AVX512VL
@@ -16268,6 +16559,31 @@
 	      ]
 	      (const_string "<sseinsnmode>")))])
 
+(define_insn "<code>v1ti3"
+  [(set (match_operand:V1TI 0 "register_operand" "=x,x,v")
+	(any_logic:V1TI
+	  (match_operand:V1TI 1 "register_operand" "%0,x,v")
+	  (match_operand:V1TI 2 "vector_operand" "xBm,xm,vm")))]
+  "TARGET_SSE2"
+  "@
+   p<logic>\t{%2, %0|%0, %2}
+   vp<logic>\t{%2, %1, %0|%0, %1, %2}
+   vp<logic>\t{%2, %1, %0|%0, %1, %2}"
+  [(set_attr "isa" "noavx,avx,avx")
+   (set_attr "prefix" "orig,vex,evex")
+   (set_attr "prefix_data16" "1,*,*")
+   (set_attr "type" "sselog")
+   (set_attr "mode" "TI")])
+
+(define_expand "one_cmplv1ti2"
+  [(set (match_operand:V1TI 0 "register_operand")
+	(xor:V1TI (match_operand:V1TI 1 "register_operand")
+		  (match_dup 2)))]
+  "TARGET_SSE2"
+{
+  operands[2] = force_reg (V1TImode, CONSTM1_RTX (V1TImode));
+})
+
 (define_mode_iterator AVX512ZEXTMASK
   [(DI "TARGET_AVX512BW") (SI "TARGET_AVX512BW") HI])
 
@@ -21758,18 +22074,18 @@
    (set_attr "mode" "TI")])
 
 (define_expand "nearbyint<mode>2"
-  [(set (match_operand:VF 0 "register_operand")
-	(unspec:VF
-	  [(match_operand:VF 1 "vector_operand")
+  [(set (match_operand:VFH 0 "register_operand")
+	(unspec:VFH
+	  [(match_operand:VFH 1 "vector_operand")
 	   (match_dup 2)]
 	  UNSPEC_ROUND))]
   "TARGET_SSE4_1"
   "operands[2] = GEN_INT (ROUND_MXCSR | ROUND_NO_EXC);")
 
 (define_expand "rint<mode>2"
-  [(set (match_operand:VF 0 "register_operand")
-	(unspec:VF
-	  [(match_operand:VF 1 "vector_operand")
+  [(set (match_operand:VFH 0 "register_operand")
+	(unspec:VFH
+	  [(match_operand:VFH 1 "vector_operand")
 	   (match_dup 2)]
 	  UNSPEC_ROUND))]
   "TARGET_SSE4_1"
@@ -26876,6 +27192,29 @@
    [(set_attr ("prefix") ("evex"))
    (set_attr "mode" "<sseinsnmode>")])
 
+(define_mode_attr VI1SI
+ [(V64QI "V16SI") (V32QI "V8SI") (V16QI "V4SI")])
+
+(define_mode_attr vi1si
+ [(V64QI "v16si") (V32QI "v8si") (V16QI "v4si")])
+
+(define_expand "usdot_prod<mode>"
+  [(match_operand:<VI1SI> 0 "register_operand")
+   (match_operand:VI1_AVX512VNNI 1 "register_operand")
+   (match_operand:VI1_AVX512VNNI 2 "register_operand")
+   (match_operand:<VI1SI> 3 "register_operand")]
+  "(<MODE_SIZE> == 64
+    ||((TARGET_AVX512VNNI && TARGET_AVX512VL)
+	    || TARGET_AVXVNNI))"
+{
+  operands[1] = lowpart_subreg (<VI1SI>mode, operands[1], <MODE>mode);
+  operands[2] = lowpart_subreg (<VI1SI>mode, operands[2], <MODE>mode);
+  emit_insn (gen_rtx_SET (operands[0], operands[3]));
+  emit_insn (gen_vpdpbusd_<vi1si> (operands[0], operands[3],
+				  operands[1], operands[2]));
+  DONE;
+})
+
 (define_insn "vpdpbusd_v16si"
   [(set (match_operand:V16SI 0 "register_operand" "=v")
 	(unspec:V16SI
diff --git a/gcc/config/i386/sync.md b/gcc/config/i386/sync.md
index 05a8352..cc4fe72 100644
--- a/gcc/config/i386/sync.md
+++ b/gcc/config/i386/sync.md
@@ -525,6 +525,123 @@
 	      (set (reg:CCZ FLAGS_REG)
 		   (unspec_volatile:CCZ [(const_int 0)] UNSPECV_CMPXCHG))])])
 
+(define_expand "atomic_fetch_<logic><mode>"
+  [(match_operand:SWI124 0 "register_operand")
+   (any_logic:SWI124
+    (match_operand:SWI124 1 "memory_operand")
+    (match_operand:SWI124 2 "register_operand"))
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_CMPXCHG && TARGET_RELAX_CMPXCHG_LOOP"
+{
+  ix86_expand_atomic_fetch_op_loop (operands[0], operands[1],
+				    operands[2], <CODE>, false,
+				    false);
+  DONE;
+})
+
+(define_expand "atomic_<logic>_fetch<mode>"
+  [(match_operand:SWI124 0 "register_operand")
+   (any_logic:SWI124
+    (match_operand:SWI124 1 "memory_operand")
+    (match_operand:SWI124 2 "register_operand"))
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_CMPXCHG && TARGET_RELAX_CMPXCHG_LOOP"
+{
+  ix86_expand_atomic_fetch_op_loop (operands[0], operands[1],
+				    operands[2], <CODE>, true,
+				    false);
+  DONE;
+})
+
+(define_expand "atomic_fetch_nand<mode>"
+  [(match_operand:SWI124 0 "register_operand")
+   (match_operand:SWI124 1 "memory_operand")
+   (match_operand:SWI124 2 "register_operand")
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_CMPXCHG && TARGET_RELAX_CMPXCHG_LOOP"
+{
+  ix86_expand_atomic_fetch_op_loop (operands[0], operands[1],
+				    operands[2], NOT, false,
+				    false);
+  DONE;
+})
+
+(define_expand "atomic_nand_fetch<mode>"
+  [(match_operand:SWI124 0 "register_operand")
+   (match_operand:SWI124 1 "memory_operand")
+   (match_operand:SWI124 2 "register_operand")
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_CMPXCHG && TARGET_RELAX_CMPXCHG_LOOP"
+{
+  ix86_expand_atomic_fetch_op_loop (operands[0], operands[1],
+				    operands[2], NOT, true,
+				    false);
+  DONE;
+})
+
+(define_expand "atomic_fetch_<logic><mode>"
+  [(match_operand:CASMODE 0 "register_operand")
+   (any_logic:CASMODE
+    (match_operand:CASMODE 1 "memory_operand")
+    (match_operand:CASMODE 2 "register_operand"))
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_CMPXCHG && TARGET_RELAX_CMPXCHG_LOOP"
+{
+  bool doubleword = (<MODE>mode == DImode && !TARGET_64BIT)
+		    || (<MODE>mode == TImode);
+  ix86_expand_atomic_fetch_op_loop (operands[0], operands[1],
+				    operands[2], <CODE>, false,
+				    doubleword);
+  DONE;
+})
+
+(define_expand "atomic_<logic>_fetch<mode>"
+  [(match_operand:CASMODE 0 "register_operand")
+   (any_logic:CASMODE
+    (match_operand:CASMODE 1 "memory_operand")
+    (match_operand:CASMODE 2 "register_operand"))
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_CMPXCHG && TARGET_RELAX_CMPXCHG_LOOP"
+{
+  bool doubleword = (<MODE>mode == DImode && !TARGET_64BIT)
+		    || (<MODE>mode == TImode);
+  ix86_expand_atomic_fetch_op_loop (operands[0], operands[1],
+				    operands[2], <CODE>, true,
+				    doubleword);
+  DONE;
+})
+
+(define_expand "atomic_fetch_nand<mode>"
+  [(match_operand:CASMODE 0 "register_operand")
+   (match_operand:CASMODE 1 "memory_operand")
+   (match_operand:CASMODE 2 "register_operand")
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_CMPXCHG && TARGET_RELAX_CMPXCHG_LOOP"
+{
+  bool doubleword = (<MODE>mode == DImode && !TARGET_64BIT)
+		    || (<MODE>mode == TImode);
+  ix86_expand_atomic_fetch_op_loop (operands[0], operands[1],
+				    operands[2], NOT, false,
+				    doubleword);
+  DONE;
+})
+
+(define_expand "atomic_nand_fetch<mode>"
+  [(match_operand:CASMODE 0 "register_operand")
+   (match_operand:CASMODE 1 "memory_operand")
+   (match_operand:CASMODE 2 "register_operand")
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_CMPXCHG && TARGET_RELAX_CMPXCHG_LOOP"
+{
+  bool doubleword = (<MODE>mode == DImode && !TARGET_64BIT)
+		    || (<MODE>mode == TImode);
+  ix86_expand_atomic_fetch_op_loop (operands[0], operands[1],
+				    operands[2], NOT, true,
+				    doubleword);
+  DONE;
+})
+
+
 ;; For operand 2 nonmemory_operand predicate is used instead of
 ;; register_operand to allow combiner to better optimize atomic
 ;; additions of constants.
@@ -726,7 +843,7 @@
   rtx result = convert_modes (<MODE>mode, QImode, tem, 1);
   if (operands[4] == const0_rtx)
     result = expand_simple_binop (<MODE>mode, ASHIFT, result,
-				  operands[2], operands[0], 0, OPTAB_DIRECT);
+				  operands[2], operands[0], 0, OPTAB_WIDEN);
   if (result != operands[0])
     emit_move_insn (operands[0], result);
   DONE;
@@ -763,7 +880,7 @@
   rtx result = convert_modes (<MODE>mode, QImode, tem, 1);
   if (operands[4] == const0_rtx)
     result = expand_simple_binop (<MODE>mode, ASHIFT, result,
-				  operands[2], operands[0], 0, OPTAB_DIRECT);
+				  operands[2], operands[0], 0, OPTAB_WIDEN);
   if (result != operands[0])
     emit_move_insn (operands[0], result);
   DONE;
@@ -801,7 +918,7 @@
   rtx result = convert_modes (<MODE>mode, QImode, tem, 1);
   if (operands[4] == const0_rtx)
     result = expand_simple_binop (<MODE>mode, ASHIFT, result,
-				  operands[2], operands[0], 0, OPTAB_DIRECT);
+				  operands[2], operands[0], 0, OPTAB_WIDEN);
   if (result != operands[0])
     emit_move_insn (operands[0], result);
   DONE;
diff --git a/gcc/config/i386/x86-tune-costs.h b/gcc/config/i386/x86-tune-costs.h
index 93644be..dd5563d 100644
--- a/gcc/config/i386/x86-tune-costs.h
+++ b/gcc/config/i386/x86-tune-costs.h
@@ -2070,6 +2070,126 @@ struct processor_costs icelake_cost = {
   "16",					/* Func alignment.  */
 };
 
+/* alderlake_cost should produce code tuned for alderlake family of CPUs.  */
+static stringop_algs alderlake_memcpy[2] = {
+  {libcall,
+   {{256, rep_prefix_1_byte, true},
+    {256, loop, false},
+    {-1, libcall, false}}},
+  {libcall,
+   {{256, rep_prefix_1_byte, true},
+    {256, loop, false},
+    {-1, libcall, false}}}};
+static stringop_algs alderlake_memset[2] = {
+  {libcall,
+   {{256, rep_prefix_1_byte, true},
+    {256, loop, false},
+    {-1, libcall, false}}},
+  {libcall,
+   {{256, rep_prefix_1_byte, true},
+    {256, loop, false},
+    {-1, libcall, false}}}};
+static const
+struct processor_costs alderlake_cost = {
+  {
+  /* Start of register allocator costs.  integer->integer move cost is 2.  */
+  6,				     /* cost for loading QImode using movzbl */
+  {6, 6, 6},				/* cost of loading integer registers
+					   in QImode, HImode and SImode.
+					   Relative to reg-reg move (2).  */
+  {6, 6, 6},				/* cost of storing integer registers */
+  4,					/* cost of reg,reg fld/fst */
+  {6, 6, 12},				/* cost of loading fp registers
+					   in SFmode, DFmode and XFmode */
+  {6, 6, 12},				/* cost of storing fp registers
+					   in SFmode, DFmode and XFmode */
+  2,					/* cost of moving MMX register */
+  {6, 6},				/* cost of loading MMX registers
+					   in SImode and DImode */
+  {6, 6},				/* cost of storing MMX registers
+					   in SImode and DImode */
+  2, 3, 4,				/* cost of moving XMM,YMM,ZMM register */
+  {6, 6, 6, 10, 15},			/* cost of loading SSE registers
+					   in 32,64,128,256 and 512-bit */
+  {6, 6, 6, 10, 15},			/* cost of storing SSE registers
+					   in 32,64,128,256 and 512-bit */
+  6, 6,				/* SSE->integer and integer->SSE moves */
+  6, 6,				/* mask->integer and integer->mask moves */
+  {6, 6, 6},				/* cost of loading mask register
+					   in QImode, HImode, SImode.  */
+  {6, 6, 6},			/* cost if storing mask register
+					   in QImode, HImode, SImode.  */
+  2,					/* cost of moving mask register.  */
+  /* End of register allocator costs.  */
+  },
+
+  COSTS_N_INSNS (1),			/* cost of an add instruction */
+  COSTS_N_INSNS (1) + 1,		/* cost of a lea instruction */
+  COSTS_N_INSNS (1),			/* variable shift costs */
+  COSTS_N_INSNS (1),			/* constant shift costs */
+  {COSTS_N_INSNS (3),			/* cost of starting multiply for QI */
+   COSTS_N_INSNS (4),			/*				 HI */
+   COSTS_N_INSNS (3),			/*				 SI */
+   COSTS_N_INSNS (4),			/*				 DI */
+   COSTS_N_INSNS (4)},			/*			      other */
+  0,					/* cost of multiply per each bit set */
+  {COSTS_N_INSNS (16),			/* cost of a divide/mod for QI */
+   COSTS_N_INSNS (22),			/*			    HI */
+   COSTS_N_INSNS (30),			/*			    SI */
+   COSTS_N_INSNS (74),			/*			    DI */
+   COSTS_N_INSNS (74)},			/*			    other */
+  COSTS_N_INSNS (1),			/* cost of movsx */
+  COSTS_N_INSNS (1),			/* cost of movzx */
+  8,					/* "large" insn */
+  17,					/* MOVE_RATIO */
+  17,					/* CLEAR_RATIO */
+  {6, 6, 6},				/* cost of loading integer registers
+					   in QImode, HImode and SImode.
+					   Relative to reg-reg move (2).  */
+  {6, 6, 6},				/* cost of storing integer registers */
+  {6, 6, 6, 10, 15},			/* cost of loading SSE register
+					   in 32bit, 64bit, 128bit, 256bit and 512bit */
+  {6, 6, 6, 10, 15},			/* cost of storing SSE register
+					   in 32bit, 64bit, 128bit, 256bit and 512bit */
+  {6, 6, 6, 10, 15},			/* cost of unaligned loads.  */
+  {6, 6, 6, 10, 15},			/* cost of unaligned storess.  */
+  2, 3, 4,				/* cost of moving XMM,YMM,ZMM register */
+  6,					/* cost of moving SSE register to integer.  */
+  18, 6,				/* Gather load static, per_elt.  */
+  18, 6,				/* Gather store static, per_elt.  */
+  32,					/* size of l1 cache.  */
+  512,					/* size of l2 cache.  */
+  64,					/* size of prefetch block */
+  6,					/* number of parallel prefetches */
+  3,					/* Branch cost */
+  COSTS_N_INSNS (3),			/* cost of FADD and FSUB insns.  */
+  COSTS_N_INSNS (5),			/* cost of FMUL instruction.  */
+  COSTS_N_INSNS (17),			/* cost of FDIV instruction.  */
+  COSTS_N_INSNS (1),			/* cost of FABS instruction.  */
+  COSTS_N_INSNS (1),			/* cost of FCHS instruction.  */
+  COSTS_N_INSNS (14),			/* cost of FSQRT instruction.  */
+
+  COSTS_N_INSNS (1),			/* cost of cheap SSE instruction.  */
+  COSTS_N_INSNS (3),			/* cost of ADDSS/SD SUBSS/SD insns.  */
+  COSTS_N_INSNS (4),			/* cost of MULSS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of MULSD instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SS instruction.  */
+  COSTS_N_INSNS (5),			/* cost of FMA SD instruction.  */
+  COSTS_N_INSNS (13),			/* cost of DIVSS instruction.  */
+  COSTS_N_INSNS (17),			/* cost of DIVSD instruction.  */
+  COSTS_N_INSNS (14),			/* cost of SQRTSS instruction.  */
+  COSTS_N_INSNS (18),			/* cost of SQRTSD instruction.  */
+  1, 4, 3, 3,				/* reassoc int, fp, vec_int, vec_fp.  */
+  alderlake_memcpy,
+  alderlake_memset,
+  COSTS_N_INSNS (4),			/* cond_taken_branch_cost.  */
+  COSTS_N_INSNS (2),			/* cond_not_taken_branch_cost.  */
+  "16:11:8",				/* Loop alignment.  */
+  "16:11:8",				/* Jump alignment.  */
+  "0:0:8",				/* Label alignment.  */
+  "16",					/* Func alignment.  */
+};
+
   /* BTVER1 has optimized REP instruction for medium sized blocks, but for
      very small blocks it is better to use loop. For large blocks, libcall can
      do nontemporary accesses and beat inline considerably.  */
diff --git a/gcc/config/i386/x86-tune-sched-bd.c b/gcc/config/i386/x86-tune-sched-bd.c
index ad0edf7..be38e48 100644
--- a/gcc/config/i386/x86-tune-sched-bd.c
+++ b/gcc/config/i386/x86-tune-sched-bd.c
@@ -67,7 +67,7 @@ along with GCC; see the file COPYING3.  If not see
 #define BIG 100
 
 
-/* Dispatch groups.  Istructions that affect the mix in a dispatch window.  */
+/* Dispatch groups.  Instructions that affect the mix in a dispatch window.  */
 enum dispatch_group {
   disp_no_group = 0,
   disp_load,
diff --git a/gcc/config/i386/x86-tune-sched.c b/gcc/config/i386/x86-tune-sched.c
index 56ada99..0c149a0 100644
--- a/gcc/config/i386/x86-tune-sched.c
+++ b/gcc/config/i386/x86-tune-sched.c
@@ -72,6 +72,7 @@ ix86_issue_rate (void)
     case PROCESSOR_SANDYBRIDGE:
     case PROCESSOR_HASWELL:
     case PROCESSOR_TREMONT:
+    case PROCESSOR_ALDERLAKE:
     case PROCESSOR_GENERIC:
       return 4;
 
@@ -431,6 +432,7 @@ ix86_adjust_cost (rtx_insn *insn, int dep_type, rtx_insn *dep_insn, int cost,
     case PROCESSOR_SANDYBRIDGE:
     case PROCESSOR_HASWELL:
     case PROCESSOR_TREMONT:
+    case PROCESSOR_ALDERLAKE:
     case PROCESSOR_GENERIC:
       /* Stack engine allows to execute push&pop instructions in parall.  */
       if ((insn_type == TYPE_PUSH || insn_type == TYPE_POP)
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index 58e8ead..4ae0b56 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -42,7 +42,7 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 DEF_TUNE (X86_TUNE_SCHEDULE, "schedule",
           m_PENT | m_LAKEMONT | m_PPRO | m_CORE_ALL | m_BONNELL | m_SILVERMONT
 	  | m_INTEL | m_KNL | m_KNM | m_K6_GEODE | m_AMD_MULTIPLE | m_GOLDMONT
-	  | m_GOLDMONT_PLUS | m_TREMONT | m_GENERIC)
+	  | m_GOLDMONT_PLUS | m_TREMONT | m_ALDERLAKE |m_GENERIC)
 
 /* X86_TUNE_PARTIAL_REG_DEPENDENCY: Enable more register renaming
    on modern chips.  Preffer stores affecting whole integer register
@@ -51,7 +51,7 @@ DEF_TUNE (X86_TUNE_SCHEDULE, "schedule",
 DEF_TUNE (X86_TUNE_PARTIAL_REG_DEPENDENCY, "partial_reg_dependency",
           m_P4_NOCONA | m_CORE2 | m_NEHALEM  | m_SANDYBRIDGE | m_CORE_AVX2
 	  | m_BONNELL | m_SILVERMONT | m_GOLDMONT | m_GOLDMONT_PLUS | m_INTEL
-	  | m_KNL | m_KNM | m_AMD_MULTIPLE | m_TREMONT
+	  | m_KNL | m_KNM | m_AMD_MULTIPLE | m_TREMONT | m_ALDERLAKE
 	  | m_GENERIC)
 
 /* X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY: This knob promotes all store
@@ -62,7 +62,7 @@ DEF_TUNE (X86_TUNE_PARTIAL_REG_DEPENDENCY, "partial_reg_dependency",
    that can be partly masked by careful scheduling of moves.  */
 DEF_TUNE (X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY, "sse_partial_reg_dependency",
           m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_BONNELL | m_AMDFAM10
-	  | m_BDVER | m_ZNVER | m_TREMONT | m_GENERIC)
+	  | m_BDVER | m_ZNVER | m_TREMONT | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_SSE_PARTIAL_REG_FP_CONVERTS_DEPENDENCY: This knob avoids
    partial write to the destination in scalar SSE conversion from FP
@@ -70,14 +70,14 @@ DEF_TUNE (X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY, "sse_partial_reg_dependency",
 DEF_TUNE (X86_TUNE_SSE_PARTIAL_REG_FP_CONVERTS_DEPENDENCY,
 	  "sse_partial_reg_fp_converts_dependency",
 	  m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_BONNELL | m_AMDFAM10
-	  | m_BDVER | m_ZNVER | m_GENERIC)
+	  | m_BDVER | m_ZNVER | m_ALDERLAKE| m_GENERIC)
 
 /* X86_TUNE_SSE_PARTIAL_REG_CONVERTS_DEPENDENCY: This knob avoids partial
    write to the destination in scalar SSE conversion from integer to FP.  */
 DEF_TUNE (X86_TUNE_SSE_PARTIAL_REG_CONVERTS_DEPENDENCY,
 	  "sse_partial_reg_converts_dependency",
 	  m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_BONNELL | m_AMDFAM10
-	  | m_BDVER | m_ZNVER | m_GENERIC)
+	  | m_BDVER | m_ZNVER | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_SSE_SPLIT_REGS: Set for machines where the type and dependencies
    are resolved on SSE register parts instead of whole registers, so we may
@@ -103,14 +103,14 @@ DEF_TUNE (X86_TUNE_MOVX, "movx",
           m_PPRO | m_P4_NOCONA | m_CORE2 | m_NEHALEM  | m_SANDYBRIDGE
 	  | m_BONNELL | m_SILVERMONT | m_GOLDMONT | m_KNL | m_KNM | m_INTEL
 	  | m_GOLDMONT_PLUS | m_GEODE | m_AMD_MULTIPLE
-	  | m_CORE_AVX2 | m_TREMONT | m_GENERIC)
+	  | m_CORE_AVX2 | m_TREMONT | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_MEMORY_MISMATCH_STALL: Avoid partial stores that are followed by
    full sized loads.  */
 DEF_TUNE (X86_TUNE_MEMORY_MISMATCH_STALL, "memory_mismatch_stall",
           m_P4_NOCONA | m_CORE_ALL | m_BONNELL | m_SILVERMONT | m_INTEL
 	  | m_KNL | m_KNM | m_GOLDMONT | m_GOLDMONT_PLUS | m_AMD_MULTIPLE
-	  | m_TREMONT | m_GENERIC)
+	  | m_TREMONT | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_FUSE_CMP_AND_BRANCH_32: Fuse compare with a subsequent
    conditional jump instruction for 32 bit TARGET.  */
@@ -166,14 +166,14 @@ DEF_TUNE (X86_TUNE_EPILOGUE_USING_MOVE, "epilogue_using_move",
 /* X86_TUNE_USE_LEAVE: Use "leave" instruction in epilogues where it fits.  */
 DEF_TUNE (X86_TUNE_USE_LEAVE, "use_leave",
 	  m_386 | m_CORE_ALL | m_K6_GEODE | m_AMD_MULTIPLE | m_TREMONT
-	  | m_GENERIC)
+	  | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_PUSH_MEMORY: Enable generation of "push mem" instructions.
    Some chips, like 486 and Pentium works faster with separate load
    and push instructions.  */
 DEF_TUNE (X86_TUNE_PUSH_MEMORY, "push_memory",
           m_386 | m_P4_NOCONA | m_CORE_ALL | m_K6_GEODE | m_AMD_MULTIPLE
-          | m_TREMONT | m_GENERIC)
+	  | m_TREMONT | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_SINGLE_PUSH: Enable if single push insn is preferred
    over esp subtraction.  */
@@ -243,14 +243,14 @@ DEF_TUNE (X86_TUNE_READ_MODIFY, "read_modify", ~(m_PENT | m_LAKEMONT | m_PPRO))
 DEF_TUNE (X86_TUNE_USE_INCDEC, "use_incdec",
           ~(m_P4_NOCONA | m_CORE2 | m_NEHALEM  | m_SANDYBRIDGE
 	    | m_BONNELL | m_SILVERMONT | m_INTEL |  m_KNL | m_KNM | m_GOLDMONT
-	    | m_GOLDMONT_PLUS | m_TREMONT | m_GENERIC))
+	    | m_GOLDMONT_PLUS | m_TREMONT | m_ALDERLAKE | m_GENERIC))
 
 /* X86_TUNE_INTEGER_DFMODE_MOVES: Enable if integer moves are preferred
    for DFmode copies */
 DEF_TUNE (X86_TUNE_INTEGER_DFMODE_MOVES, "integer_dfmode_moves",
           ~(m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_BONNELL | m_SILVERMONT
 	    | m_KNL | m_KNM | m_INTEL | m_GEODE | m_AMD_MULTIPLE | m_GOLDMONT
-	    | m_GOLDMONT_PLUS | m_TREMONT | m_GENERIC))
+	    | m_GOLDMONT_PLUS | m_TREMONT | m_ALDERLAKE | m_GENERIC))
 
 /* X86_TUNE_OPT_AGU: Optimize for Address Generation Unit. This flag
    will impact LEA instruction selection. */
@@ -298,14 +298,14 @@ DEF_TUNE (X86_TUNE_PREFER_KNOWN_REP_MOVSB_STOSB,
 DEF_TUNE (X86_TUNE_MISALIGNED_MOVE_STRING_PRO_EPILOGUES,
 	  "misaligned_move_string_pro_epilogues",
 	  m_386 | m_486 | m_CORE_ALL | m_AMD_MULTIPLE | m_TREMONT
-	  | m_GENERIC)
+	  | m_ALDERLAKE |m_GENERIC)
 
 /* X86_TUNE_USE_SAHF: Controls use of SAHF.  */
 DEF_TUNE (X86_TUNE_USE_SAHF, "use_sahf",
           m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_BONNELL | m_SILVERMONT
 	  | m_KNL | m_KNM | m_INTEL | m_K6_GEODE | m_K8 | m_AMDFAM10 | m_BDVER
 	  | m_BTVER | m_ZNVER | m_GOLDMONT | m_GOLDMONT_PLUS | m_TREMONT
-	  | m_GENERIC)
+	  | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_USE_CLTD: Controls use of CLTD and CTQO instructions.  */
 DEF_TUNE (X86_TUNE_USE_CLTD, "use_cltd",
@@ -316,12 +316,12 @@ DEF_TUNE (X86_TUNE_USE_CLTD, "use_cltd",
 DEF_TUNE (X86_TUNE_USE_BT, "use_bt",
           m_CORE_ALL | m_BONNELL | m_SILVERMONT | m_KNL | m_KNM | m_INTEL
 	  | m_LAKEMONT | m_AMD_MULTIPLE | m_GOLDMONT | m_GOLDMONT_PLUS
-	  | m_TREMONT | m_GENERIC)
+	  | m_TREMONT | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_AVOID_FALSE_DEP_FOR_BMI: Avoid false dependency
    for bit-manipulation instructions.  */
 DEF_TUNE (X86_TUNE_AVOID_FALSE_DEP_FOR_BMI, "avoid_false_dep_for_bmi",
-	  m_SANDYBRIDGE | m_CORE_AVX2 | m_TREMONT | m_GENERIC)
+	  m_SANDYBRIDGE | m_CORE_AVX2 | m_TREMONT | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_ADJUST_UNROLL: This enables adjusting the unroll factor based
    on hardware capabilities. Bdver3 hardware has a loop buffer which makes
@@ -333,11 +333,11 @@ DEF_TUNE (X86_TUNE_ADJUST_UNROLL, "adjust_unroll_factor", m_BDVER3 | m_BDVER4)
    if-converted sequence to one.  */
 DEF_TUNE (X86_TUNE_ONE_IF_CONV_INSN, "one_if_conv_insn",
 	  m_SILVERMONT | m_KNL | m_KNM | m_INTEL | m_CORE_ALL | m_GOLDMONT
-	  | m_GOLDMONT_PLUS | m_TREMONT | m_GENERIC)
+	  | m_GOLDMONT_PLUS | m_TREMONT | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_AVOID_MFENCE: Use lock prefixed instructions instead of mfence.  */
 DEF_TUNE (X86_TUNE_AVOID_MFENCE, "avoid_mfence",
-	 m_CORE_ALL | m_BDVER | m_ZNVER | m_TREMONT | m_GENERIC)
+	 m_CORE_ALL | m_BDVER | m_ZNVER | m_TREMONT | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_EXPAND_ABS: This enables a new abs pattern by
    generating instructions for abs (x) = (((signed) x >> (W-1) ^ x) -
@@ -361,7 +361,8 @@ DEF_TUNE (X86_TUNE_USE_HIMODE_FIOP, "use_himode_fiop",
 DEF_TUNE (X86_TUNE_USE_SIMODE_FIOP, "use_simode_fiop",
           ~(m_PENT | m_LAKEMONT | m_PPRO | m_CORE_ALL | m_BONNELL
 	    | m_SILVERMONT | m_KNL | m_KNM | m_INTEL | m_AMD_MULTIPLE
-	    | m_GOLDMONT | m_GOLDMONT_PLUS | m_TREMONT | m_GENERIC))
+	    | m_GOLDMONT | m_GOLDMONT_PLUS | m_TREMONT | m_ALDERLAKE
+	    | m_GENERIC))
 
 /* X86_TUNE_USE_FFREEP: Use freep instruction instead of fstp.  */
 DEF_TUNE (X86_TUNE_USE_FFREEP, "use_ffreep", m_AMD_MULTIPLE)
@@ -370,7 +371,7 @@ DEF_TUNE (X86_TUNE_USE_FFREEP, "use_ffreep", m_AMD_MULTIPLE)
 DEF_TUNE (X86_TUNE_EXT_80387_CONSTANTS, "ext_80387_constants",
           m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_BONNELL | m_SILVERMONT
 	  | m_KNL | m_KNM | m_INTEL | m_K6_GEODE | m_ATHLON_K8 | m_GOLDMONT
-	  | m_GOLDMONT_PLUS | m_TREMONT | m_GENERIC)
+	  | m_GOLDMONT_PLUS | m_TREMONT | m_ALDERLAKE | m_GENERIC)
 
 /*****************************************************************************/
 /* SSE instruction selection tuning                                          */
@@ -385,15 +386,15 @@ DEF_TUNE (X86_TUNE_GENERAL_REGS_SSE_SPILL, "general_regs_sse_spill",
    of a sequence loading registers by parts.  */
 DEF_TUNE (X86_TUNE_SSE_UNALIGNED_LOAD_OPTIMAL, "sse_unaligned_load_optimal",
 	  m_NEHALEM | m_SANDYBRIDGE | m_CORE_AVX2 | m_SILVERMONT | m_KNL | m_KNM
-	  | m_INTEL | m_GOLDMONT | m_GOLDMONT_PLUS
-	  | m_TREMONT | m_AMDFAM10 | m_BDVER | m_BTVER | m_ZNVER | m_GENERIC)
+	  | m_INTEL | m_GOLDMONT | m_GOLDMONT_PLUS | m_TREMONT | m_ALDERLAKE
+	  | m_AMDFAM10 | m_BDVER | m_BTVER | m_ZNVER | m_GENERIC)
 
 /* X86_TUNE_SSE_UNALIGNED_STORE_OPTIMAL: Use movups for misaligned stores
    instead of a sequence loading registers by parts.  */
 DEF_TUNE (X86_TUNE_SSE_UNALIGNED_STORE_OPTIMAL, "sse_unaligned_store_optimal",
 	  m_NEHALEM | m_SANDYBRIDGE | m_CORE_AVX2 | m_SILVERMONT | m_KNL | m_KNM
 	  | m_INTEL | m_GOLDMONT | m_GOLDMONT_PLUS
-	  | m_TREMONT | m_BDVER | m_ZNVER | m_GENERIC)
+	  | m_TREMONT | m_ALDERLAKE | m_BDVER | m_ZNVER | m_GENERIC)
 
 /* X86_TUNE_SSE_PACKED_SINGLE_INSN_OPTIMAL: Use packed single
    precision 128bit instructions instead of double where possible.   */
@@ -402,13 +403,13 @@ DEF_TUNE (X86_TUNE_SSE_PACKED_SINGLE_INSN_OPTIMAL, "sse_packed_single_insn_optim
 
 /* X86_TUNE_SSE_TYPELESS_STORES: Always movaps/movups for 128bit stores.   */
 DEF_TUNE (X86_TUNE_SSE_TYPELESS_STORES, "sse_typeless_stores",
-	  m_AMD_MULTIPLE | m_CORE_ALL | m_TREMONT | m_GENERIC)
+	  m_AMD_MULTIPLE | m_CORE_ALL | m_TREMONT | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_SSE_LOAD0_BY_PXOR: Always use pxor to load0 as opposed to
    xorps/xorpd and other variants.  */
 DEF_TUNE (X86_TUNE_SSE_LOAD0_BY_PXOR, "sse_load0_by_pxor",
 	  m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_BDVER | m_BTVER | m_ZNVER
-	  | m_TREMONT | m_GENERIC)
+	  | m_TREMONT | m_ALDERLAKE | m_GENERIC)
 
 /* X86_TUNE_INTER_UNIT_MOVES_TO_VEC: Enable moves in from integer
    to SSE registers.  If disabled, the moves will be done by storing
@@ -454,11 +455,12 @@ DEF_TUNE (X86_TUNE_SLOW_PSHUFB, "slow_pshufb",
 
 /* X86_TUNE_AVOID_4BYTE_PREFIXES: Avoid instructions requiring 4+ bytes of prefixes.  */
 DEF_TUNE (X86_TUNE_AVOID_4BYTE_PREFIXES, "avoid_4byte_prefixes",
-	  m_SILVERMONT | m_GOLDMONT | m_GOLDMONT_PLUS | m_TREMONT | m_INTEL)
+	  m_SILVERMONT | m_GOLDMONT | m_GOLDMONT_PLUS | m_TREMONT | m_ALDERLAKE
+	  | m_INTEL)
 
 /* X86_TUNE_USE_GATHER: Use gather instructions.  */
 DEF_TUNE (X86_TUNE_USE_GATHER, "use_gather",
-	  ~(m_ZNVER1 | m_ZNVER2 | m_GENERIC))
+	  ~(m_ZNVER1 | m_ZNVER2 | m_GENERIC | m_ALDERLAKE))
 
 /* X86_TUNE_AVOID_128FMA_CHAINS: Avoid creating loops with tight 128bit or
    smaller FMA chain.  */
@@ -503,12 +505,12 @@ DEF_TUNE (X86_TUNE_AVX256_OPTIMAL, "avx256_optimal", m_CORE_AVX512)
 /* X86_TUNE_AVX256_MOVE_BY_PIECES: Optimize move_by_pieces with 256-bit
    AVX instructions.  */
 DEF_TUNE (X86_TUNE_AVX256_MOVE_BY_PIECES, "avx256_move_by_pieces",
-	  m_ALDERLAKE | m_CORE_AVX512)
+	  m_CORE_AVX512)
 
 /* X86_TUNE_AVX256_STORE_BY_PIECES: Optimize store_by_pieces with 256-bit
    AVX instructions.  */
 DEF_TUNE (X86_TUNE_AVX256_STORE_BY_PIECES, "avx256_store_by_pieces",
-	  m_ALDERLAKE | m_CORE_AVX512)
+	  m_CORE_AVX512)
 
 /*****************************************************************************/
 /*****************************************************************************/
diff --git a/gcc/config/nios2/linux.h b/gcc/config/nios2/linux.h
index 08edf152..15696d8 100644
--- a/gcc/config/nios2/linux.h
+++ b/gcc/config/nios2/linux.h
@@ -30,6 +30,7 @@
 #define CPP_SPEC "%{posix:-D_POSIX_SOURCE} %{pthread:-D_REENTRANT}"
 
 #define GLIBC_DYNAMIC_LINKER "/lib/ld-linux-nios2.so.1"
+#define MUSL_DYNAMIC_LINKER  "/lib/ld-musl-nios2.so.1"
 
 #undef LINK_SPEC
 #define LINK_SPEC LINK_SPEC_ENDIAN \
diff --git a/gcc/config/nios2/nios2.c b/gcc/config/nios2/nios2.c
index 7a613e4..f77f7e6 100644
--- a/gcc/config/nios2/nios2.c
+++ b/gcc/config/nios2/nios2.c
@@ -4162,6 +4162,40 @@ nios2_option_restore (struct gcc_options *opts ATTRIBUTE_UNUSED,
 	  sizeof (custom_code_index));
 }
 
+static bool
+nios2_can_inline_p (tree caller, tree callee)
+{
+  tree callee_opts = DECL_FUNCTION_SPECIFIC_TARGET (callee);
+  tree caller_opts = DECL_FUNCTION_SPECIFIC_TARGET (caller);
+  struct cl_target_option *callee_ptr, *caller_ptr;
+  unsigned int i;
+
+  if (! callee_opts)
+    callee_opts = target_option_default_node;
+  if (! caller_opts)
+    caller_opts = target_option_default_node;
+
+  /* If both caller and callee have attributes, assume that if the
+     pointer is different, the two functions have different target
+     options since build_target_option_node uses a hash table for the
+     options.  */
+  if (callee_opts == caller_opts)
+    return true;
+
+  /* The only target options we recognize via function attributes are
+     those related to custom instructions.  If we failed the above test,
+     check that any custom instructions enabled in the callee are also
+     enabled with the same value in the caller.  */
+  callee_ptr = TREE_TARGET_OPTION (callee_opts);
+  caller_ptr = TREE_TARGET_OPTION (caller_opts);
+  for (i = 0; i < ARRAY_SIZE (nios2_fpu_insn); i++)
+    if (callee_ptr->saved_fpu_custom_code[i] != -1
+	&& (callee_ptr->saved_fpu_custom_code[i]
+	    != caller_ptr->saved_fpu_custom_code[i]))
+      return false;
+  return true;
+}
+
 /* Inner function to process the attribute((target(...))), take an argument and
    set the current options from the argument.  If we have a list, recursively
    go over the list.  */
@@ -5554,6 +5588,9 @@ nios2_adjust_reg_alloc_order (void)
 #undef TARGET_OPTION_RESTORE
 #define TARGET_OPTION_RESTORE nios2_option_restore
 
+#undef TARGET_CAN_INLINE_P
+#define TARGET_CAN_INLINE_P nios2_can_inline_p
+
 #undef TARGET_SET_CURRENT_FUNCTION
 #define TARGET_SET_CURRENT_FUNCTION nios2_set_current_function
 
diff --git a/gcc/config/or1k/or1k-protos.h b/gcc/config/or1k/or1k-protos.h
index bbb54c8..56554f2 100644
--- a/gcc/config/or1k/or1k-protos.h
+++ b/gcc/config/or1k/or1k-protos.h
@@ -20,6 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 extern HOST_WIDE_INT or1k_initial_elimination_offset (int, int);
 extern void or1k_expand_prologue (void);
 extern void or1k_expand_epilogue (void);
+extern void or1k_profile_hook (void);
 extern void or1k_expand_eh_return (rtx);
 extern rtx  or1k_initial_frame_addr (void);
 extern rtx  or1k_dynamic_chain_addr (rtx);
diff --git a/gcc/config/or1k/or1k.c b/gcc/config/or1k/or1k.c
index 27d3fa1..bb7270c 100644
--- a/gcc/config/or1k/or1k.c
+++ b/gcc/config/or1k/or1k.c
@@ -73,6 +73,10 @@ struct GTY(()) machine_function
 
   /* Remember where the set_got_placeholder is located.  */
   rtx_insn *set_got_insn;
+
+  /* Remember where mcount args are stored so we can insert set_got_insn
+     after.  */
+  rtx_insn *set_mcount_arg_insn;
 };
 
 /* Zero initialization is OK for all current fields.  */
@@ -415,6 +419,25 @@ or1k_expand_epilogue (void)
 			   EH_RETURN_STACKADJ_RTX));
 }
 
+/* Worker for PROFILE_HOOK.
+   The OpenRISC profile hook uses the link register which will get clobbered by
+   the GOT setup RTX.  This sets up a placeholder to allow injecting of the GOT
+   setup RTX to avoid clobbering.  */
+
+void
+or1k_profile_hook (void)
+{
+  rtx a1 = gen_rtx_REG (Pmode, 3);
+  rtx ra = get_hard_reg_initial_val (Pmode, LR_REGNUM);
+  rtx fun = gen_rtx_SYMBOL_REF (Pmode, "_mcount");
+
+  /* Keep track of where we setup the _mcount argument so we can insert the
+     GOT setup RTX after it.  */
+  cfun->machine->set_mcount_arg_insn = emit_move_insn (a1, ra);
+
+  emit_library_call (fun, LCT_NORMAL, VOIDmode, a1, Pmode);
+}
+
 /* Worker for TARGET_INIT_PIC_REG.
    Initialize the cfun->machine->set_got_insn rtx and insert it at the entry
    of the current function.  The rtx is just a temporary placeholder for
@@ -423,17 +446,25 @@ or1k_expand_epilogue (void)
 static void
 or1k_init_pic_reg (void)
 {
-  start_sequence ();
 
-  cfun->machine->set_got_insn
-    = emit_insn (gen_set_got_tmp (pic_offset_table_rtx));
+  if (crtl->profile)
+    cfun->machine->set_got_insn =
+      emit_insn_after (gen_set_got_tmp (pic_offset_table_rtx),
+		       cfun->machine->set_mcount_arg_insn);
+  else
+    {
+      start_sequence ();
+
+      cfun->machine->set_got_insn =
+	emit_insn (gen_set_got_tmp (pic_offset_table_rtx));
 
-  rtx_insn *seq = get_insns ();
-  end_sequence ();
+      rtx_insn *seq = get_insns ();
+      end_sequence ();
 
-  edge entry_edge = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
-  insert_insn_on_edge (seq, entry_edge);
-  commit_one_edge_insertion (entry_edge);
+      edge entry_edge = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
+      insert_insn_on_edge (seq, entry_edge);
+      commit_one_edge_insertion (entry_edge);
+    }
 }
 
 #undef TARGET_INIT_PIC_REG
@@ -480,7 +511,7 @@ or1k_frame_pointer_required ()
 {
   /* ??? While IRA checks accesses_prior_frames, reload does not.
      We do want the frame pointer for this case.  */
-  return (crtl->accesses_prior_frames || crtl->profile);
+  return (crtl->accesses_prior_frames);
 }
 
 /* Expand the "eh_return" pattern.
diff --git a/gcc/config/or1k/or1k.h b/gcc/config/or1k/or1k.h
index 669907e..bea7ba4 100644
--- a/gcc/config/or1k/or1k.h
+++ b/gcc/config/or1k/or1k.h
@@ -392,12 +392,7 @@ do {                                                    \
 
 /* Emit rtl for profiling.  Output assembler code to call "_mcount" for
    profiling a function entry.  */
-#define PROFILE_HOOK(LABEL)						\
-  {									\
-    rtx fun;								\
-    fun = gen_rtx_SYMBOL_REF (Pmode, "_mcount");			\
-    emit_library_call (fun, LCT_NORMAL, VOIDmode);			\
-  }
+#define PROFILE_HOOK(LABEL)  or1k_profile_hook()
 
 /* All the work is done in PROFILE_HOOK, but this is still required.  */
 #define FUNCTION_PROFILER(STREAM, LABELNO) do { } while (0)
diff --git a/gcc/config/pa/pa-d.c b/gcc/config/pa/pa-d.c
index 6802738..14ef8ca 100644
--- a/gcc/config/pa/pa-d.c
+++ b/gcc/config/pa/pa-d.c
@@ -47,7 +47,7 @@ pa_d_handle_target_float_abi (void)
 {
   const char *abi;
 
-  if (TARGET_DISABLE_FPREGS || TARGET_SOFT_FLOAT)
+  if (TARGET_SOFT_FLOAT)
     abi = "soft";
   else
     abi = "hard";
diff --git a/gcc/config/pa/pa-modes.def b/gcc/config/pa/pa-modes.def
index 769de66..6020233 100644
--- a/gcc/config/pa/pa-modes.def
+++ b/gcc/config/pa/pa-modes.def
@@ -30,3 +30,6 @@ FLOAT_MODE (TF, 16, mips_quad_format);
 
 /* HPPA floating comparisons produce distinct condition codes.  */
 CC_MODE (CCFP);
+
+/* Mode used for signed overflow checking of TImode.  */
+INT_MODE (OI, 32);
diff --git a/gcc/config/pa/pa.c b/gcc/config/pa/pa.c
index d13021a..f22d25a 100644
--- a/gcc/config/pa/pa.c
+++ b/gcc/config/pa/pa.c
@@ -497,7 +497,7 @@ fix_range (const char *const_str)
       break;
 
   if (i > FP_REG_LAST)
-    target_flags |= MASK_DISABLE_FPREGS;
+    target_flags |= MASK_SOFT_FLOAT;
 }
 
 /* Implement the TARGET_OPTION_OVERRIDE hook.  */
@@ -1578,14 +1578,14 @@ hppa_rtx_costs (rtx x, machine_mode mode, int outer_code,
 	}
       else if (mode == DImode)
 	{
-	  if (TARGET_PA_11 && !TARGET_DISABLE_FPREGS && !TARGET_SOFT_FLOAT)
-	    *total = COSTS_N_INSNS (32);
+	  if (TARGET_PA_11 && !TARGET_SOFT_FLOAT && !TARGET_SOFT_MULT)
+	    *total = COSTS_N_INSNS (25);
 	  else
 	    *total = COSTS_N_INSNS (80);
 	}
       else
 	{
-	  if (TARGET_PA_11 && !TARGET_DISABLE_FPREGS && !TARGET_SOFT_FLOAT)
+	  if (TARGET_PA_11 && !TARGET_SOFT_FLOAT && !TARGET_SOFT_MULT)
 	    *total = COSTS_N_INSNS (8);
 	  else
 	    *total = COSTS_N_INSNS (20);
@@ -6550,18 +6550,16 @@ hppa_gimplify_va_arg_expr (tree valist, tree type, gimple_seq *pre_p,
 
 /* True if MODE is valid for the target.  By "valid", we mean able to
    be manipulated in non-trivial ways.  In particular, this means all
-   the arithmetic is supported.
-
-   Currently, TImode is not valid as the HP 64-bit runtime documentation
-   doesn't document the alignment and calling conventions for this type. 
-   Thus, we return false when PRECISION is 2 * BITS_PER_WORD and
-   2 * BITS_PER_WORD isn't equal LONG_LONG_TYPE_SIZE.  */
+   the arithmetic is supported.  */
 
 static bool
 pa_scalar_mode_supported_p (scalar_mode mode)
 {
   int precision = GET_MODE_PRECISION (mode);
 
+  if (TARGET_64BIT && mode == TImode)
+    return true;
+
   switch (GET_MODE_CLASS (mode))
     {
     case MODE_PARTIAL_INT:
@@ -10627,7 +10625,7 @@ pa_conditional_register_usage (void)
       for (i = 33; i < 56; i += 2)
 	fixed_regs[i] = call_used_regs[i] = 1;
     }
-  if (TARGET_DISABLE_FPREGS || TARGET_SOFT_FLOAT)
+  if (TARGET_SOFT_FLOAT)
     {
       for (i = FP_REG_FIRST; i <= FP_REG_LAST; i++)
 	fixed_regs[i] = call_used_regs[i] = 1;
diff --git a/gcc/config/pa/pa.h b/gcc/config/pa/pa.h
index fbb9604..96815ec 100644
--- a/gcc/config/pa/pa.h
+++ b/gcc/config/pa/pa.h
@@ -136,6 +136,9 @@ extern unsigned long total_code_bytes;
    by default.  */
 #define DEFAULT_GDB_EXTENSIONS 1
 
+/* Select dwarf2 as the preferred debug format.  */
+#define PREFERRED_DEBUGGING_TYPE DWARF2_DEBUG
+
 /* This used to be zero (no max length), but big enums and such can
    cause huge strings which killed gas.
 
@@ -255,11 +258,17 @@ typedef struct GTY(()) machine_function
    is UNITS_PER_WORD.  Otherwise, it is the constant value that is the
    smallest value that UNITS_PER_WORD can have at run-time.
 
-   FIXME: This needs to be 4 when TARGET_64BIT is true to suppress the
-   building of various TImode routines in libgcc.  The HP runtime
-   specification doesn't provide the alignment requirements and calling
-   conventions for TImode variables.  */
-#define MIN_UNITS_PER_WORD 4
+   This needs to be 8 when TARGET_64BIT is true to allow building various
+   TImode routines in libgcc.  However, we also need the DImode DIVMOD
+   routines because they are not currently implemented in pa.md.
+   
+   The HP runtime specification doesn't provide the alignment requirements
+   and calling conventions for TImode variables.  */
+#ifdef IN_LIBGCC2
+#define MIN_UNITS_PER_WORD      UNITS_PER_WORD
+#else
+#define MIN_UNITS_PER_WORD      4
+#endif
 
 /* The widest floating point format supported by the hardware.  Note that
    setting this influences some Ada floating point type sizes, currently
@@ -833,7 +842,6 @@ extern int may_call_alloca;
 
 #define INT14_OK_STRICT \
   (TARGET_SOFT_FLOAT                                                   \
-   || TARGET_DISABLE_FPREGS                                            \
    || (TARGET_PA_20 && !TARGET_ELF32))
 
 /* The macros REG_OK_FOR..._P assume that the arg is a REG rtx
diff --git a/gcc/config/pa/pa.md b/gcc/config/pa/pa.md
index 5cda3b7..f124c30 100644
--- a/gcc/config/pa/pa.md
+++ b/gcc/config/pa/pa.md
@@ -2186,14 +2186,14 @@
   [(set (match_operand:SI 0 "move_dest_operand"
 			  "=r,r,r,r,r,r,Q,!*q,!r,!*f,*f,T,?r,?*f")
 	(match_operand:SI 1 "move_src_operand"
-			  "A,rG,J,N,K,RQ,rM,!rM,!*q,!*fM,RT,*f,*f,r"))]
+			  "A,r,J,N,K,RQ,rM,!rM,!*q,!*fM,RT,*f,*f,r"))]
   "(register_operand (operands[0], SImode)
     || reg_or_0_operand (operands[1], SImode))
    && !TARGET_SOFT_FLOAT
    && !TARGET_64BIT"
   "@
    ldw RT'%A1,%0
-   copy %r1,%0
+   copy %1,%0
    ldi %1,%0
    ldil L'%1,%0
    {zdepi|depwi,z} %Z1,%0
@@ -2214,14 +2214,14 @@
   [(set (match_operand:SI 0 "move_dest_operand"
 			  "=r,r,r,r,r,r,Q,!*q,!r,!*f,*f,T")
 	(match_operand:SI 1 "move_src_operand"
-			  "A,rG,J,N,K,RQ,rM,!rM,!*q,!*fM,RT,*f"))]
+			  "A,r,J,N,K,RQ,rM,!rM,!*q,!*fM,RT,*f"))]
   "(register_operand (operands[0], SImode)
     || reg_or_0_operand (operands[1], SImode))
    && !TARGET_SOFT_FLOAT
    && TARGET_64BIT"
   "@
    ldw RT'%A1,%0
-   copy %r1,%0
+   copy %1,%0
    ldi %1,%0
    ldil L'%1,%0
    {zdepi|depwi,z} %Z1,%0
@@ -2240,14 +2240,14 @@
   [(set (match_operand:SI 0 "move_dest_operand"
 			  "=r,r,r,r,r,r,Q,!*q,!r")
 	(match_operand:SI 1 "move_src_operand"
-			  "A,rG,J,N,K,RQ,rM,!rM,!*q"))]
+			  "A,r,J,N,K,RQ,rM,!rM,!*q"))]
   "(register_operand (operands[0], SImode)
     || reg_or_0_operand (operands[1], SImode))
    && TARGET_SOFT_FLOAT
    && TARGET_64BIT"
   "@
    ldw RT'%A1,%0
-   copy %r1,%0
+   copy %1,%0
    ldi %1,%0
    ldil L'%1,%0
    {zdepi|depwi,z} %Z1,%0
@@ -2381,13 +2381,13 @@
   [(set (match_operand:SI 0 "move_dest_operand"
 			  "=r,r,r,r,r,r,Q,!*q,!r")
 	(match_operand:SI 1 "move_src_operand"
-			  "A,rG,J,N,K,RQ,rM,!rM,!*q"))]
+			  "A,r,J,N,K,RQ,rM,!rM,!*q"))]
   "(register_operand (operands[0], SImode)
     || reg_or_0_operand (operands[1], SImode))
    && TARGET_SOFT_FLOAT"
   "@
    ldw RT'%A1,%0
-   copy %r1,%0
+   copy %1,%0
    ldi %1,%0
    ldil L'%1,%0
    {zdepi|depwi,z} %Z1,%0
@@ -2909,11 +2909,11 @@
   [(set (match_operand:HI 0 "move_dest_operand"
 	 		  "=r,r,r,r,r,Q,!*q,!r")
 	(match_operand:HI 1 "move_src_operand"
-			  "rG,J,N,K,RQ,rM,!rM,!*q"))]
+			  "r,J,N,K,RQ,rM,!rM,!*q"))]
   "(register_operand (operands[0], HImode)
     || reg_or_0_operand (operands[1], HImode))"
   "@
-   copy %r1,%0
+   copy %1,%0
    ldi %1,%0
    ldil L'%1,%0
    {zdepi|depwi,z} %Z1,%0
@@ -3069,11 +3069,11 @@
   [(set (match_operand:QI 0 "move_dest_operand"
 			  "=r,r,r,r,r,Q,!*q,!r")
 	(match_operand:QI 1 "move_src_operand"
-			  "rG,J,N,K,RQ,rM,!rM,!*q"))]
+			  "r,J,N,K,RQ,rM,!rM,!*q"))]
   "(register_operand (operands[0], QImode)
     || reg_or_0_operand (operands[1], QImode))"
   "@
-   copy %r1,%0
+   copy %1,%0
    ldi %1,%0
    ldil L'%1,%0
    {zdepi|depwi,z} %Z1,%0
@@ -4221,13 +4221,13 @@
   [(set (match_operand:DI 0 "move_dest_operand"
 			  "=r,r,r,r,r,r,Q,!*q,!r,!*f,*f,T")
 	(match_operand:DI 1 "move_src_operand"
-			  "A,rG,J,N,K,RQ,rM,!rM,!*q,!*fM,RT,*f"))]
+			  "A,r,J,N,K,RQ,rM,!rM,!*q,!*fM,RT,*f"))]
   "(register_operand (operands[0], DImode)
     || reg_or_0_operand (operands[1], DImode))
    && !TARGET_SOFT_FLOAT && TARGET_64BIT"
   "@
    ldd RT'%A1,%0
-   copy %r1,%0
+   copy %1,%0
    ldi %1,%0
    ldil L'%1,%0
    depdi,z %z1,%0
@@ -4246,13 +4246,13 @@
   [(set (match_operand:DI 0 "move_dest_operand"
 			  "=r,r,r,r,r,r,Q,!*q,!r")
 	(match_operand:DI 1 "move_src_operand"
-			  "A,rG,J,N,K,RQ,rM,!rM,!*q"))]
+			  "A,r,J,N,K,RQ,rM,!rM,!*q"))]
   "(register_operand (operands[0], DImode)
     || reg_or_0_operand (operands[1], DImode))
    && TARGET_SOFT_FLOAT && TARGET_64BIT"
   "@
    ldd RT'%A1,%0
-   copy %r1,%0
+   copy %1,%0
    ldi %1,%0
    ldil L'%1,%0
    depdi,z %z1,%0
@@ -5357,6 +5357,88 @@
   [(set_attr "type" "binary,binary")
    (set_attr "length" "4,4")])
 
+(define_insn "addti3"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(plus:TI (match_operand:TI 1 "register_operand" "r")
+		 (match_operand:TI 2 "register_operand" "r")))]
+  "TARGET_64BIT"
+  "*
+{
+  operands[3] = gen_lowpart (DImode, operands[0]);
+  operands[4] = gen_lowpart (DImode, operands[1]);
+  operands[5] = gen_lowpart (DImode, operands[2]);
+  operands[0] = gen_highpart (DImode, operands[0]);
+  operands[1] = gen_highpart (DImode, operands[1]);
+  operands[2] = gen_highpart (DImode, operands[2]);
+  return \"add %4,%5,%3\;add,dc %1,%2,%0\";
+}"
+  [(set_attr "type" "multi")
+   (set_attr "length" "8")])
+
+(define_insn "addvti3"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(plus:TI (match_operand:TI 1 "register_operand" "r")
+		 (match_operand:TI 2 "register_operand" "r")))
+   (trap_if (ne (plus:OI (sign_extend:OI (match_dup 1))
+			 (sign_extend:OI (match_dup 2)))
+		(sign_extend:OI (plus:TI (match_dup 1)
+					 (match_dup 2))))
+	    (const_int 0))]
+  "TARGET_64BIT"
+  "*
+{
+  operands[3] = gen_lowpart (DImode, operands[0]);
+  operands[4] = gen_lowpart (DImode, operands[1]);
+  operands[5] = gen_lowpart (DImode, operands[2]);
+  operands[0] = gen_highpart (DImode, operands[0]);
+  operands[1] = gen_highpart (DImode, operands[1]);
+  operands[2] = gen_highpart (DImode, operands[2]);
+  return \"add %4,%5,%3\;add,dc,tsv %1,%2,%0\";
+}"
+  [(set_attr "type" "multi")
+   (set_attr "length" "8")])
+
+(define_insn "subti3"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(minus:TI (match_operand:TI 1 "register_operand" "r")
+		  (match_operand:TI 2 "register_operand" "r")))]
+  "TARGET_64BIT"
+  "*
+{
+  operands[3] = gen_lowpart (DImode, operands[0]);
+  operands[4] = gen_lowpart (DImode, operands[1]);
+  operands[5] = gen_lowpart (DImode, operands[2]);
+  operands[0] = gen_highpart (DImode, operands[0]);
+  operands[1] = gen_highpart (DImode, operands[1]);
+  operands[2] = gen_highpart (DImode, operands[2]);
+  return \"sub %4,%5,%3\;sub,db %1,%2,%0\";
+}"
+  [(set_attr "type" "multi")
+   (set_attr "length" "8")])
+
+(define_insn "subvti3"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(minus:TI (match_operand:TI 1 "register_operand" "r")
+		  (match_operand:TI 2 "register_operand" "r")))
+   (trap_if (ne (minus:OI (sign_extend:OI (match_dup 1))
+			  (sign_extend:OI (match_dup 2)))
+		(sign_extend:OI (minus:TI (match_dup 1)
+					  (match_dup 2))))
+	    (const_int 0))]
+  "TARGET_64BIT"
+  "*
+{
+  operands[3] = gen_lowpart (DImode, operands[0]);
+  operands[4] = gen_lowpart (DImode, operands[1]);
+  operands[5] = gen_lowpart (DImode, operands[2]);
+  operands[0] = gen_highpart (DImode, operands[0]);
+  operands[1] = gen_highpart (DImode, operands[1]);
+  operands[2] = gen_highpart (DImode, operands[2]);
+  return \"sub %4,%5,%3\;sub,db,tsv %1,%2,%0\";
+}"
+  [(set_attr "type" "multi")
+   (set_attr "length" "8")])
+
 ;; Trap instructions.
 
 (define_insn "trap"
@@ -5384,7 +5466,7 @@
   "
 {
   operands[4] = gen_rtx_REG (SImode, TARGET_64BIT ? 2 : 31);
-  if (TARGET_PA_11 && !TARGET_DISABLE_FPREGS && !TARGET_SOFT_FLOAT)
+  if (TARGET_PA_11 && !TARGET_SOFT_FLOAT && !TARGET_SOFT_MULT)
     {
       rtx scratch = gen_reg_rtx (DImode);
       operands[1] = force_reg (SImode, operands[1]);
@@ -5402,7 +5484,7 @@
   [(set (match_operand:DI 0 "register_operand" "=f")
 	(mult:DI (zero_extend:DI (match_operand:SI 1 "register_operand" "f"))
 		 (zero_extend:DI (match_operand:SI 2 "register_operand" "f"))))]
-  "TARGET_PA_11 && ! TARGET_DISABLE_FPREGS && ! TARGET_SOFT_FLOAT"
+  "TARGET_PA_11 && ! TARGET_SOFT_FLOAT && ! TARGET_SOFT_MULT"
   "xmpyu %1,%2,%0"
   [(set_attr "type" "fpmuldbl")
    (set_attr "length" "4")])
@@ -5411,7 +5493,7 @@
   [(set (match_operand:DI 0 "register_operand" "=f")
 	(mult:DI (zero_extend:DI (match_operand:SI 1 "register_operand" "f"))
 		 (match_operand:DI 2 "uint32_operand" "f")))]
-  "TARGET_PA_11 && ! TARGET_DISABLE_FPREGS && ! TARGET_SOFT_FLOAT && !TARGET_64BIT"
+  "TARGET_PA_11 && ! TARGET_SOFT_FLOAT && ! TARGET_SOFT_MULT && !TARGET_64BIT"
   "xmpyu %1,%R2,%0"
   [(set_attr "type" "fpmuldbl")
    (set_attr "length" "4")])
@@ -5420,7 +5502,7 @@
   [(set (match_operand:DI 0 "register_operand" "=f")
 	(mult:DI (zero_extend:DI (match_operand:SI 1 "register_operand" "f"))
 		 (match_operand:DI 2 "uint32_operand" "f")))]
-  "TARGET_PA_11 && ! TARGET_DISABLE_FPREGS && ! TARGET_SOFT_FLOAT && TARGET_64BIT"
+  "TARGET_PA_11 && ! TARGET_SOFT_FLOAT && ! TARGET_SOFT_MULT && TARGET_64BIT"
   "xmpyu %1,%2R,%0"
   [(set_attr "type" "fpmuldbl")
    (set_attr "length" "4")])
@@ -5457,8 +5539,8 @@
 		 (match_operand:DI 2 "register_operand" "")))]
   "! optimize_size
    && TARGET_PA_11
-   && ! TARGET_DISABLE_FPREGS
-   && ! TARGET_SOFT_FLOAT"
+   && ! TARGET_SOFT_FLOAT
+   && ! TARGET_SOFT_MULT"
   "
 {
   rtx low_product = gen_reg_rtx (DImode);
@@ -5897,7 +5979,7 @@
 	(neg:DI (match_operand:DI 1 "register_operand" "r")))]
   "!TARGET_64BIT"
   "sub %%r0,%R1,%R0\;{subb|sub,b} %%r0,%1,%0"
-  [(set_attr "type" "unary")
+  [(set_attr "type" "multi")
    (set_attr "length" "8")])
 
 (define_insn ""
@@ -5908,6 +5990,21 @@
   [(set_attr "type" "unary")
    (set_attr "length" "4")])
 
+(define_insn "negti2"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(neg:TI (match_operand:TI 1 "register_operand" "r")))]
+  "TARGET_64BIT"
+  "*
+{
+  operands[2] = gen_lowpart (DImode, operands[0]);
+  operands[3] = gen_lowpart (DImode, operands[1]);
+  operands[0] = gen_highpart (DImode, operands[0]);
+  operands[1] = gen_highpart (DImode, operands[1]);
+  return \"sub %%r0,%3,%2\;sub,db %%r0,%1,%0\";
+}"
+  [(set_attr "type" "multi")
+   (set_attr "length" "8")])
+
 (define_expand "negvdi2"
   [(parallel [(set (match_operand:DI 0 "register_operand" "")
 		   (neg:DI (match_operand:DI 1 "register_operand" "")))
@@ -5925,7 +6022,7 @@
 	    (const_int 0))]
   "!TARGET_64BIT"
   "sub %%r0,%R1,%R0\;{subbo|sub,b,tsv} %%r0,%1,%0"
-  [(set_attr "type" "unary")
+  [(set_attr "type" "multi")
    (set_attr "length" "8")])
 
 (define_insn ""
@@ -5939,6 +6036,24 @@
   [(set_attr "type" "unary")
    (set_attr "length" "4")])
 
+(define_insn "negvti2"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(neg:TI (match_operand:TI 1 "register_operand" "r")))
+   (trap_if (ne (neg:OI (sign_extend:OI (match_dup 1)))
+		(sign_extend:OI (neg:TI (match_dup 1))))
+	    (const_int 0))]
+  "TARGET_64BIT"
+  "*
+{
+  operands[2] = gen_lowpart (DImode, operands[0]);
+  operands[3] = gen_lowpart (DImode, operands[1]);
+  operands[0] = gen_highpart (DImode, operands[0]);
+  operands[1] = gen_highpart (DImode, operands[1]);
+  return \"sub %%r0,%3,%2\;sub,db,tsv %%r0,%1,%0\";
+}"
+  [(set_attr "type" "multi")
+   (set_attr "length" "8")])
+
 (define_insn "negsi2"
   [(set (match_operand:SI 0 "register_operand" "=r")
 	(neg:SI (match_operand:SI 1 "register_operand" "r")))]
@@ -6566,6 +6681,42 @@
   operands[1] = force_reg (DImode, operands[1]);
 }")
 
+(define_expand "ashlti3"
+  [(set (match_operand:TI 0 "register_operand" "")
+	(ashift:TI (match_operand:TI 1 "lhs_lshift_operand" "")
+		   (match_operand:TI 2 "arith32_operand" "")))]
+  "TARGET_64BIT"
+{
+  if (REG_P (operands[0]) && GET_CODE (operands[2]) == CONST_INT)
+    {
+      unsigned HOST_WIDE_INT shift = UINTVAL (operands[2]);
+      rtx dst = operands[0];
+      rtx src = force_reg (TImode, operands[1]);
+      if (shift >= 1 && shift <= 63)
+	{
+	  emit_insn (gen_shrpd_internal (gen_highpart (DImode, dst),
+					 gen_lowpart (DImode, src),
+					 GEN_INT (64-shift),
+					 gen_highpart (DImode, src),
+					 GEN_INT (shift)));
+	  emit_insn (gen_ashldi3 (gen_lowpart (DImode, dst),
+				  gen_lowpart (DImode, src),
+				  GEN_INT (shift)));
+	  DONE;
+	}
+      else if (shift >= 64 && shift <= 127)
+	{
+	  emit_insn (gen_ashldi3 (gen_highpart (DImode, dst),
+				  gen_lowpart (DImode, src),
+				  GEN_INT (shift - 64)));
+	  emit_move_insn (gen_lowpart (DImode, dst), GEN_INT (0));
+	  DONE;
+	}
+    }
+  /* Fallback to using optabs.c's expand_doubleword_shift.  */
+  FAIL;
+})
+
 (define_insn ""
   [(set (match_operand:DI 0 "register_operand" "=r")
 	(ashift:DI (match_operand:DI 1 "register_operand" "r")
@@ -6944,6 +7095,15 @@
 		     (match_operand:SI 4 "const_int_operand"))))]
   "")
 
+(define_expand "shrpd_internal"
+  [(set (match_operand:DI 0 "register_operand")
+	(ior:DI
+	  (lshiftrt:DI (match_operand:DI 1 "register_operand")
+		       (match_operand:DI 2 "const_int_operand"))
+	  (ashift:DI (match_operand:DI 3 "register_operand")
+		     (match_operand:DI 4 "const_int_operand"))))]
+  "TARGET_64BIT")
+
 (define_insn ""
   [(set (match_operand:SI 0 "register_operand" "=r")
 	(and:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
@@ -7805,7 +7965,7 @@ add,l %2,%3,%3\;bv,n %%r0(%3)"
       if (GET_CODE (op) == SYMBOL_REF)
 	{
 	  /* Handle special call to buggy powf function.  */
-	  if (TARGET_HPUX && !TARGET_DISABLE_FPREGS && !TARGET_SOFT_FLOAT
+	  if (TARGET_HPUX && !TARGET_SOFT_FLOAT
 	      && !strcmp (targetm.strip_name_encoding (XSTR (op, 0)), "powf"))
 	    call_powf = true;
 
@@ -10260,7 +10420,7 @@ add,l %2,%3,%3\;bv,n %%r0(%3)"
 {
   enum memmodel model;
 
-  if (TARGET_64BIT || TARGET_DISABLE_FPREGS || TARGET_SOFT_FLOAT)
+  if (TARGET_64BIT || TARGET_SOFT_FLOAT)
     FAIL;
 
   model = memmodel_from_int (INTVAL (operands[2]));
@@ -10276,7 +10436,7 @@ add,l %2,%3,%3\;bv,n %%r0(%3)"
   [(set (match_operand:DI 0 "register_operand" "=r")
         (mem:DI (match_operand:SI 1 "register_operand" "r")))
    (clobber (match_scratch:DI 2 "=f"))]
-  "!TARGET_64BIT && !TARGET_DISABLE_FPREGS && !TARGET_SOFT_FLOAT"
+  "!TARGET_64BIT && !TARGET_SOFT_FLOAT"
   "{fldds|fldd} 0(%1),%2\n\t{fstds|fstd} %2,-16(%%sp)\n\t{ldws|ldw} -16(%%sp),%0\n\t{ldws|ldw} -12(%%sp),%R0"
   [(set_attr "type" "move")
    (set_attr "length" "16")])
@@ -10299,7 +10459,7 @@ add,l %2,%3,%3\;bv,n %%r0(%3)"
 	DONE;
     }
 
-  if (TARGET_64BIT || TARGET_DISABLE_FPREGS || TARGET_SOFT_FLOAT)
+  if (TARGET_64BIT || TARGET_SOFT_FLOAT)
     FAIL;
 
   model = memmodel_from_int (INTVAL (operands[2]));
@@ -10317,7 +10477,7 @@ add,l %2,%3,%3\;bv,n %%r0(%3)"
   [(set (mem:DI (match_operand:SI 0 "register_operand" "r,r"))
         (match_operand:DI 1 "reg_or_0_operand" "M,r"))
    (clobber (match_scratch:DI 2 "=X,f"))]
-  "!TARGET_64BIT && !TARGET_DISABLE_FPREGS && !TARGET_SOFT_FLOAT"
+  "!TARGET_64BIT && !TARGET_SOFT_FLOAT"
   "@
    {fstds|fstd} %%fr0,0(%0)
    {stws|stw} %1,-16(%%sp)\n\t{stws|stw} %R1,-12(%%sp)\n\t{fldds|fldd} -16(%%sp),%2\n\t{fstds|fstd} %2,0(%0)"
diff --git a/gcc/config/pa/pa.opt b/gcc/config/pa/pa.opt
index 09660c4..47995f7 100644
--- a/gcc/config/pa/pa.opt
+++ b/gcc/config/pa/pa.opt
@@ -50,8 +50,8 @@ Target Var(TARGET_COHERENT_LDCW) Init(1)
 Use ldcw/ldcd coherent cache-control hint.
 
 mdisable-fpregs
-Target Mask(DISABLE_FPREGS)
-Disable FP regs.
+Target Mask(SOFT_FLOAT)
+Disable FP regs.  Equivalent to -msoft-float.
 
 mdisable-indexing
 Target Mask(DISABLE_INDEXING)
@@ -143,6 +143,10 @@ msoft-float
 Target Mask(SOFT_FLOAT)
 Use software floating point.
 
+msoft-mult
+Target Mask(SOFT_MULT)
+Use software integer multiplication.
+
 msnake
 Target RejectNegative
 Generate PA1.1 code.
diff --git a/gcc/config/pa/pa64-hpux.h b/gcc/config/pa/pa64-hpux.h
index c25bc38..3ee97a4 100644
--- a/gcc/config/pa/pa64-hpux.h
+++ b/gcc/config/pa/pa64-hpux.h
@@ -266,7 +266,6 @@ do {								\
 /* It looks like DWARF2 will be the easiest debug format to handle on this
    platform.  */
 #define DWARF2_DEBUGGING_INFO 1
-#define PREFERRED_DEBUGGING_TYPE DWARF2_DEBUG
 
 /* This target uses the ELF object file format.  */
 #define OBJECT_FORMAT_ELF
diff --git a/gcc/config/riscv/arch-canonicalize b/gcc/config/riscv/arch-canonicalize
index ea95a06..c7df3c8 100755
--- a/gcc/config/riscv/arch-canonicalize
+++ b/gcc/config/riscv/arch-canonicalize
@@ -28,7 +28,7 @@ import itertools
 from functools import reduce
 
 
-CANONICAL_ORDER = "mafdgqlcbjtpvn"
+CANONICAL_ORDER = "imafdgqlcbjtpvn"
 LONG_EXT_PREFIXES = ['z', 's', 'h', 'x']
 
 #
diff --git a/gcc/config/riscv/bitmanip.md b/gcc/config/riscv/bitmanip.md
new file mode 100644
index 0000000..59779b4
--- /dev/null
+++ b/gcc/config/riscv/bitmanip.md
@@ -0,0 +1,342 @@
+;; Machine description for RISC-V Bit Manipulation operations.
+;; Copyright (C) 2021 Free Software Foundation, Inc.
+
+;; This file is part of GCC.
+
+;; GCC is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3, or (at your option)
+;; any later version.
+
+;; GCC is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GCC; see the file COPYING3.  If not see
+;; <http://www.gnu.org/licenses/>.
+
+(define_code_iterator bitmanip_bitwise [and ior])
+
+(define_code_iterator bitmanip_minmax [smin umin smax umax])
+
+(define_code_iterator clz_ctz_pcnt [clz ctz popcount])
+
+(define_code_attr bitmanip_optab [(smin "smin")
+				  (smax "smax")
+				  (umin "umin")
+				  (umax "umax")
+				  (clz "clz")
+				  (ctz "ctz")
+				  (popcount "popcount")])
+
+
+(define_code_attr bitmanip_insn [(smin "min")
+				 (smax "max")
+				 (umin "minu")
+				 (umax "maxu")
+				 (clz "clz")
+				 (ctz "ctz")
+				 (popcount "cpop")])
+
+(define_mode_attr shiftm1 [(SI "const31_operand") (DI "const63_operand")])
+
+;; ZBA extension.
+
+(define_insn "*zero_extendsidi2_bitmanip"
+  [(set (match_operand:DI 0 "register_operand" "=r,r")
+	(zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
+  "TARGET_64BIT && TARGET_ZBA"
+  "@
+   zext.w\t%0,%1
+   lwu\t%0,%1"
+  [(set_attr "type" "bitmanip,load")
+   (set_attr "mode" "DI")])
+
+(define_insn "*shNadd"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(plus:X (ashift:X (match_operand:X 1 "register_operand" "r")
+			  (match_operand:QI 2 "immediate_operand" "I"))
+		(match_operand:X 3 "register_operand" "r")))]
+  "TARGET_ZBA
+   && (INTVAL (operands[2]) >= 1) && (INTVAL (operands[2]) <= 3)"
+  "sh%2add\t%0,%1,%3"
+  [(set_attr "type" "bitmanip")
+   (set_attr "mode" "<X:MODE>")])
+
+(define_insn "*shNadduw"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(plus:DI
+	  (and:DI (ashift:DI (match_operand:DI 1 "register_operand" "r")
+			     (match_operand:QI 2 "immediate_operand" "I"))
+		 (match_operand 3 "immediate_operand" ""))
+	  (match_operand:DI 4 "register_operand" "r")))]
+  "TARGET_64BIT && TARGET_ZBA
+   && (INTVAL (operands[2]) >= 1) && (INTVAL (operands[2]) <= 3)
+   && (INTVAL (operands[3]) >> INTVAL (operands[2])) == 0xffffffff"
+  "sh%2add.uw\t%0,%1,%4"
+  [(set_attr "type" "bitmanip")
+   (set_attr "mode" "DI")])
+
+(define_insn "*add.uw"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(plus:DI (zero_extend:DI
+		   (match_operand:SI 1 "register_operand" "r"))
+		 (match_operand:DI 2 "register_operand" "r")))]
+  "TARGET_64BIT && TARGET_ZBA"
+  "add.uw\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")
+   (set_attr "mode" "DI")])
+
+(define_insn "*slliuw"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(and:DI (ashift:DI (match_operand:DI 1 "register_operand" "r")
+			   (match_operand:QI 2 "immediate_operand" "I"))
+		(match_operand 3 "immediate_operand" "")))]
+  "TARGET_64BIT && TARGET_ZBA
+   && (INTVAL (operands[3]) >> INTVAL (operands[2])) == 0xffffffff"
+  "slli.uw\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")
+   (set_attr "mode" "DI")])
+
+;; ZBB extension.
+
+(define_insn "*<optab>_not<mode>"
+  [(set (match_operand:X 0 "register_operand" "=r")
+        (bitmanip_bitwise:X (not:X (match_operand:X 1 "register_operand" "r"))
+                            (match_operand:X 2 "register_operand" "r")))]
+  "TARGET_ZBB"
+  "<insn>n\t%0,%2,%1"
+  [(set_attr "type" "bitmanip")
+   (set_attr "mode" "<X:MODE>")])
+
+(define_insn "*xor_not<mode>"
+  [(set (match_operand:X 0 "register_operand" "=r")
+        (not:X (xor:X (match_operand:X 1 "register_operand" "r")
+                      (match_operand:X 2 "register_operand" "r"))))]
+  "TARGET_ZBB"
+  "xnor\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")
+   (set_attr "mode" "<X:MODE>")])
+
+(define_insn "<bitmanip_optab>si2"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+        (clz_ctz_pcnt:SI (match_operand:SI 1 "register_operand" "r")))]
+  "TARGET_ZBB"
+  { return TARGET_64BIT ? "<bitmanip_insn>w\t%0,%1" : "<bitmanip_insn>\t%0,%1"; }
+  [(set_attr "type" "bitmanip")
+   (set_attr "mode" "SI")])
+
+(define_insn "*<bitmanip_optab>disi2"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+        (sign_extend:DI
+          (clz_ctz_pcnt:SI (match_operand:SI 1 "register_operand" "r"))))]
+  "TARGET_64BIT && TARGET_ZBB"
+  "<bitmanip_insn>w\t%0,%1"
+  [(set_attr "type" "bitmanip")
+   (set_attr "mode" "SI")])
+
+(define_insn "<bitmanip_optab>di2"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+        (clz_ctz_pcnt:DI (match_operand:DI 1 "register_operand" "r")))]
+  "TARGET_64BIT && TARGET_ZBB"
+  "<bitmanip_insn>\t%0,%1"
+  [(set_attr "type" "bitmanip")
+   (set_attr "mode" "DI")])
+
+(define_insn "*zero_extendhi<GPR:mode>2_bitmanip"
+  [(set (match_operand:GPR 0 "register_operand" "=r,r")
+        (zero_extend:GPR (match_operand:HI 1 "nonimmediate_operand" "r,m")))]
+  "TARGET_ZBB"
+  "@
+   zext.h\t%0,%1
+   lhu\t%0,%1"
+  [(set_attr "type" "bitmanip,load")
+   (set_attr "mode" "<GPR:MODE>")])
+
+(define_insn "*extend<SHORT:mode><SUPERQI:mode>2_zbb"
+  [(set (match_operand:SUPERQI   0 "register_operand"     "=r,r")
+	(sign_extend:SUPERQI
+	    (match_operand:SHORT 1 "nonimmediate_operand" " r,m")))]
+  "TARGET_ZBB"
+  "@
+   sext.<SHORT:size>\t%0,%1
+   l<SHORT:size>\t%0,%1"
+  [(set_attr "type" "bitmanip,load")
+   (set_attr "mode" "<SUPERQI:MODE>")])
+
+(define_insn "*zero_extendhi<GPR:mode>2_zbb"
+  [(set (match_operand:GPR    0 "register_operand"     "=r,r")
+	(zero_extend:GPR
+	    (match_operand:HI 1 "nonimmediate_operand" " r,m")))]
+  "TARGET_ZBB"
+  "@
+   zext.h\t%0,%1
+   lhu\t%0,%1"
+  [(set_attr "type" "bitmanip,load")
+   (set_attr "mode" "HI")])
+
+(define_insn "rotrsi3"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+	(rotatert:SI (match_operand:SI 1 "register_operand" "r")
+		     (match_operand:QI 2 "arith_operand" "rI")))]
+  "TARGET_ZBB"
+  { return TARGET_64BIT ? "ror%i2w\t%0,%1,%2" : "ror%i2\t%0,%1,%2"; }
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "rotrdi3"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(rotatert:DI (match_operand:DI 1 "register_operand" "r")
+		     (match_operand:QI 2 "arith_operand" "rI")))]
+  "TARGET_64BIT && TARGET_ZBB"
+  "ror%i2\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "rotrsi3_sext"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(sign_extend:DI (rotatert:SI (match_operand:SI 1 "register_operand" "r")
+				     (match_operand:QI 2 "register_operand" "r"))))]
+  "TARGET_64BIT && TARGET_ZBB"
+  "rorw\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "rotlsi3"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+	(rotate:SI (match_operand:SI 1 "register_operand" "r")
+		   (match_operand:QI 2 "register_operand" "r")))]
+  "TARGET_ZBB"
+  { return TARGET_64BIT ? "rolw\t%0,%1,%2" : "rol\t%0,%1,%2"; }
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "rotldi3"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(rotate:DI (match_operand:DI 1 "register_operand" "r")
+		   (match_operand:QI 2 "register_operand" "r")))]
+  "TARGET_64BIT && TARGET_ZBB"
+  "rol\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "rotlsi3_sext"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(sign_extend:DI (rotate:SI (match_operand:SI 1 "register_operand" "r")
+				   (match_operand:QI 2 "register_operand" "r"))))]
+  "TARGET_64BIT && TARGET_ZBB"
+  "rolw\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "bswap<mode>2"
+  [(set (match_operand:X 0 "register_operand" "=r")
+        (bswap:X (match_operand:X 1 "register_operand" "r")))]
+  "TARGET_64BIT && TARGET_ZBB"
+  "rev8\t%0,%1"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "<bitmanip_optab><mode>3"
+  [(set (match_operand:X 0 "register_operand" "=r")
+        (bitmanip_minmax:X (match_operand:X 1 "register_operand" "r")
+			   (match_operand:X 2 "register_operand" "r")))]
+  "TARGET_ZBB"
+  "<bitmanip_insn>\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")])
+
+;; ZBS extension.
+
+(define_insn "*bset<mode>"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(ior:X (ashift:X (const_int 1)
+			 (match_operand:QI 2 "register_operand" "r"))
+	       (match_operand:X 1 "register_operand" "r")))]
+  "TARGET_ZBS"
+  "bset\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "*bset<mode>_mask"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(ior:X (ashift:X (const_int 1)
+			 (subreg:QI
+			  (and:X (match_operand:X 2 "register_operand" "r")
+				 (match_operand 3 "<X:shiftm1>" "i")) 0))
+	       (match_operand:X 1 "register_operand" "r")))]
+  "TARGET_ZBS"
+  "bset\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "*bset<mode>_1"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(ashift:X (const_int 1)
+		  (match_operand:QI 1 "register_operand" "r")))]
+  "TARGET_ZBS"
+  "bset\t%0,x0,%1"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "*bset<mode>_1_mask"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(ashift:X (const_int 1)
+		  (subreg:QI
+		   (and:X (match_operand:X 1 "register_operand" "r")
+			  (match_operand 2 "<X:shiftm1>" "i")) 0)))]
+  "TARGET_ZBS"
+  "bset\t%0,x0,%1"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "*bseti<mode>"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(ior:X (match_operand:X 1 "register_operand" "r")
+	       (match_operand 2 "single_bit_mask_operand" "i")))]
+  "TARGET_ZBS"
+  "bseti\t%0,%1,%S2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "*bclr<mode>"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(and:X (rotate:X (const_int -2)
+			 (match_operand:QI 2 "register_operand" "r"))
+	       (match_operand:X 1 "register_operand" "r")))]
+  "TARGET_ZBS"
+  "bclr\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "*bclri<mode>"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(and:X (match_operand:X 1 "register_operand" "r")
+	       (match_operand 2 "not_single_bit_mask_operand" "i")))]
+  "TARGET_ZBS"
+  "bclri\t%0,%1,%T2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "*binv<mode>"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(xor:X (ashift:X (const_int 1)
+			 (match_operand:QI 2 "register_operand" "r"))
+	       (match_operand:X 1 "register_operand" "r")))]
+  "TARGET_ZBS"
+  "binv\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "*binvi<mode>"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(xor:X (match_operand:X 1 "register_operand" "r")
+	       (match_operand 2 "single_bit_mask_operand" "i")))]
+  "TARGET_ZBS"
+  "binvi\t%0,%1,%S2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "*bext<mode>"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(zero_extract:X (match_operand:X 1 "register_operand" "r")
+			(const_int 1)
+			(zero_extend:X
+			 (match_operand:QI 2 "register_operand" "r"))))]
+  "TARGET_ZBS"
+  "bext\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")])
+
+(define_insn "*bexti"
+  [(set (match_operand:X 0 "register_operand" "=r")
+	(zero_extract:X (match_operand:X 1 "register_operand" "r")
+			(const_int 1)
+			(match_operand 2 "immediate_operand" "i")))]
+  "TARGET_ZBS"
+  "bexti\t%0,%1,%2"
+  [(set_attr "type" "bitmanip")])
diff --git a/gcc/config/riscv/predicates.md b/gcc/config/riscv/predicates.md
index 2321151..3da6fd4 100644
--- a/gcc/config/riscv/predicates.md
+++ b/gcc/config/riscv/predicates.md
@@ -74,6 +74,11 @@
   if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
     return false;
 
+  /* Check whether the constant can be loaded in a single
+     instruction with zbs extensions.  */
+  if (TARGET_64BIT && TARGET_ZBS && SINGLE_BIT_MASK_OPERAND (INTVAL (op)))
+    return false;
+
   /* Otherwise check whether the constant can be loaded in a single
      instruction.  */
   return !LUI_OPERAND (INTVAL (op)) && !SMALL_OPERAND (INTVAL (op));
@@ -217,3 +222,20 @@
 {
   return riscv_gpr_save_operation_p (op);
 })
+
+;; Predicates for the ZBS extension.
+(define_predicate "single_bit_mask_operand"
+  (and (match_code "const_int")
+       (match_test "pow2p_hwi (INTVAL (op))")))
+
+(define_predicate "not_single_bit_mask_operand"
+  (and (match_code "const_int")
+       (match_test "pow2p_hwi (~INTVAL (op))")))
+
+(define_predicate "const31_operand"
+  (and (match_code "const_int")
+       (match_test "INTVAL (op) == 31")))
+
+(define_predicate "const63_operand"
+  (and (match_code "const_int")
+       (match_test "INTVAL (op) == 63")))
diff --git a/gcc/config/riscv/riscv-opts.h b/gcc/config/riscv/riscv-opts.h
index f4cf6ca..2efc4b8 100644
--- a/gcc/config/riscv/riscv-opts.h
+++ b/gcc/config/riscv/riscv-opts.h
@@ -73,4 +73,14 @@ enum stack_protector_guard {
 #define TARGET_ZICSR    ((riscv_zi_subext & MASK_ZICSR) != 0)
 #define TARGET_ZIFENCEI ((riscv_zi_subext & MASK_ZIFENCEI) != 0)
 
+#define MASK_ZBA      (1 << 0)
+#define MASK_ZBB      (1 << 1)
+#define MASK_ZBC      (1 << 2)
+#define MASK_ZBS      (1 << 3)
+
+#define TARGET_ZBA    ((riscv_zb_subext & MASK_ZBA) != 0)
+#define TARGET_ZBB    ((riscv_zb_subext & MASK_ZBB) != 0)
+#define TARGET_ZBC    ((riscv_zb_subext & MASK_ZBC) != 0)
+#define TARGET_ZBS    ((riscv_zb_subext & MASK_ZBS) != 0)
+
 #endif /* ! GCC_RISCV_OPTS_H */
diff --git a/gcc/config/riscv/riscv.c b/gcc/config/riscv/riscv.c
index 126572c..a545dbf 100644
--- a/gcc/config/riscv/riscv.c
+++ b/gcc/config/riscv/riscv.c
@@ -410,6 +410,13 @@ riscv_build_integer_1 (struct riscv_integer_op codes[RISCV_MAX_INTEGER_OPS],
       codes[0].value = value;
       return 1;
     }
+  if (TARGET_ZBS && SINGLE_BIT_MASK_OPERAND (value))
+    {
+      /* Simply BSETI.  */
+      codes[0].code = UNKNOWN;
+      codes[0].value = value;
+      return 1;
+    }
 
   /* End with ADDI.  When constructing HImode constants, do not generate any
      intermediate value that is not itself a valid HImode constant.  The
@@ -462,6 +469,47 @@ riscv_build_integer_1 (struct riscv_integer_op codes[RISCV_MAX_INTEGER_OPS],
 	}
     }
 
+  if (cost > 2 && TARGET_64BIT && TARGET_ZBB)
+    {
+      int leading_ones = clz_hwi (~value);
+      int trailing_ones = ctz_hwi (~value);
+
+      /* If all bits are one except a few that are zero, and the zero bits
+	 are within a range of 11 bits, and at least one of the upper 32-bits
+	 is a zero, then we can generate a constant by loading a small
+	 negative constant and rotating.  */
+      if (leading_ones < 32
+	  && ((64 - leading_ones - trailing_ones) < 12))
+	{
+	  codes[0].code = UNKNOWN;
+	  /* The sign-bit might be zero, so just rotate to be safe.  */
+	  codes[0].value = (((unsigned HOST_WIDE_INT) value >> trailing_ones)
+			    | (value << (64 - trailing_ones)));
+	  codes[1].code = ROTATERT;
+	  codes[1].value = 64 - trailing_ones;
+	  cost = 2;
+	}
+      /* Handle the case where the 11 bit range of zero bits wraps around.  */
+      else
+	{
+	  int upper_trailing_ones = ctz_hwi (~value >> 32);
+	  int lower_leading_ones = clz_hwi (~value << 32);
+
+	  if (upper_trailing_ones < 32 && lower_leading_ones < 32
+	      && ((64 - upper_trailing_ones - lower_leading_ones) < 12))
+	    {
+	      codes[0].code = UNKNOWN;
+	      /* The sign-bit might be zero, so just rotate to be safe.  */
+	      codes[0].value = ((value << (32 - upper_trailing_ones))
+				| ((unsigned HOST_WIDE_INT) value
+				   >> (32 + upper_trailing_ones)));
+	      codes[1].code = ROTATERT;
+	      codes[1].value = 32 - upper_trailing_ones;
+	      cost = 2;
+	    }
+	}
+    }
+
   gcc_assert (cost <= RISCV_MAX_INTEGER_OPS);
   return cost;
 }
@@ -1703,6 +1751,20 @@ riscv_extend_cost (rtx op, bool unsigned_p)
     /* We can use ANDI.  */
     return COSTS_N_INSNS (1);
 
+  /* ZBA provide zext.w.  */
+  if (TARGET_ZBA && TARGET_64BIT && unsigned_p && GET_MODE (op) == SImode)
+    return COSTS_N_INSNS (1);
+
+  /* ZBB provide zext.h, sext.b and sext.h.  */
+  if (TARGET_ZBB)
+    {
+      if (!unsigned_p && GET_MODE (op) == QImode)
+	return COSTS_N_INSNS (1);
+
+      if (GET_MODE (op) == HImode)
+	return COSTS_N_INSNS (1);
+    }
+
   if (!unsigned_p && GET_MODE (op) == SImode)
     /* We can use SEXT.W.  */
     return COSTS_N_INSNS (1);
@@ -1776,8 +1838,60 @@ riscv_rtx_costs (rtx x, machine_mode mode, int outer_code, int opno ATTRIBUTE_UN
       return false;
 
     case AND:
+      /* slli.uw pattern for zba.  */
+      if (TARGET_ZBA && TARGET_64BIT && mode == DImode
+	  && GET_CODE (XEXP (x, 0)) == ASHIFT)
+	{
+	  rtx and_rhs = XEXP (x, 1);
+	  rtx ashift_lhs = XEXP (XEXP (x, 0), 0);
+	  rtx ashift_rhs = XEXP (XEXP (x, 0), 1);
+	  if (REG_P (ashift_lhs)
+	      && CONST_INT_P (ashift_rhs)
+	      && CONST_INT_P (and_rhs)
+	      && ((INTVAL (and_rhs) >> INTVAL (ashift_rhs)) == 0xffffffff))
+	    *total = COSTS_N_INSNS (1);
+	    return true;
+	}
+      /* bclri pattern for zbs.  */
+      if (TARGET_ZBS
+	  && not_single_bit_mask_operand (XEXP (x, 1), VOIDmode))
+	{
+	  *total = COSTS_N_INSNS (1);
+	  return true;
+	}
+      /* bclr pattern for zbs.  */
+      if (TARGET_ZBS
+	  && REG_P (XEXP (x, 1))
+	  && GET_CODE (XEXP (x, 0)) == ROTATE
+	  && CONST_INT_P (XEXP ((XEXP (x, 0)), 0))
+	  && INTVAL (XEXP ((XEXP (x, 0)), 0)) == -2)
+	{
+	  *total = COSTS_N_INSNS (1);
+	  return true;
+	}
+
+      gcc_fallthrough ();
     case IOR:
     case XOR:
+      /* orn, andn and xorn pattern for zbb.  */
+      if (TARGET_ZBB
+	  && GET_CODE (XEXP (x, 0)) == NOT)
+	{
+	  *total = riscv_binary_cost (x, 1, 2);
+	  return true;
+	}
+
+      /* bset[i] and binv[i] pattern for zbs.  */
+      if ((GET_CODE (x) == IOR || GET_CODE (x) == XOR)
+	  && TARGET_ZBS
+	  && ((GET_CODE (XEXP (x, 0)) == ASHIFT
+	      && CONST_INT_P (XEXP (XEXP (x, 0), 0)))
+	      || single_bit_mask_operand (XEXP (x, 1), VOIDmode)))
+	{
+	  *total = COSTS_N_INSNS (1);
+	  return true;
+	}
+
       /* Double-word operations use two single-word operations.  */
       *total = riscv_binary_cost (x, 1, 2);
       return false;
@@ -1793,9 +1907,26 @@ riscv_rtx_costs (rtx x, machine_mode mode, int outer_code, int opno ATTRIBUTE_UN
 	  *total = COSTS_N_INSNS (SINGLE_SHIFT_COST);
 	  return true;
 	}
+      /* bext pattern for zbs.  */
+      if (TARGET_ZBS && outer_code == SET
+	  && GET_CODE (XEXP (x, 1)) == CONST_INT
+	  && INTVAL (XEXP (x, 1)) == 1)
+	{
+	  *total = COSTS_N_INSNS (SINGLE_SHIFT_COST);
+	  return true;
+	}
       return false;
 
     case ASHIFT:
+      /* bset pattern for zbs.  */
+      if (TARGET_ZBS
+	  && CONST_INT_P (XEXP (x, 0))
+	  && INTVAL (XEXP (x, 0)) == 1)
+	{
+	  *total = COSTS_N_INSNS (1);
+	  return true;
+	}
+      gcc_fallthrough ();
     case ASHIFTRT:
     case LSHIFTRT:
       *total = riscv_binary_cost (x, SINGLE_SHIFT_COST,
@@ -1867,6 +1998,67 @@ riscv_rtx_costs (rtx x, machine_mode mode, int outer_code, int opno ATTRIBUTE_UN
 
     case MINUS:
     case PLUS:
+      /* add.uw pattern for zba.  */
+      if (TARGET_ZBA
+	  && (TARGET_64BIT && (mode == DImode))
+	  && GET_CODE (XEXP (x, 0)) == ZERO_EXTEND
+	  && REG_P (XEXP (XEXP (x, 0), 0))
+	  && GET_MODE (XEXP (XEXP (x, 0), 0)) == SImode)
+	{
+	  *total = COSTS_N_INSNS (1);
+	  return true;
+	}
+      /* shNadd pattern for zba.  */
+      if (TARGET_ZBA
+	  && ((!TARGET_64BIT && (mode == SImode)) ||
+	      (TARGET_64BIT && (mode == DImode)))
+	  && (GET_CODE (XEXP (x, 0)) == ASHIFT)
+	  && REG_P (XEXP (XEXP (x, 0), 0))
+	  && CONST_INT_P (XEXP (XEXP (x, 0), 1))
+	  && IN_RANGE (INTVAL (XEXP (XEXP (x, 0), 1)), 1, 3))
+	{
+	  *total = COSTS_N_INSNS (1);
+	  return true;
+	}
+      /* shNadd.uw pattern for zba.
+	 [(set (match_operand:DI 0 "register_operand" "=r")
+	       (plus:DI
+		 (and:DI (ashift:DI (match_operand:DI 1 "register_operand" "r")
+				    (match_operand:QI 2 "immediate_operand" "I"))
+			 (match_operand 3 "immediate_operand" ""))
+		 (match_operand:DI 4 "register_operand" "r")))]
+	 "TARGET_64BIT && TARGET_ZBA
+	  && (INTVAL (operands[2]) >= 1) && (INTVAL (operands[2]) <= 3)
+	  && (INTVAL (operands[3]) >> INTVAL (operands[2])) == 0xffffffff"
+      */
+      if (TARGET_ZBA
+	  && (TARGET_64BIT && (mode == DImode))
+	  && (GET_CODE (XEXP (x, 0)) == AND)
+	  && (REG_P (XEXP (x, 1))))
+	{
+	  do {
+	    rtx and_lhs = XEXP (XEXP (x, 0), 0);
+	    rtx and_rhs = XEXP (XEXP (x, 0), 1);
+	    if (GET_CODE (and_lhs) != ASHIFT)
+	      break;
+	    if (!CONST_INT_P (and_rhs))
+	      break;
+
+	    rtx ashift_rhs = XEXP (and_lhs, 1);
+
+	    if (!CONST_INT_P (ashift_rhs)
+		|| !IN_RANGE (INTVAL (ashift_rhs), 1, 3))
+	      break;
+
+	    if (CONST_INT_P (and_rhs)
+		&& ((INTVAL (and_rhs) >> INTVAL (ashift_rhs)) == 0xffffffff))
+	      {
+		*total = COSTS_N_INSNS (1);
+		return true;
+	      }
+	  } while (false);
+	}
+
       if (float_mode_p)
 	*total = tune_param->fp_add[mode == DFmode];
       else
@@ -2081,7 +2273,17 @@ riscv_output_move (rtx dest, rtx src)
 	  }
 
       if (src_code == CONST_INT)
-	return "li\t%0,%1";
+	{
+	  if (SMALL_OPERAND (INTVAL (src)) || LUI_OPERAND (INTVAL (src)))
+	    return "li\t%0,%1";
+
+	  if (TARGET_ZBS
+	      && SINGLE_BIT_MASK_OPERAND (INTVAL (src)))
+	    return "bseti\t%0,zero,%S1";
+
+	  /* Should never reach here.  */
+	  abort ();
+	}
 
       if (src_code == HIGH)
 	return "lui\t%0,%h1";
@@ -3422,7 +3624,9 @@ riscv_memmodel_needs_release_fence (enum memmodel model)
    'A'	Print the atomic operation suffix for memory model OP.
    'F'	Print a FENCE if the memory model requires a release.
    'z'	Print x0 if OP is zero, otherwise print OP normally.
-   'i'	Print i if the operand is not a register.  */
+   'i'	Print i if the operand is not a register.
+   'S'	Print shift-index of single-bit mask OP.
+   'T'	Print shift-index of inverted single-bit mask OP.  */
 
 static void
 riscv_print_operand (FILE *file, rtx op, int letter)
@@ -3462,6 +3666,18 @@ riscv_print_operand (FILE *file, rtx op, int letter)
         fputs ("i", file);
       break;
 
+    case 'S':
+      {
+	rtx newop = GEN_INT (ctz_hwi (INTVAL (op)));
+	output_addr_const (file, newop);
+	break;
+      }
+    case 'T':
+      {
+	rtx newop = GEN_INT (ctz_hwi (~INTVAL (op)));
+	output_addr_const (file, newop);
+	break;
+      }
     default:
       switch (code)
 	{
@@ -4594,10 +4810,10 @@ riscv_modes_tieable_p (machine_mode mode1, machine_mode mode2)
 static unsigned char
 riscv_class_max_nregs (reg_class_t rclass, machine_mode mode)
 {
-  if (reg_class_subset_p (FP_REGS, rclass))
+  if (reg_class_subset_p (rclass, FP_REGS))
     return riscv_hard_regno_nregs (FP_REG_FIRST, mode);
 
-  if (reg_class_subset_p (GR_REGS, rclass))
+  if (reg_class_subset_p (rclass, GR_REGS))
     return riscv_hard_regno_nregs (GP_REG_FIRST, mode);
 
   return 0;
diff --git a/gcc/config/riscv/riscv.h b/gcc/config/riscv/riscv.h
index f47d5b4..6428712 100644
--- a/gcc/config/riscv/riscv.h
+++ b/gcc/config/riscv/riscv.h
@@ -526,6 +526,14 @@ enum reg_class
   (((VALUE) | ((1UL<<31) - IMM_REACH)) == ((1UL<<31) - IMM_REACH)	\
    || ((VALUE) | ((1UL<<31) - IMM_REACH)) + IMM_REACH == 0)
 
+/* If this is a single bit mask, then we can load it with bseti.  But this
+   is not useful for any of the low 31 bits because we can use addi or lui
+   to load them.  It is wrong for loading SImode 0x80000000 on rv64 because it
+   needs to be sign-extended.  So we restrict this to the upper 32-bits
+   only.  */
+#define SINGLE_BIT_MASK_OPERAND(VALUE) \
+  (pow2p_hwi (VALUE) && (ctz_hwi (VALUE) >= 32))
+
 /* Stack layout; function entry, exit and calling.  */
 
 #define STACK_GROWS_DOWNWARD 1
diff --git a/gcc/config/riscv/riscv.md b/gcc/config/riscv/riscv.md
index 98364f0..225e5b2 100644
--- a/gcc/config/riscv/riscv.md
+++ b/gcc/config/riscv/riscv.md
@@ -170,7 +170,7 @@
 (define_attr "type"
   "unknown,branch,jump,call,load,fpload,store,fpstore,
    mtc,mfc,const,arith,logical,shift,slt,imul,idiv,move,fmove,fadd,fmul,
-   fmadd,fdiv,fcmp,fcvt,fsqrt,multi,auipc,sfb_alu,nop,ghost"
+   fmadd,fdiv,fcmp,fcvt,fsqrt,multi,auipc,sfb_alu,nop,ghost,bitmanip,rotate"
   (cond [(eq_attr "got" "load") (const_string "load")
 
 	 ;; If a doubleword move uses these expensive instructions,
@@ -1302,11 +1302,16 @@
 
 ;; Extension insns.
 
-(define_insn_and_split "zero_extendsidi2"
+(define_expand "zero_extendsidi2"
+  [(set (match_operand:DI 0 "register_operand")
+	(zero_extend:DI (match_operand:SI 1 "nonimmediate_operand")))]
+  "TARGET_64BIT")
+
+(define_insn_and_split "*zero_extendsidi2_internal"
   [(set (match_operand:DI     0 "register_operand"     "=r,r")
 	(zero_extend:DI
 	    (match_operand:SI 1 "nonimmediate_operand" " r,m")))]
-  "TARGET_64BIT"
+  "TARGET_64BIT && !TARGET_ZBA"
   "@
    #
    lwu\t%0,%1"
@@ -1321,11 +1326,17 @@
   [(set_attr "move_type" "shift_shift,load")
    (set_attr "mode" "DI")])
 
-(define_insn_and_split "zero_extendhi<GPR:mode>2"
+(define_expand "zero_extendhi<GPR:mode>2"
+  [(set (match_operand:GPR    0 "register_operand")
+	(zero_extend:GPR
+	    (match_operand:HI 1 "nonimmediate_operand")))]
+  "")
+
+(define_insn_and_split "*zero_extendhi<GPR:mode>2"
   [(set (match_operand:GPR    0 "register_operand"     "=r,r")
 	(zero_extend:GPR
 	    (match_operand:HI 1 "nonimmediate_operand" " r,m")))]
-  ""
+  "!TARGET_ZBB"
   "@
    #
    lhu\t%0,%1"
@@ -1372,11 +1383,16 @@
   [(set_attr "move_type" "move,load")
    (set_attr "mode" "DI")])
 
-(define_insn_and_split "extend<SHORT:mode><SUPERQI:mode>2"
+(define_expand "extend<SHORT:mode><SUPERQI:mode>2"
+  [(set (match_operand:SUPERQI 0 "register_operand")
+	(sign_extend:SUPERQI (match_operand:SHORT 1 "nonimmediate_operand")))]
+  "")
+
+(define_insn_and_split "*extend<SHORT:mode><SUPERQI:mode>2"
   [(set (match_operand:SUPERQI   0 "register_operand"     "=r,r")
 	(sign_extend:SUPERQI
 	    (match_operand:SHORT 1 "nonimmediate_operand" " r,m")))]
-  ""
+  "!TARGET_ZBB"
   "@
    #
    l<SHORT:size>\t%0,%1"
@@ -2078,7 +2094,7 @@
 			   (match_operand:QI 2 "immediate_operand" "I"))
 		(match_operand 3 "immediate_operand" "")))
    (clobber (match_scratch:DI 4 "=&r"))]
-  "TARGET_64BIT
+  "TARGET_64BIT && !TARGET_ZBA
    && ((INTVAL (operands[3]) >> INTVAL (operands[2])) == 0xffffffff)"
   "#"
   "&& reload_completed"
@@ -2845,6 +2861,7 @@
   "<load>\t%3, %1\;<load>\t%0, %2\;xor\t%0, %3, %0\;li\t%3, 0"
   [(set_attr "length" "12")])
 
+(include "bitmanip.md")
 (include "sync.md")
 (include "peephole.md")
 (include "pic.md")
diff --git a/gcc/config/riscv/riscv.opt b/gcc/config/riscv/riscv.opt
index 5ff85c2..15bf89e 100644
--- a/gcc/config/riscv/riscv.opt
+++ b/gcc/config/riscv/riscv.opt
@@ -195,6 +195,9 @@ long riscv_stack_protector_guard_offset = 0
 TargetVariable
 int riscv_zi_subext
 
+TargetVariable
+int riscv_zb_subext
+
 Enum
 Name(isa_spec_class) Type(enum riscv_isa_spec_class)
 Supported ISA specs (for use with the -misa-spec= option):
diff --git a/gcc/config/rs6000/altivec.md b/gcc/config/rs6000/altivec.md
index 51a4760..a057218 100644
--- a/gcc/config/rs6000/altivec.md
+++ b/gcc/config/rs6000/altivec.md
@@ -681,27 +681,69 @@
   "vcmpgefp %0,%1,%2"
   [(set_attr "type" "veccmp")])
 
-(define_insn "*altivec_vsel<mode>"
-  [(set (match_operand:VM 0 "altivec_register_operand" "=v")
-	(if_then_else:VM
-	 (ne:CC (match_operand:VM 1 "altivec_register_operand" "v")
-		(match_operand:VM 4 "zero_constant" ""))
-	 (match_operand:VM 2 "altivec_register_operand" "v")
-	 (match_operand:VM 3 "altivec_register_operand" "v")))]
-  "VECTOR_MEM_ALTIVEC_P (<MODE>mode)"
-  "vsel %0,%3,%2,%1"
-  [(set_attr "type" "vecmove")])
-
-(define_insn "*altivec_vsel<mode>_uns"
-  [(set (match_operand:VM 0 "altivec_register_operand" "=v")
-	(if_then_else:VM
-	 (ne:CCUNS (match_operand:VM 1 "altivec_register_operand" "v")
-		   (match_operand:VM 4 "zero_constant" ""))
-	 (match_operand:VM 2 "altivec_register_operand" "v")
-	 (match_operand:VM 3 "altivec_register_operand" "v")))]
-  "VECTOR_MEM_ALTIVEC_P (<MODE>mode)"
-  "vsel %0,%3,%2,%1"
-  [(set_attr "type" "vecmove")])
+(define_insn "altivec_vsel<mode>"
+  [(set (match_operand:VM 0 "register_operand" "=wa,v")
+	(ior:VM
+	  (and:VM
+	    (not:VM (match_operand:VM 3 "register_operand" "wa,v"))
+	    (match_operand:VM 1 "register_operand" "wa,v"))
+	  (and:VM
+	    (match_dup 3)
+	    (match_operand:VM 2 "register_operand" "wa,v"))))]
+  "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
+  "@
+   xxsel %x0,%x1,%x2,%x3
+   vsel %0,%1,%2,%3"
+  [(set_attr "type" "vecmove")
+   (set_attr "isa" "<VSisa>")])
+
+(define_insn "altivec_vsel<mode>2"
+  [(set (match_operand:VM 0 "register_operand" "=wa,v")
+	(ior:VM
+	  (and:VM
+	    (not:VM (match_operand:VM 3 "register_operand" "wa,v"))
+	    (match_operand:VM 1 "register_operand" "wa,v"))
+	  (and:VM
+	    (match_operand:VM 2 "register_operand" "wa,v")
+	    (match_dup 3))))]
+  "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
+  "@
+   xxsel %x0,%x1,%x2,%x3
+   vsel %0,%1,%2,%3"
+  [(set_attr "type" "vecmove")
+   (set_attr "isa" "<VSisa>")])
+
+(define_insn "altivec_vsel<mode>3"
+  [(set (match_operand:VM 0 "register_operand" "=wa,v")
+	(ior:VM
+	  (and:VM
+	    (match_operand:VM 3 "register_operand" "wa,v")
+	    (match_operand:VM 1 "register_operand" "wa,v"))
+	  (and:VM
+	    (not:VM (match_dup 3))
+	    (match_operand:VM 2 "register_operand" "wa,v"))))]
+  "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
+  "@
+   xxsel %x0,%x2,%x1,%x3
+   vsel %0,%2,%1,%3"
+  [(set_attr "type" "vecmove")
+   (set_attr "isa" "<VSisa>")])
+
+(define_insn "altivec_vsel<mode>4"
+  [(set (match_operand:VM 0 "register_operand" "=wa,v")
+	(ior:VM
+	  (and:VM
+	    (match_operand:VM 1 "register_operand" "wa,v")
+	    (match_operand:VM 3 "register_operand" "wa,v"))
+	  (and:VM
+	    (not:VM (match_dup 3))
+	    (match_operand:VM 2 "register_operand" "wa,v"))))]
+  "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
+  "@
+   xxsel %x0,%x2,%x1,%x3
+   vsel %0,%2,%1,%3"
+  [(set_attr "type" "vecmove")
+   (set_attr "isa" "<VSisa>")])
 
 ;; Fused multiply add.
 
diff --git a/gcc/config/rs6000/fusion.md b/gcc/config/rs6000/fusion.md
index 516baa0..d11cecb 100644
--- a/gcc/config/rs6000/fusion.md
+++ b/gcc/config/rs6000/fusion.md
@@ -1874,7 +1874,7 @@
         (and:VM (and:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "%v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vand %3,%1,%0\;vand %3,%3,%2
@@ -1892,7 +1892,7 @@
         (and:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vandc %3,%1,%0\;vand %3,%3,%2
@@ -1910,7 +1910,7 @@
         (and:VM (not:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    veqv %3,%1,%0\;vand %3,%3,%2
@@ -1928,7 +1928,7 @@
         (and:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnand %3,%1,%0\;vand %3,%3,%2
@@ -1946,7 +1946,7 @@
         (and:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnor %3,%1,%0\;vand %3,%3,%2
@@ -1964,7 +1964,7 @@
         (and:VM (ior:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vor %3,%1,%0\;vand %3,%3,%2
@@ -1982,7 +1982,7 @@
         (and:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vorc %3,%1,%0\;vand %3,%3,%2
@@ -2000,7 +2000,7 @@
         (and:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vxor %3,%1,%0\;vand %3,%3,%2
@@ -2018,7 +2018,7 @@
         (and:VM (and:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vand %3,%1,%0\;vandc %3,%3,%2
@@ -2036,7 +2036,7 @@
         (and:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vandc %3,%1,%0\;vandc %3,%3,%2
@@ -2054,7 +2054,7 @@
         (and:VM (not:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    veqv %3,%1,%0\;vandc %3,%3,%2
@@ -2072,7 +2072,7 @@
         (and:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnand %3,%1,%0\;vandc %3,%3,%2
@@ -2090,7 +2090,7 @@
         (and:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnor %3,%1,%0\;vandc %3,%3,%2
@@ -2108,7 +2108,7 @@
         (and:VM (ior:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vor %3,%1,%0\;vandc %3,%3,%2
@@ -2126,7 +2126,7 @@
         (and:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vorc %3,%1,%0\;vandc %3,%3,%2
@@ -2144,7 +2144,7 @@
         (and:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vxor %3,%1,%0\;vandc %3,%3,%2
@@ -2162,7 +2162,7 @@
         (not:VM (xor:VM (and:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vand %3,%1,%0\;veqv %3,%3,%2
@@ -2180,7 +2180,7 @@
         (not:VM (xor:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vandc %3,%1,%0\;veqv %3,%3,%2
@@ -2198,7 +2198,7 @@
         (not:VM (xor:VM (not:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "%v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    veqv %3,%1,%0\;veqv %3,%3,%2
@@ -2216,7 +2216,7 @@
         (not:VM (xor:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnand %3,%1,%0\;veqv %3,%3,%2
@@ -2234,7 +2234,7 @@
         (not:VM (xor:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnor %3,%1,%0\;veqv %3,%3,%2
@@ -2252,7 +2252,7 @@
         (not:VM (xor:VM (ior:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vor %3,%1,%0\;veqv %3,%3,%2
@@ -2270,7 +2270,7 @@
         (not:VM (xor:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vorc %3,%1,%0\;veqv %3,%3,%2
@@ -2288,7 +2288,7 @@
         (not:VM (xor:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vxor %3,%1,%0\;veqv %3,%3,%2
@@ -2306,7 +2306,7 @@
         (ior:VM (not:VM (and:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vand %3,%1,%0\;vnand %3,%3,%2
@@ -2324,7 +2324,7 @@
         (ior:VM (not:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vandc %3,%1,%0\;vnand %3,%3,%2
@@ -2342,7 +2342,7 @@
         (ior:VM (not:VM (not:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    veqv %3,%1,%0\;vnand %3,%3,%2
@@ -2360,7 +2360,7 @@
         (ior:VM (not:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnand %3,%1,%0\;vnand %3,%3,%2
@@ -2378,7 +2378,7 @@
         (ior:VM (not:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnor %3,%1,%0\;vnand %3,%3,%2
@@ -2396,7 +2396,7 @@
         (ior:VM (not:VM (ior:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vor %3,%1,%0\;vnand %3,%3,%2
@@ -2414,7 +2414,7 @@
         (ior:VM (not:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vorc %3,%1,%0\;vnand %3,%3,%2
@@ -2432,7 +2432,7 @@
         (ior:VM (not:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vxor %3,%1,%0\;vnand %3,%3,%2
@@ -2450,7 +2450,7 @@
         (and:VM (not:VM (and:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vand %3,%1,%0\;vnor %3,%3,%2
@@ -2468,7 +2468,7 @@
         (and:VM (not:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vandc %3,%1,%0\;vnor %3,%3,%2
@@ -2486,7 +2486,7 @@
         (and:VM (not:VM (not:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    veqv %3,%1,%0\;vnor %3,%3,%2
@@ -2504,7 +2504,7 @@
         (and:VM (not:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnand %3,%1,%0\;vnor %3,%3,%2
@@ -2522,7 +2522,7 @@
         (and:VM (not:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnor %3,%1,%0\;vnor %3,%3,%2
@@ -2540,7 +2540,7 @@
         (and:VM (not:VM (ior:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vor %3,%1,%0\;vnor %3,%3,%2
@@ -2558,7 +2558,7 @@
         (and:VM (not:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vorc %3,%1,%0\;vnor %3,%3,%2
@@ -2576,7 +2576,7 @@
         (and:VM (not:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vxor %3,%1,%0\;vnor %3,%3,%2
@@ -2594,7 +2594,7 @@
         (ior:VM (and:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vand %3,%1,%0\;vor %3,%3,%2
@@ -2612,7 +2612,7 @@
         (ior:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vandc %3,%1,%0\;vor %3,%3,%2
@@ -2630,7 +2630,7 @@
         (ior:VM (not:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    veqv %3,%1,%0\;vor %3,%3,%2
@@ -2648,7 +2648,7 @@
         (ior:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnand %3,%1,%0\;vor %3,%3,%2
@@ -2666,7 +2666,7 @@
         (ior:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnor %3,%1,%0\;vor %3,%3,%2
@@ -2684,7 +2684,7 @@
         (ior:VM (ior:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "%v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vor %3,%1,%0\;vor %3,%3,%2
@@ -2702,7 +2702,7 @@
         (ior:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vorc %3,%1,%0\;vor %3,%3,%2
@@ -2720,7 +2720,7 @@
         (ior:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vxor %3,%1,%0\;vor %3,%3,%2
@@ -2738,7 +2738,7 @@
         (ior:VM (and:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vand %3,%1,%0\;vorc %3,%3,%2
@@ -2756,7 +2756,7 @@
         (ior:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vandc %3,%1,%0\;vorc %3,%3,%2
@@ -2774,7 +2774,7 @@
         (ior:VM (not:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    veqv %3,%1,%0\;vorc %3,%3,%2
@@ -2792,7 +2792,7 @@
         (ior:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnand %3,%1,%0\;vorc %3,%3,%2
@@ -2810,7 +2810,7 @@
         (ior:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnor %3,%1,%0\;vorc %3,%3,%2
@@ -2828,7 +2828,7 @@
         (ior:VM (ior:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vor %3,%1,%0\;vorc %3,%3,%2
@@ -2846,7 +2846,7 @@
         (ior:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vorc %3,%1,%0\;vorc %3,%3,%2
@@ -2864,7 +2864,7 @@
         (ior:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (not:VM (match_operand:VM 2 "altivec_register_operand" "v,v,v,v"))))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vxor %3,%1,%0\;vorc %3,%3,%2
@@ -2882,7 +2882,7 @@
         (xor:VM (and:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vand %3,%1,%0\;vxor %3,%3,%2
@@ -2900,7 +2900,7 @@
         (xor:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vandc %3,%1,%0\;vxor %3,%3,%2
@@ -2918,7 +2918,7 @@
         (xor:VM (not:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    veqv %3,%1,%0\;vxor %3,%3,%2
@@ -2936,7 +2936,7 @@
         (xor:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnand %3,%1,%0\;vxor %3,%3,%2
@@ -2954,7 +2954,7 @@
         (xor:VM (and:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (not:VM (match_operand:VM 1 "altivec_register_operand" "v,v,v,v")))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vnor %3,%1,%0\;vxor %3,%3,%2
@@ -2972,7 +2972,7 @@
         (xor:VM (ior:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vor %3,%1,%0\;vxor %3,%3,%2
@@ -2990,7 +2990,7 @@
         (xor:VM (ior:VM (not:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v"))
                           (match_operand:VM 1 "altivec_register_operand" "v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vorc %3,%1,%0\;vxor %3,%3,%2
@@ -3008,7 +3008,7 @@
         (xor:VM (xor:VM (match_operand:VM 0 "altivec_register_operand" "v,v,v,v")
                           (match_operand:VM 1 "altivec_register_operand" "%v,v,v,v"))
                  (match_operand:VM 2 "altivec_register_operand" "v,v,v,v")))
-   (clobber (match_scratch:VM 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:VM 4 "=X,X,X,&v"))]
   "(TARGET_P10_FUSION && TARGET_P10_FUSION_2LOGICAL)"
   "@
    vxor %3,%1,%0\;vxor %3,%3,%2
diff --git a/gcc/config/rs6000/genfusion.pl b/gcc/config/rs6000/genfusion.pl
index ac22852..0fef75b 100755
--- a/gcc/config/rs6000/genfusion.pl
+++ b/gcc/config/rs6000/genfusion.pl
@@ -265,7 +265,7 @@ sub gen_logical_addsubf
 (define_insn "*fuse_${inner_op}_${outer_name}"
   [(set (match_operand:${mode} 3 "${pred}" "=&0,&1,&${constraint},${constraint}")
         ${outer_exp})
-   (clobber (match_scratch:${mode} 4 "=X,X,X,&r"))]
+   (clobber (match_scratch:${mode} 4 "=X,X,X,&${constraint}"))]
   "(TARGET_P10_FUSION && $target_flag)"
   "@
    ${inner_op} %3,%1,%0\\;${outer_op} %3,${outer_32}
diff --git a/gcc/config/rs6000/immintrin.h b/gcc/config/rs6000/immintrin.h
new file mode 100644
index 0000000..647a5ae
--- /dev/null
+++ b/gcc/config/rs6000/immintrin.h
@@ -0,0 +1,41 @@
+/* Copyright (C) 2021 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef _IMMINTRIN_H_INCLUDED
+#define _IMMINTRIN_H_INCLUDED
+
+#include <x86gprintrin.h>
+
+#include <mmintrin.h>
+
+#include <xmmintrin.h>
+
+#include <emmintrin.h>
+
+#include <pmmintrin.h>
+
+#include <tmmintrin.h>
+
+#include <smmintrin.h>
+
+#endif /* _IMMINTRIN_H_INCLUDED */
diff --git a/gcc/config/rs6000/rbtree.c b/gcc/config/rs6000/rbtree.c
index 37a559c..d3d03a6 100644
--- a/gcc/config/rs6000/rbtree.c
+++ b/gcc/config/rs6000/rbtree.c
@@ -1,5 +1,5 @@
 /* Partial red-black tree implementation for rs6000-gen-builtins.c.
-   Copyright (C) 2020-21 Free Software Foundation, Inc.
+   Copyright (C) 2020-2021 Free Software Foundation, Inc.
    Contributed by Bill Schmidt, IBM <wschmidt@linux.ibm.com>
 
 This file is part of GCC.
diff --git a/gcc/config/rs6000/rbtree.h b/gcc/config/rs6000/rbtree.h
index fab0001..bd462f1 100644
--- a/gcc/config/rs6000/rbtree.h
+++ b/gcc/config/rs6000/rbtree.h
@@ -1,5 +1,5 @@
 /* Partial red-black tree implementation for rs6000-gen-builtins.c.
-   Copyright (C) 2020-21 Free Software Foundation, Inc.
+   Copyright (C) 2020-2021 Free Software Foundation, Inc.
    Contributed by Bill Schmidt, IBM <wschmidt@linux.ibm.com>
 
 This file is part of GCC.
diff --git a/gcc/config/rs6000/rs6000-builtin-new.def b/gcc/config/rs6000/rs6000-builtin-new.def
index 1966516..1dd8f6b 100644
--- a/gcc/config/rs6000/rs6000-builtin-new.def
+++ b/gcc/config/rs6000/rs6000-builtin-new.def
@@ -1,5 +1,5 @@
 ; Built-in functions for PowerPC.
-; Copyright (C) 2020-21 Free Software Foundation, Inc.
+; Copyright (C) 2020-2021 Free Software Foundation, Inc.
 ; Contributed by Bill Schmidt, IBM <wschmidt@linux.ibm.com>
 ;
 ; This file is part of GCC.
@@ -1094,16 +1094,22 @@
   const signed short __builtin_vec_ext_v8hi (vss, signed int);
     VEC_EXT_V8HI nothing {extract}
 
-  const vsc __builtin_vec_init_v16qi (signed char, signed char, signed char, signed char, signed char, signed char, signed char, signed char, signed char, signed char, signed char, signed char, signed char, signed char, signed char, signed char);
+  const vsc __builtin_vec_init_v16qi (signed char, signed char, signed char, \
+            signed char, signed char, signed char, signed char, signed char, \
+            signed char, signed char, signed char, signed char, signed char, \
+            signed char, signed char, signed char);
     VEC_INIT_V16QI nothing {init}
 
   const vf __builtin_vec_init_v4sf (float, float, float, float);
     VEC_INIT_V4SF nothing {init}
 
-  const vsi __builtin_vec_init_v4si (signed int, signed int, signed int, signed int);
+  const vsi __builtin_vec_init_v4si (signed int, signed int, signed int, \
+                                     signed int);
     VEC_INIT_V4SI nothing {init}
 
-  const vss __builtin_vec_init_v8hi (signed short, signed short, signed short, signed short, signed short, signed short, signed short, signed short);
+  const vss __builtin_vec_init_v8hi (signed short, signed short, signed short,\
+             signed short, signed short, signed short, signed short, \
+             signed short);
     VEC_INIT_V8HI nothing {init}
 
   const vsc __builtin_vec_set_v16qi (vsc, signed char, const int<4>);
@@ -2023,7 +2029,8 @@
   const unsigned int __builtin_divweu (unsigned int, unsigned int);
     DIVWEU diveu_si {}
 
-  const vsq __builtin_pack_vector_int128 (unsigned long long, unsigned long long);
+  const vsq __builtin_pack_vector_int128 (unsigned long long, \
+                                          unsigned long long);
     PACK_V1TI packv1ti {}
 
   void __builtin_ppc_speculation_barrier ();
@@ -2038,7 +2045,8 @@
   const signed long long __builtin_divde (signed long long, signed long long);
     DIVDE dive_di {}
 
-  const unsigned long long __builtin_divdeu (unsigned long long, unsigned long long);
+  const unsigned long long __builtin_divdeu (unsigned long long, \
+                                             unsigned long long);
     DIVDEU diveu_di {}
 
 
@@ -2515,13 +2523,16 @@
   const signed int __builtin_altivec_first_mismatch_index_v4si (vsi, vsi);
     VFIRSTMISMATCHINDEX_V4SI first_mismatch_index_v4si {}
 
-  const signed int __builtin_altivec_first_mismatch_or_eos_index_v16qi (vsc, vsc);
+  const signed int \
+      __builtin_altivec_first_mismatch_or_eos_index_v16qi (vsc, vsc);
     VFIRSTMISMATCHOREOSINDEX_V16QI first_mismatch_or_eos_index_v16qi {}
 
-  const signed int __builtin_altivec_first_mismatch_or_eos_index_v8hi (vss, vss);
+  const signed int \
+      __builtin_altivec_first_mismatch_or_eos_index_v8hi (vss, vss);
     VFIRSTMISMATCHOREOSINDEX_V8HI first_mismatch_or_eos_index_v8hi {}
 
-  const signed int __builtin_altivec_first_mismatch_or_eos_index_v4si (vsi, vsi);
+  const signed int \
+      __builtin_altivec_first_mismatch_or_eos_index_v4si (vsi, vsi);
     VFIRSTMISMATCHOREOSINDEX_V4SI first_mismatch_or_eos_index_v4si {}
 
   const vsc __builtin_altivec_vadub (vsc, vsc);
@@ -2695,7 +2706,8 @@
   const signed int __builtin_scalar_byte_in_range (signed int, signed int);
     CMPRB cmprb {}
 
-  const signed int __builtin_scalar_byte_in_either_range (signed int, signed int);
+  const signed int \
+      __builtin_scalar_byte_in_either_range (signed int, signed int);
     CMPRB2 cmprb2 {}
 
   const vsll __builtin_vsx_extract4b (vsc, const int[0,12]);
@@ -2734,10 +2746,12 @@
   const signed int __builtin_vsx_scalar_cmp_exp_dp_unordered (double, double);
     VSCEDPUO xscmpexpdp_unordered {}
 
-  const signed int __builtin_vsx_scalar_test_data_class_dp (double, const int<7>);
+  const signed int \
+      __builtin_vsx_scalar_test_data_class_dp (double, const int<7>);
     VSTDCDP xststdcdp {}
 
-  const signed int __builtin_vsx_scalar_test_data_class_sp (float, const int<7>);
+  const signed int \
+      __builtin_vsx_scalar_test_data_class_sp (float, const int<7>);
     VSTDCSP xststdcsp {}
 
   const signed int __builtin_vsx_scalar_test_neg_dp (double);
@@ -2835,7 +2849,8 @@
   const signed long __builtin_vsx_scalar_extract_sig (double);
     VSESDP xsxsigdp {}
 
-  const double __builtin_vsx_scalar_insert_exp (unsigned long long, unsigned long long);
+  const double __builtin_vsx_scalar_insert_exp (unsigned long long, \
+                                                unsigned long long);
     VSIEDP xsiexpdp {}
 
   const double __builtin_vsx_scalar_insert_exp_dp (double, unsigned long long);
@@ -2853,7 +2868,8 @@
   fpmath _Float128 __builtin_divf128_round_to_odd (_Float128, _Float128);
     DIVF128_ODD divkf3_odd {}
 
-  fpmath _Float128 __builtin_fmaf128_round_to_odd (_Float128, _Float128, _Float128);
+  fpmath _Float128 __builtin_fmaf128_round_to_odd (_Float128, _Float128, \
+                                                   _Float128);
     FMAF128_ODD fmakf4_odd {}
 
   fpmath _Float128 __builtin_mulf128_round_to_odd (_Float128, _Float128);
@@ -2868,7 +2884,8 @@
   const signed int __builtin_vsx_scalar_cmp_exp_qp_lt (_Float128, _Float128);
     VSCEQPLT xscmpexpqp_lt_kf {}
 
-  const signed int __builtin_vsx_scalar_cmp_exp_qp_unordered (_Float128, _Float128);
+  const signed int \
+      __builtin_vsx_scalar_cmp_exp_qp_unordered (_Float128, _Float128);
     VSCEQPUO xscmpexpqp_unordered_kf {}
 
   fpmath _Float128 __builtin_sqrtf128_round_to_odd (_Float128);
@@ -2886,13 +2903,16 @@
   const signed __int128 __builtin_vsx_scalar_extract_sigq (_Float128);
     VSESQP xsxsigqp_kf {}
 
-  const _Float128 __builtin_vsx_scalar_insert_exp_q (unsigned __int128, unsigned long long);
+  const _Float128 __builtin_vsx_scalar_insert_exp_q (unsigned __int128, \
+                                                     unsigned long long);
     VSIEQP xsiexpqp_kf {}
 
-  const _Float128 __builtin_vsx_scalar_insert_exp_qp (_Float128, unsigned long long);
+  const _Float128 __builtin_vsx_scalar_insert_exp_qp (_Float128, \
+                                                      unsigned long long);
     VSIEQPF xsiexpqpf_kf {}
 
-  const signed int __builtin_vsx_scalar_test_data_class_qp (_Float128, const int<7>);
+  const signed int __builtin_vsx_scalar_test_data_class_qp (_Float128, \
+                                                            const int<7>);
     VSTDCQP xststdcqp_kf {}
 
   const signed int __builtin_vsx_scalar_test_neg_qp (_Float128);
@@ -2941,7 +2961,8 @@
   const signed long long __builtin_dxexq (_Decimal128);
     DXEXQ dfp_dxex_td {}
 
-  const _Decimal128 __builtin_pack_dec128 (unsigned long long, unsigned long long);
+  const _Decimal128 __builtin_pack_dec128 (unsigned long long, \
+                                           unsigned long long);
     PACK_TD packtd {}
 
   void __builtin_set_fpscr_drn (const int[0,7]);
@@ -3017,10 +3038,12 @@
   unsigned int __builtin_tabort (unsigned int);
     TABORT tabort {htm,htmcr}
 
-  unsigned int __builtin_tabortdc (unsigned long long, unsigned long long, unsigned long long);
+  unsigned int __builtin_tabortdc (unsigned long long, unsigned long long, \
+                                   unsigned long long);
     TABORTDC tabortdc {htm,htmcr}
 
-  unsigned int __builtin_tabortdci (unsigned long long, unsigned long long, unsigned long long);
+  unsigned int __builtin_tabortdci (unsigned long long, unsigned long long, \
+                                    unsigned long long);
     TABORTDCI tabortdci {htm,htmcr}
 
   unsigned int __builtin_tabortwc (unsigned int, unsigned int, unsigned int);
@@ -3115,16 +3138,20 @@
   const vui __builtin_altivec_mtvsrwm (unsigned long long);
     MTVSRWM vec_mtvsr_v4si {}
 
-  pure signed __int128 __builtin_altivec_se_lxvrbx (signed long, const signed char *);
+  pure signed __int128 __builtin_altivec_se_lxvrbx (signed long, \
+                                                    const signed char *);
     SE_LXVRBX vsx_lxvrbx {lxvrse}
 
-  pure signed __int128 __builtin_altivec_se_lxvrhx (signed long, const signed short *);
+  pure signed __int128 __builtin_altivec_se_lxvrhx (signed long, \
+                                                    const signed short *);
     SE_LXVRHX vsx_lxvrhx {lxvrse}
 
-  pure signed __int128 __builtin_altivec_se_lxvrwx (signed long, const signed int *);
+  pure signed __int128 __builtin_altivec_se_lxvrwx (signed long, \
+                                                    const signed int *);
     SE_LXVRWX vsx_lxvrwx {lxvrse}
 
-  pure signed __int128 __builtin_altivec_se_lxvrdx (signed long, const signed long long *);
+  pure signed __int128 __builtin_altivec_se_lxvrdx (signed long, \
+                                                    const signed long long *);
     SE_LXVRDX vsx_lxvrdx {lxvrse}
 
   void __builtin_altivec_tr_stxvrbx (vsq, signed long, signed char *);
@@ -3358,16 +3385,19 @@
   const vull __builtin_altivec_vpextd (vull, vull);
     VPEXTD vpextd {}
 
-  const vull __builtin_altivec_vreplace_un_uv2di (vull, unsigned long long, const int<4>);
+  const vull __builtin_altivec_vreplace_un_uv2di (vull, unsigned long long, \
+                                                  const int<4>);
     VREPLACE_UN_UV2DI vreplace_un_v2di {}
 
-  const vui __builtin_altivec_vreplace_un_uv4si (vui, unsigned int, const int<4>);
+  const vui __builtin_altivec_vreplace_un_uv4si (vui, unsigned int, \
+                                                 const int<4>);
     VREPLACE_UN_UV4SI vreplace_un_v4si {}
 
   const vd __builtin_altivec_vreplace_un_v2df (vd, double, const int<4>);
     VREPLACE_UN_V2DF vreplace_un_v2df {}
 
-  const vsll __builtin_altivec_vreplace_un_v2di (vsll, signed long long, const int<4>);
+  const vsll __builtin_altivec_vreplace_un_v2di (vsll, signed long long, \
+                                                 const int<4>);
     VREPLACE_UN_V2DI vreplace_un_v2di {}
 
   const vf __builtin_altivec_vreplace_un_v4sf (vf, float, const int<4>);
@@ -3376,7 +3406,8 @@
   const vsi __builtin_altivec_vreplace_un_v4si (vsi, signed int, const int<4>);
     VREPLACE_UN_V4SI vreplace_un_v4si {}
 
-  const vull __builtin_altivec_vreplace_uv2di (vull, unsigned long long, const int<1>);
+  const vull __builtin_altivec_vreplace_uv2di (vull, unsigned long long, \
+                                               const int<1>);
     VREPLACE_ELT_UV2DI vreplace_elt_v2di {}
 
   const vui __builtin_altivec_vreplace_uv4si (vui, unsigned int, const int<2>);
@@ -3385,7 +3416,8 @@
   const vd __builtin_altivec_vreplace_v2df (vd, double, const int<1>);
     VREPLACE_ELT_V2DF vreplace_elt_v2df {}
 
-  const vsll __builtin_altivec_vreplace_v2di (vsll, signed long long, const int<1>);
+  const vsll __builtin_altivec_vreplace_v2di (vsll, signed long long, \
+                                              const int<1>);
     VREPLACE_ELT_V2DI vreplace_elt_v2di {}
 
   const vf __builtin_altivec_vreplace_v4sf (vf, float, const int<2>);
@@ -3553,33 +3585,42 @@
   const vss __builtin_vsx_xxpermx_v8hi (vss, vss, vuc, const int<3>);
     XXPERMX_V8HI xxpermx {}
 
-  pure unsigned __int128 __builtin_altivec_ze_lxvrbx (signed long, const unsigned char *);
+  pure unsigned __int128 __builtin_altivec_ze_lxvrbx (signed long, \
+                                                      const unsigned char *);
     ZE_LXVRBX vsx_lxvrbx {lxvrze}
 
-  pure unsigned __int128 __builtin_altivec_ze_lxvrhx (signed long, const unsigned short *);
+  pure unsigned __int128 __builtin_altivec_ze_lxvrhx (signed long, \
+                                                      const unsigned short *);
     ZE_LXVRHX vsx_lxvrhx {lxvrze}
 
-  pure unsigned __int128 __builtin_altivec_ze_lxvrwx (signed long, const unsigned int *);
+  pure unsigned __int128 __builtin_altivec_ze_lxvrwx (signed long, \
+                                                      const unsigned int *);
     ZE_LXVRWX vsx_lxvrwx {lxvrze}
 
-  pure unsigned __int128 __builtin_altivec_ze_lxvrdx (signed long, const unsigned long long *);
+  pure unsigned __int128 \
+      __builtin_altivec_ze_lxvrdx (signed long, const unsigned long long *);
     ZE_LXVRDX vsx_lxvrdx {lxvrze}
 
 
 [power10-64]
-  const unsigned long long __builtin_cfuged (unsigned long long, unsigned long long);
+  const unsigned long long __builtin_cfuged (unsigned long long, \
+                                             unsigned long long);
     CFUGED cfuged {}
 
-  const unsigned long long __builtin_cntlzdm (unsigned long long, unsigned long long);
+  const unsigned long long __builtin_cntlzdm (unsigned long long, \
+                                              unsigned long long);
     CNTLZDM cntlzdm {}
 
-  const unsigned long long __builtin_cnttzdm (unsigned long long, unsigned long long);
+  const unsigned long long __builtin_cnttzdm (unsigned long long, \
+                                              unsigned long long);
     CNTTZDM cnttzdm {}
 
-  const unsigned long long __builtin_pdepd (unsigned long long, unsigned long long);
+  const unsigned long long __builtin_pdepd (unsigned long long, \
+                                            unsigned long long);
     PDEPD pdepd {}
 
-  const unsigned long long __builtin_pextd (unsigned long long, unsigned long long);
+  const unsigned long long __builtin_pextd (unsigned long long, \
+                                            unsigned long long);
     PEXTD pextd {}
 
 
@@ -3614,178 +3655,235 @@
   vuc __builtin_mma_disassemble_pair_internal (v256, const int<2>);
     DISASSEMBLE_PAIR_INTERNAL vsx_disassemble_pair {mma}
 
-  void __builtin_mma_pmxvbf16ger2 (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvbf16ger2 (v512 *, vuc, vuc, const int<4>, \
+                                   const int<4>, const int<2>);
     PMXVBF16GER2 nothing {mma,mmaint}
 
-  v512 __builtin_mma_pmxvbf16ger2_internal (vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvbf16ger2_internal (vuc, vuc, const int<4>, \
+                                            const int<4>, const int<2>);
     PMXVBF16GER2_INTERNAL mma_pmxvbf16ger2 {mma}
 
-  void __builtin_mma_pmxvbf16ger2nn (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvbf16ger2nn (v512 *, vuc, vuc, const int<4>, \
+                                     const int<4>, const int<2>);
     PMXVBF16GER2NN nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvbf16ger2nn_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvbf16ger2nn_internal (v512, vuc, vuc, const int<4>, \
+                                              const int<4>, const int<2>);
     PMXVBF16GER2NN_INTERNAL mma_pmxvbf16ger2nn {mma,quad}
 
-  void __builtin_mma_pmxvbf16ger2np (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvbf16ger2np (v512 *, vuc, vuc, const int<4>, \
+                                     const int<4>, const int<2>);
     PMXVBF16GER2NP nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvbf16ger2np_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvbf16ger2np_internal (v512, vuc, vuc, const int<4>, \
+                                              const int<4>, const int<2>);
     PMXVBF16GER2NP_INTERNAL mma_pmxvbf16ger2np {mma,quad}
 
-  void __builtin_mma_pmxvbf16ger2pn (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvbf16ger2pn (v512 *, vuc, vuc, const int<4>, \
+                                     const int<4>, const int<2>);
     PMXVBF16GER2PN nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvbf16ger2pn_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvbf16ger2pn_internal (v512, vuc, vuc, const int<4>, \
+                                              const int<4>, const int<2>);
     PMXVBF16GER2PN_INTERNAL mma_pmxvbf16ger2pn {mma,quad}
 
-  void __builtin_mma_pmxvbf16ger2pp (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvbf16ger2pp (v512 *, vuc, vuc, const int<4>, \
+                                     const int<4>, const int<2>);
     PMXVBF16GER2PP nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvbf16ger2pp_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvbf16ger2pp_internal (v512, vuc, vuc, const int<4>, \
+                                              const int<4>, const int<2>);
     PMXVBF16GER2PP_INTERNAL mma_pmxvbf16ger2pp {mma,quad}
 
-  void __builtin_mma_pmxvf16ger2 (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvf16ger2 (v512 *, vuc, vuc, const int<4>, \
+                                  const int<4>, const int<2>);
     PMXVF16GER2 nothing {mma,mmaint}
 
-  v512 __builtin_mma_pmxvf16ger2_internal (vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvf16ger2_internal (vuc, vuc, const int<4>, \
+                                           const int<4>, const int<2>);
     PMXVF16GER2_INTERNAL mma_pmxvf16ger2 {mma}
 
-  void __builtin_mma_pmxvf16ger2nn (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvf16ger2nn (v512 *, vuc, vuc, const int<4>, \
+                                    const int<4>, const int<2>);
     PMXVF16GER2NN nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf16ger2nn_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvf16ger2nn_internal (v512, vuc, vuc, const int<4>, \
+                                             const int<4>, const int<2>);
     PMXVF16GER2NN_INTERNAL mma_pmxvf16ger2nn {mma,quad}
 
-  void __builtin_mma_pmxvf16ger2np (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvf16ger2np (v512 *, vuc, vuc, const int<4>, \
+                                    const int<4>, const int<2>);
     PMXVF16GER2NP nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf16ger2np_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvf16ger2np_internal (v512, vuc, vuc, const int<4>, \
+                                             const int<4>, const int<2>);
     PMXVF16GER2NP_INTERNAL mma_pmxvf16ger2np {mma,quad}
 
-  void __builtin_mma_pmxvf16ger2pn (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvf16ger2pn (v512 *, vuc, vuc, const int<4>, \
+                                    const int<4>, const int<2>);
     PMXVF16GER2PN nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf16ger2pn_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvf16ger2pn_internal (v512, vuc, vuc, const int<4>, \
+                                             const int<4>, const int<2>);
     PMXVF16GER2PN_INTERNAL mma_pmxvf16ger2pn {mma,quad}
 
-  void __builtin_mma_pmxvf16ger2pp (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvf16ger2pp (v512 *, vuc, vuc, const int<4>, \
+                                    const int<4>, const int<2>);
     PMXVF16GER2PP nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf16ger2pp_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvf16ger2pp_internal (v512, vuc, vuc, const int<4>, \
+                                             const int<4>, const int<2>);
     PMXVF16GER2PP_INTERNAL mma_pmxvf16ger2pp {mma,quad}
 
   void __builtin_mma_pmxvf32ger (v512 *, vuc, vuc, const int<4>, const int<4>);
     PMXVF32GER nothing {mma,mmaint}
 
-  v512 __builtin_mma_pmxvf32ger_internal (vuc, vuc, const int<4>, const int<4>);
+  v512 __builtin_mma_pmxvf32ger_internal (vuc, vuc, const int<4>, \
+                                          const int<4>);
     PMXVF32GER_INTERNAL mma_pmxvf32ger {mma}
 
-  void __builtin_mma_pmxvf32gernn (v512 *, vuc, vuc, const int<4>, const int<4>);
+  void __builtin_mma_pmxvf32gernn (v512 *, vuc, vuc, const int<4>, \
+                                   const int<4>);
     PMXVF32GERNN nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf32gernn_internal (v512, vuc, vuc, const int<4>, const int<4>);
+  v512 __builtin_mma_pmxvf32gernn_internal (v512, vuc, vuc, const int<4>, \
+                                            const int<4>);
     PMXVF32GERNN_INTERNAL mma_pmxvf32gernn {mma,quad}
 
-  void __builtin_mma_pmxvf32gernp (v512 *, vuc, vuc, const int<4>, const int<4>);
+  void __builtin_mma_pmxvf32gernp (v512 *, vuc, vuc, const int<4>, \
+                                   const int<4>);
     PMXVF32GERNP nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf32gernp_internal (v512, vuc, vuc, const int<4>, const int<4>);
+  v512 __builtin_mma_pmxvf32gernp_internal (v512, vuc, vuc, const int<4>, \
+                                            const int<4>);
     PMXVF32GERNP_INTERNAL mma_pmxvf32gernp {mma,quad}
 
-  void __builtin_mma_pmxvf32gerpn (v512 *, vuc, vuc, const int<4>, const int<4>);
+  void __builtin_mma_pmxvf32gerpn (v512 *, vuc, vuc, const int<4>, \
+                                   const int<4>);
     PMXVF32GERPN nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf32gerpn_internal (v512, vuc, vuc, const int<4>, const int<4>);
+  v512 __builtin_mma_pmxvf32gerpn_internal (v512, vuc, vuc, const int<4>, \
+                                            const int<4>);
     PMXVF32GERPN_INTERNAL mma_pmxvf32gerpn {mma,quad}
 
-  void __builtin_mma_pmxvf32gerpp (v512 *, vuc, vuc, const int<4>, const int<4>);
+  void __builtin_mma_pmxvf32gerpp (v512 *, vuc, vuc, const int<4>, \
+                                   const int<4>);
     PMXVF32GERPP nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf32gerpp_internal (v512, vuc, vuc, const int<4>, const int<4>);
+  v512 __builtin_mma_pmxvf32gerpp_internal (v512, vuc, vuc, const int<4>, \
+                                            const int<4>);
     PMXVF32GERPP_INTERNAL mma_pmxvf32gerpp {mma,quad}
 
-  void __builtin_mma_pmxvf64ger (v512 *, v256, vuc, const int<4>, const int<2>);
+  void __builtin_mma_pmxvf64ger (v512 *, v256, vuc, const int<4>, \
+                                 const int<2>);
     PMXVF64GER nothing {mma,pair,mmaint}
 
-  v512 __builtin_mma_pmxvf64ger_internal (v256, vuc, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvf64ger_internal (v256, vuc, const int<4>, \
+                                          const int<2>);
     PMXVF64GER_INTERNAL mma_pmxvf64ger {mma,pair}
 
-  void __builtin_mma_pmxvf64gernn (v512 *, v256, vuc, const int<4>, const int<2>);
+  void __builtin_mma_pmxvf64gernn (v512 *, v256, vuc, const int<4>, \
+                                   const int<2>);
     PMXVF64GERNN nothing {mma,pair,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf64gernn_internal (v512, v256, vuc, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvf64gernn_internal (v512, v256, vuc, const int<4>, \
+                                            const int<2>);
     PMXVF64GERNN_INTERNAL mma_pmxvf64gernn {mma,pair,quad}
 
-  void __builtin_mma_pmxvf64gernp (v512 *, v256, vuc, const int<4>, const int<2>);
+  void __builtin_mma_pmxvf64gernp (v512 *, v256, vuc, const int<4>, \
+                                   const int<2>);
     PMXVF64GERNP nothing {mma,pair,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf64gernp_internal (v512, v256, vuc, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvf64gernp_internal (v512, v256, vuc, const int<4>, \
+                                            const int<2>);
     PMXVF64GERNP_INTERNAL mma_pmxvf64gernp {mma,pair,quad}
 
-  void __builtin_mma_pmxvf64gerpn (v512 *, v256, vuc, const int<4>, const int<2>);
+  void __builtin_mma_pmxvf64gerpn (v512 *, v256, vuc, const int<4>, \
+                                   const int<2>);
     PMXVF64GERPN nothing {mma,pair,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf64gerpn_internal (v512, v256, vuc, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvf64gerpn_internal (v512, v256, vuc, const int<4>, \
+                                            const int<2>);
     PMXVF64GERPN_INTERNAL mma_pmxvf64gerpn {mma,pair,quad}
 
-  void __builtin_mma_pmxvf64gerpp (v512 *, v256, vuc, const int<4>, const int<2>);
+  void __builtin_mma_pmxvf64gerpp (v512 *, v256, vuc, const int<4>, \
+                                   const int<2>);
     PMXVF64GERPP nothing {mma,pair,quad,mmaint}
 
-  v512 __builtin_mma_pmxvf64gerpp_internal (v512, v256, vuc, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvf64gerpp_internal (v512, v256, vuc, const int<4>, \
+                                            const int<2>);
     PMXVF64GERPP_INTERNAL mma_pmxvf64gerpp {mma,pair,quad}
 
-  void __builtin_mma_pmxvi16ger2 (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvi16ger2 (v512 *, vuc, vuc, const int<4>, \
+                                  const int<4>, const int<2>);
     PMXVI16GER2 nothing {mma,mmaint}
 
-  v512 __builtin_mma_pmxvi16ger2_internal (vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvi16ger2_internal (vuc, vuc, const int<4>, \
+                                           const int<4>, const int<2>);
     PMXVI16GER2_INTERNAL mma_pmxvi16ger2 {mma}
 
-  void __builtin_mma_pmxvi16ger2pp (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvi16ger2pp (v512 *, vuc, vuc, const int<4>, \
+                                    const int<4>, const int<2>);
     PMXVI16GER2PP nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvi16ger2pp_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvi16ger2pp_internal (v512, vuc, vuc, const int<4>, \
+                                             const int<4>, const int<2>);
     PMXVI16GER2PP_INTERNAL mma_pmxvi16ger2pp {mma,quad}
 
-  void __builtin_mma_pmxvi16ger2s (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvi16ger2s (v512 *, vuc, vuc, const int<4>, \
+                                   const int<4>, const int<2>);
     PMXVI16GER2S nothing {mma,mmaint}
 
-  v512 __builtin_mma_pmxvi16ger2s_internal (vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvi16ger2s_internal (vuc, vuc, const int<4>, \
+                                            const int<4>, const int<2>);
     PMXVI16GER2S_INTERNAL mma_pmxvi16ger2s {mma}
 
-  void __builtin_mma_pmxvi16ger2spp (v512 *, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  void __builtin_mma_pmxvi16ger2spp (v512 *, vuc, vuc, const int<4>, \
+                                     const int<4>, const int<2>);
     PMXVI16GER2SPP nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvi16ger2spp_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<2>);
+  v512 __builtin_mma_pmxvi16ger2spp_internal (v512, vuc, vuc, const int<4>, \
+                                              const int<4>, const int<2>);
     PMXVI16GER2SPP_INTERNAL mma_pmxvi16ger2spp {mma,quad}
 
-  void __builtin_mma_pmxvi4ger8 (v512 *, vuc, vuc, const int<4>, const int<4>, const int<8>);
+  void __builtin_mma_pmxvi4ger8 (v512 *, vuc, vuc, const int<4>, \
+                                 const int<4>, const int<8>);
     PMXVI4GER8 nothing {mma,mmaint}
 
-  v512 __builtin_mma_pmxvi4ger8_internal (vuc, vuc, const int<4>, const int<4>, const int<8>);
+  v512 __builtin_mma_pmxvi4ger8_internal (vuc, vuc, const int<4>, \
+                                          const int<4>, const int<8>);
     PMXVI4GER8_INTERNAL mma_pmxvi4ger8 {mma}
 
-  void __builtin_mma_pmxvi4ger8pp (v512 *, vuc, vuc, const int<4>, const int<4>, const int<4>);
+  void __builtin_mma_pmxvi4ger8pp (v512 *, vuc, vuc, const int<4>, \
+                                   const int<4>, const int<4>);
     PMXVI4GER8PP nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvi4ger8pp_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<4>);
+  v512 __builtin_mma_pmxvi4ger8pp_internal (v512, vuc, vuc, const int<4>, \
+                                            const int<4>, const int<4>);
     PMXVI4GER8PP_INTERNAL mma_pmxvi4ger8pp {mma,quad}
 
-  void __builtin_mma_pmxvi8ger4 (v512 *, vuc, vuc, const int<4>, const int<4>, const int<4>);
+  void __builtin_mma_pmxvi8ger4 (v512 *, vuc, vuc, const int<4>, \
+                                 const int<4>, const int<4>);
     PMXVI8GER4 nothing {mma,mmaint}
 
-  v512 __builtin_mma_pmxvi8ger4_internal (vuc, vuc, const int<4>, const int<4>, const int<4>);
+  v512 __builtin_mma_pmxvi8ger4_internal (vuc, vuc, const int<4>, \
+                                          const int<4>, const int<4>);
     PMXVI8GER4_INTERNAL mma_pmxvi8ger4 {mma}
 
-  void __builtin_mma_pmxvi8ger4pp (v512 *, vuc, vuc, const int<4>, const int<4>, const int<4>);
+  void __builtin_mma_pmxvi8ger4pp (v512 *, vuc, vuc, const int<4>, \
+                                   const int<4>, const int<4>);
     PMXVI8GER4PP nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvi8ger4pp_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<4>);
+  v512 __builtin_mma_pmxvi8ger4pp_internal (v512, vuc, vuc, const int<4>, \
+                                            const int<4>, const int<4>);
     PMXVI8GER4PP_INTERNAL mma_pmxvi8ger4pp {mma,quad}
 
-  void __builtin_mma_pmxvi8ger4spp (v512 *, vuc, vuc, const int<4>, const int<4>, const int<4>);
+  void __builtin_mma_pmxvi8ger4spp (v512 *, vuc, vuc, const int<4>, \
+                                    const int<4>, const int<4>);
     PMXVI8GER4SPP nothing {mma,quad,mmaint}
 
-  v512 __builtin_mma_pmxvi8ger4spp_internal (v512, vuc, vuc, const int<4>, const int<4>, const int<4>);
+  v512 __builtin_mma_pmxvi8ger4spp_internal (v512, vuc, vuc, const int<4>, \
+                                             const int<4>, const int<4>);
     PMXVI8GER4SPP_INTERNAL mma_pmxvi8ger4spp {mma,quad}
 
   void __builtin_mma_xvbf16ger2 (v512 *, vuc, vuc);
diff --git a/gcc/config/rs6000/rs6000-call.c b/gcc/config/rs6000/rs6000-call.c
index 6d8727c..83e1abb 100644
--- a/gcc/config/rs6000/rs6000-call.c
+++ b/gcc/config/rs6000/rs6000-call.c
@@ -190,6 +190,7 @@ static tree builtin_function_type (machine_mode, machine_mode,
 static void rs6000_common_init_builtins (void);
 static void htm_init_builtins (void);
 static void mma_init_builtins (void);
+static rtx rs6000_expand_new_builtin (tree, rtx, rtx, machine_mode, int);
 static bool rs6000_gimple_fold_new_builtin (gimple_stmt_iterator *gsi);
 
 
@@ -8929,7 +8930,105 @@ rs6000_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
   return build_va_arg_indirect_ref (addr);
 }
 
-/* Builtins.  */
+/* Debug utility to translate a type node to a single textual token.  */
+static
+const char *rs6000_type_string (tree type_node)
+{
+  if (type_node == void_type_node)
+    return "void";
+  else if (type_node == long_integer_type_node)
+    return "long";
+  else if (type_node == long_unsigned_type_node)
+    return "ulong";
+  else if (type_node == long_long_integer_type_node)
+    return "longlong";
+  else if (type_node == long_long_unsigned_type_node)
+    return "ulonglong";
+  else if (type_node == bool_V2DI_type_node)
+    return "vbll";
+  else if (type_node == bool_V4SI_type_node)
+    return "vbi";
+  else if (type_node == bool_V8HI_type_node)
+    return "vbs";
+  else if (type_node == bool_V16QI_type_node)
+    return "vbc";
+  else if (type_node == bool_int_type_node)
+    return "bool";
+  else if (type_node == dfloat64_type_node)
+    return "_Decimal64";
+  else if (type_node == double_type_node)
+    return "double";
+  else if (type_node == intDI_type_node)
+    return "sll";
+  else if (type_node == intHI_type_node)
+    return "ss";
+  else if (type_node == ibm128_float_type_node)
+    return "__ibm128";
+  else if (type_node == opaque_V4SI_type_node)
+    return "opaque";
+  else if (POINTER_TYPE_P (type_node))
+    return "void*";
+  else if (type_node == intQI_type_node || type_node == char_type_node)
+    return "sc";
+  else if (type_node == dfloat32_type_node)
+    return "_Decimal32";
+  else if (type_node == float_type_node)
+    return "float";
+  else if (type_node == intSI_type_node || type_node == integer_type_node)
+    return "si";
+  else if (type_node == dfloat128_type_node)
+    return "_Decimal128";
+  else if (type_node == long_double_type_node)
+    return "longdouble";
+  else if (type_node == intTI_type_node)
+    return "sq";
+  else if (type_node == unsigned_intDI_type_node)
+    return "ull";
+  else if (type_node == unsigned_intHI_type_node)
+    return "us";
+  else if (type_node == unsigned_intQI_type_node)
+    return "uc";
+  else if (type_node == unsigned_intSI_type_node)
+    return "ui";
+  else if (type_node == unsigned_intTI_type_node)
+    return "uq";
+  else if (type_node == unsigned_V1TI_type_node)
+    return "vuq";
+  else if (type_node == unsigned_V2DI_type_node)
+    return "vull";
+  else if (type_node == unsigned_V4SI_type_node)
+    return "vui";
+  else if (type_node == unsigned_V8HI_type_node)
+    return "vus";
+  else if (type_node == unsigned_V16QI_type_node)
+    return "vuc";
+  else if (type_node == V16QI_type_node)
+    return "vsc";
+  else if (type_node == V1TI_type_node)
+    return "vsq";
+  else if (type_node == V2DF_type_node)
+    return "vd";
+  else if (type_node == V2DI_type_node)
+    return "vsll";
+  else if (type_node == V4SF_type_node)
+    return "vf";
+  else if (type_node == V4SI_type_node)
+    return "vsi";
+  else if (type_node == V8HI_type_node)
+    return "vss";
+  else if (type_node == pixel_V8HI_type_node)
+    return "vp";
+  else if (type_node == pcvoid_type_node)
+    return "voidc*";
+  else if (type_node == float128_type_node)
+    return "_Float128";
+  else if (type_node == vector_pair_type_node)
+    return "__vector_pair";
+  else if (type_node == vector_quad_type_node)
+    return "__vector_quad";
+
+  return "unknown";
+}
 
 static void
 def_builtin (const char *name, tree type, enum rs6000_builtins code)
@@ -8959,7 +9058,7 @@ def_builtin (const char *name, tree type, enum rs6000_builtins code)
       /* const function, function only depends on the inputs.  */
       TREE_READONLY (t) = 1;
       TREE_NOTHROW (t) = 1;
-      attr_string = ", const";
+      attr_string = "= const";
     }
   else if ((classify & RS6000_BTC_PURE) != 0)
     {
@@ -8967,7 +9066,7 @@ def_builtin (const char *name, tree type, enum rs6000_builtins code)
 	 external state.  */
       DECL_PURE_P (t) = 1;
       TREE_NOTHROW (t) = 1;
-      attr_string = ", pure";
+      attr_string = "= pure";
     }
   else if ((classify & RS6000_BTC_FP) != 0)
     {
@@ -8981,12 +9080,12 @@ def_builtin (const char *name, tree type, enum rs6000_builtins code)
 	{
 	  DECL_PURE_P (t) = 1;
 	  DECL_IS_NOVOPS (t) = 1;
-	  attr_string = ", fp, pure";
+	  attr_string = "= fp, pure";
 	}
       else
 	{
 	  TREE_READONLY (t) = 1;
-	  attr_string = ", fp, const";
+	  attr_string = "= fp, const";
 	}
     }
   else if ((classify & (RS6000_BTC_QUAD | RS6000_BTC_PAIR)) != 0)
@@ -8996,8 +9095,22 @@ def_builtin (const char *name, tree type, enum rs6000_builtins code)
     gcc_unreachable ();
 
   if (TARGET_DEBUG_BUILTIN)
-    fprintf (stderr, "rs6000_builtin, code = %4d, %s%s\n",
-	     (int)code, name, attr_string);
+    {
+      tree t = TREE_TYPE (type);
+      fprintf (stderr, "%s %s (", rs6000_type_string (t), name);
+      t = TYPE_ARG_TYPES (type);
+      gcc_assert (t);
+
+      while (TREE_VALUE (t) != void_type_node)
+	{
+	  fprintf (stderr, "%s", rs6000_type_string (TREE_VALUE (t)));
+	  t = TREE_CHAIN (t);
+	  gcc_assert (t);
+	  if (TREE_VALUE (t) != void_type_node)
+	    fprintf (stderr, ", ");
+	}
+      fprintf (stderr, "); %s [%4d]\n", attr_string, (int) code);
+    }
 }
 
 static const struct builtin_compatibility bdesc_compat[] =
@@ -11173,6 +11286,45 @@ altivec_expand_vec_ext_builtin (tree exp, rtx target)
   return target;
 }
 
+/* Expand vec_sel builtin.  */
+static rtx
+altivec_expand_vec_sel_builtin (enum insn_code icode, tree exp, rtx target)
+{
+  rtx op0, op1, op2, pat;
+  tree arg0, arg1, arg2;
+
+  arg0 = CALL_EXPR_ARG (exp, 0);
+  op0 = expand_normal (arg0);
+  arg1 = CALL_EXPR_ARG (exp, 1);
+  op1 = expand_normal (arg1);
+  arg2 = CALL_EXPR_ARG (exp, 2);
+  op2 = expand_normal (arg2);
+
+  machine_mode tmode = insn_data[icode].operand[0].mode;
+  machine_mode mode0 = insn_data[icode].operand[1].mode;
+  machine_mode mode1 = insn_data[icode].operand[2].mode;
+  machine_mode mode2 = insn_data[icode].operand[3].mode;
+
+  if (target == 0 || GET_MODE (target) != tmode
+      || !(*insn_data[icode].operand[0].predicate) (target, tmode))
+    target = gen_reg_rtx (tmode);
+
+  if (!(*insn_data[icode].operand[1].predicate) (op0, mode0))
+    op0 = copy_to_mode_reg (mode0, op0);
+  if (!(*insn_data[icode].operand[2].predicate) (op1, mode1))
+    op1 = copy_to_mode_reg (mode1, op1);
+  if (!(*insn_data[icode].operand[3].predicate) (op2, mode2))
+    op2 = copy_to_mode_reg (mode2, op2);
+
+  pat = GEN_FCN (icode) (target, op0, op1, op2, op2);
+  if (pat)
+    emit_insn (pat);
+  else
+    return NULL_RTX;
+
+  return target;
+}
+
 /* Expand the builtin in EXP and store the result in TARGET.  Store
    true in *EXPANDEDP if we found a builtin to expand.  */
 static rtx
@@ -11358,6 +11510,29 @@ altivec_expand_builtin (tree exp, rtx target, bool *expandedp)
 	emit_insn (pat);
       return NULL_RTX;
 
+    case ALTIVEC_BUILTIN_VSEL_2DF:
+      return altivec_expand_vec_sel_builtin (CODE_FOR_altivec_vselv2df, exp,
+					     target);
+    case ALTIVEC_BUILTIN_VSEL_2DI:
+    case ALTIVEC_BUILTIN_VSEL_2DI_UNS:
+      return altivec_expand_vec_sel_builtin (CODE_FOR_altivec_vselv2di, exp,
+					     target);
+    case ALTIVEC_BUILTIN_VSEL_4SF:
+      return altivec_expand_vec_sel_builtin (CODE_FOR_altivec_vselv4sf, exp,
+					     target);
+    case ALTIVEC_BUILTIN_VSEL_4SI:
+    case ALTIVEC_BUILTIN_VSEL_4SI_UNS:
+      return altivec_expand_vec_sel_builtin (CODE_FOR_altivec_vselv4si, exp,
+					     target);
+    case ALTIVEC_BUILTIN_VSEL_8HI:
+    case ALTIVEC_BUILTIN_VSEL_8HI_UNS:
+      return altivec_expand_vec_sel_builtin (CODE_FOR_altivec_vselv8hi, exp,
+					     target);
+    case ALTIVEC_BUILTIN_VSEL_16QI:
+    case ALTIVEC_BUILTIN_VSEL_16QI_UNS:
+      return altivec_expand_vec_sel_builtin (CODE_FOR_altivec_vselv16qi, exp,
+					     target);
+
     case ALTIVEC_BUILTIN_DSSALL:
       emit_insn (gen_altivec_dssall ());
       return NULL_RTX;
@@ -11727,6 +11902,83 @@ rs6000_invalid_builtin (enum rs6000_builtins fncode)
     error ("%qs is not supported with the current options", name);
 }
 
+/* Raise an error message for a builtin function that is called without the
+   appropriate target options being set.  */
+
+static void
+rs6000_invalid_new_builtin (enum rs6000_gen_builtins fncode)
+{
+  size_t j = (size_t) fncode;
+  const char *name = rs6000_builtin_info_x[j].bifname;
+
+  switch (rs6000_builtin_info_x[j].enable)
+    {
+    case ENB_P5:
+      error ("%qs requires the %qs option", name, "-mcpu=power5");
+      break;
+    case ENB_P6:
+      error ("%qs requires the %qs option", name, "-mcpu=power6");
+      break;
+    case ENB_ALTIVEC:
+      error ("%qs requires the %qs option", name, "-maltivec");
+      break;
+    case ENB_CELL:
+      error ("%qs requires the %qs option", name, "-mcpu=cell");
+      break;
+    case ENB_VSX:
+      error ("%qs requires the %qs option", name, "-mvsx");
+      break;
+    case ENB_P7:
+      error ("%qs requires the %qs option", name, "-mcpu=power7");
+      break;
+    case ENB_P7_64:
+      error ("%qs requires the %qs option and either the %qs or %qs option",
+	     name, "-mcpu=power7", "-m64", "-mpowerpc64");
+      break;
+    case ENB_P8:
+      error ("%qs requires the %qs option", name, "-mcpu=power8");
+      break;
+    case ENB_P8V:
+      error ("%qs requires the %qs option", name, "-mpower8-vector");
+      break;
+    case ENB_P9:
+      error ("%qs requires the %qs option", name, "-mcpu=power9");
+      break;
+    case ENB_P9_64:
+      error ("%qs requires the %qs option and either the %qs or %qs option",
+	     name, "-mcpu=power9", "-m64", "-mpowerpc64");
+      break;
+    case ENB_P9V:
+      error ("%qs requires the %qs option", name, "-mpower9-vector");
+      break;
+    case ENB_IEEE128_HW:
+      error ("%qs requires ISA 3.0 IEEE 128-bit floating point", name);
+      break;
+    case ENB_DFP:
+      error ("%qs requires the %qs option", name, "-mhard-dfp");
+      break;
+    case ENB_CRYPTO:
+      error ("%qs requires the %qs option", name, "-mcrypto");
+      break;
+    case ENB_HTM:
+      error ("%qs requires the %qs option", name, "-mhtm");
+      break;
+    case ENB_P10:
+      error ("%qs requires the %qs option", name, "-mcpu=power10");
+      break;
+    case ENB_P10_64:
+      error ("%qs requires the %qs option and either the %qs or %qs option",
+	     name, "-mcpu=power10", "-m64", "-mpowerpc64");
+      break;
+    case ENB_MMA:
+      error ("%qs requires the %qs option", name, "-mmma");
+      break;
+    default:
+    case ENB_ALWAYS:
+      gcc_unreachable ();
+    }
+}
+
 /* Target hook for early folding of built-ins, shamelessly stolen
    from ia64.c.  */
 
@@ -12396,6 +12648,14 @@ rs6000_gimple_fold_builtin (gimple_stmt_iterator *gsi)
       return true;
     /* flavors of vec_min.  */
     case VSX_BUILTIN_XVMINDP:
+    case ALTIVEC_BUILTIN_VMINFP:
+      {
+	lhs = gimple_call_lhs (stmt);
+	tree type = TREE_TYPE (lhs);
+	if (HONOR_NANS (type))
+	  return false;
+	gcc_fallthrough ();
+      }
     case P8V_BUILTIN_VMINSD:
     case P8V_BUILTIN_VMINUD:
     case ALTIVEC_BUILTIN_VMINSB:
@@ -12404,7 +12664,6 @@ rs6000_gimple_fold_builtin (gimple_stmt_iterator *gsi)
     case ALTIVEC_BUILTIN_VMINUB:
     case ALTIVEC_BUILTIN_VMINUH:
     case ALTIVEC_BUILTIN_VMINUW:
-    case ALTIVEC_BUILTIN_VMINFP:
       arg0 = gimple_call_arg (stmt, 0);
       arg1 = gimple_call_arg (stmt, 1);
       lhs = gimple_call_lhs (stmt);
@@ -12414,6 +12673,14 @@ rs6000_gimple_fold_builtin (gimple_stmt_iterator *gsi)
       return true;
     /* flavors of vec_max.  */
     case VSX_BUILTIN_XVMAXDP:
+    case ALTIVEC_BUILTIN_VMAXFP:
+      {
+	lhs = gimple_call_lhs (stmt);
+	tree type = TREE_TYPE (lhs);
+	if (HONOR_NANS (type))
+	  return false;
+	gcc_fallthrough ();
+      }
     case P8V_BUILTIN_VMAXSD:
     case P8V_BUILTIN_VMAXUD:
     case ALTIVEC_BUILTIN_VMAXSB:
@@ -12422,7 +12689,6 @@ rs6000_gimple_fold_builtin (gimple_stmt_iterator *gsi)
     case ALTIVEC_BUILTIN_VMAXUB:
     case ALTIVEC_BUILTIN_VMAXUH:
     case ALTIVEC_BUILTIN_VMAXUW:
-    case ALTIVEC_BUILTIN_VMAXFP:
       arg0 = gimple_call_arg (stmt, 0);
       arg1 = gimple_call_arg (stmt, 1);
       lhs = gimple_call_lhs (stmt);
@@ -13636,6 +13902,14 @@ rs6000_gimple_fold_new_builtin (gimple_stmt_iterator *gsi)
     /* flavors of vec_min.  */
     case RS6000_BIF_XVMINDP:
     case RS6000_BIF_XVMINSP:
+    case RS6000_BIF_VMINFP:
+      {
+	lhs = gimple_call_lhs (stmt);
+	tree type = TREE_TYPE (lhs);
+	if (HONOR_NANS (type))
+	  return false;
+	gcc_fallthrough ();
+      }
     case RS6000_BIF_VMINSD:
     case RS6000_BIF_VMINUD:
     case RS6000_BIF_VMINSB:
@@ -13644,7 +13918,6 @@ rs6000_gimple_fold_new_builtin (gimple_stmt_iterator *gsi)
     case RS6000_BIF_VMINUB:
     case RS6000_BIF_VMINUH:
     case RS6000_BIF_VMINUW:
-    case RS6000_BIF_VMINFP:
       arg0 = gimple_call_arg (stmt, 0);
       arg1 = gimple_call_arg (stmt, 1);
       lhs = gimple_call_lhs (stmt);
@@ -13655,6 +13928,14 @@ rs6000_gimple_fold_new_builtin (gimple_stmt_iterator *gsi)
     /* flavors of vec_max.  */
     case RS6000_BIF_XVMAXDP:
     case RS6000_BIF_XVMAXSP:
+    case RS6000_BIF_VMAXFP:
+      {
+	lhs = gimple_call_lhs (stmt);
+	tree type = TREE_TYPE (lhs);
+	if (HONOR_NANS (type))
+	  return false;
+	gcc_fallthrough ();
+      }
     case RS6000_BIF_VMAXSD:
     case RS6000_BIF_VMAXUD:
     case RS6000_BIF_VMAXSB:
@@ -13663,7 +13944,6 @@ rs6000_gimple_fold_new_builtin (gimple_stmt_iterator *gsi)
     case RS6000_BIF_VMAXUB:
     case RS6000_BIF_VMAXUH:
     case RS6000_BIF_VMAXUW:
-    case RS6000_BIF_VMAXFP:
       arg0 = gimple_call_arg (stmt, 0);
       arg1 = gimple_call_arg (stmt, 1);
       lhs = gimple_call_lhs (stmt);
@@ -14300,6 +14580,9 @@ rs6000_expand_builtin (tree exp, rtx target, rtx subtarget ATTRIBUTE_UNUSED,
 		       machine_mode mode ATTRIBUTE_UNUSED,
 		       int ignore ATTRIBUTE_UNUSED)
 {
+  if (new_builtins_are_live)
+    return rs6000_expand_new_builtin (exp, target, subtarget, mode, ignore);
+
   tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
   enum rs6000_builtins fcode
     = (enum rs6000_builtins) DECL_MD_FUNCTION_CODE (fndecl);
@@ -14592,6 +14875,1149 @@ rs6000_expand_builtin (tree exp, rtx target, rtx subtarget ATTRIBUTE_UNUSED,
   gcc_unreachable ();
 }
 
+/* Expand ALTIVEC_BUILTIN_MASK_FOR_LOAD.  */
+rtx
+rs6000_expand_ldst_mask (rtx target, tree arg0)
+{
+  int icode2 = BYTES_BIG_ENDIAN ? (int) CODE_FOR_altivec_lvsr_direct
+				: (int) CODE_FOR_altivec_lvsl_direct;
+  machine_mode tmode = insn_data[icode2].operand[0].mode;
+  machine_mode mode = insn_data[icode2].operand[1].mode;
+
+  gcc_assert (TARGET_ALTIVEC);
+
+  gcc_assert (POINTER_TYPE_P (TREE_TYPE (arg0)));
+  rtx op = expand_expr (arg0, NULL_RTX, Pmode, EXPAND_NORMAL);
+  rtx addr = memory_address (mode, op);
+  /* We need to negate the address.  */
+  op = gen_reg_rtx (GET_MODE (addr));
+  emit_insn (gen_rtx_SET (op, gen_rtx_NEG (GET_MODE (addr), addr)));
+  op = gen_rtx_MEM (mode, op);
+
+  if (target == 0
+      || GET_MODE (target) != tmode
+      || !insn_data[icode2].operand[0].predicate (target, tmode))
+    target = gen_reg_rtx (tmode);
+
+  rtx pat = GEN_FCN (icode2) (target, op);
+  if (!pat)
+    return 0;
+  emit_insn (pat);
+
+  return target;
+}
+
+/* Expand the CPU builtin in FCODE and store the result in TARGET.  */
+static rtx
+new_cpu_expand_builtin (enum rs6000_gen_builtins fcode,
+			tree exp ATTRIBUTE_UNUSED, rtx target)
+{
+  /* __builtin_cpu_init () is a nop, so expand to nothing.  */
+  if (fcode == RS6000_BIF_CPU_INIT)
+    return const0_rtx;
+
+  if (target == 0 || GET_MODE (target) != SImode)
+    target = gen_reg_rtx (SImode);
+
+  /* TODO: Factor the #ifdef'd code into a separate function.  */
+#ifdef TARGET_LIBC_PROVIDES_HWCAP_IN_TCB
+  tree arg = TREE_OPERAND (CALL_EXPR_ARG (exp, 0), 0);
+  /* Target clones creates an ARRAY_REF instead of STRING_CST, convert it back
+     to a STRING_CST.  */
+  if (TREE_CODE (arg) == ARRAY_REF
+      && TREE_CODE (TREE_OPERAND (arg, 0)) == STRING_CST
+      && TREE_CODE (TREE_OPERAND (arg, 1)) == INTEGER_CST
+      && compare_tree_int (TREE_OPERAND (arg, 1), 0) == 0)
+    arg = TREE_OPERAND (arg, 0);
+
+  if (TREE_CODE (arg) != STRING_CST)
+    {
+      error ("builtin %qs only accepts a string argument",
+	     rs6000_builtin_info_x[(size_t) fcode].bifname);
+      return const0_rtx;
+    }
+
+  if (fcode == RS6000_BIF_CPU_IS)
+    {
+      const char *cpu = TREE_STRING_POINTER (arg);
+      rtx cpuid = NULL_RTX;
+      for (size_t i = 0; i < ARRAY_SIZE (cpu_is_info); i++)
+	if (strcmp (cpu, cpu_is_info[i].cpu) == 0)
+	  {
+	    /* The CPUID value in the TCB is offset by _DL_FIRST_PLATFORM.  */
+	    cpuid = GEN_INT (cpu_is_info[i].cpuid + _DL_FIRST_PLATFORM);
+	    break;
+	  }
+      if (cpuid == NULL_RTX)
+	{
+	  /* Invalid CPU argument.  */
+	  error ("cpu %qs is an invalid argument to builtin %qs",
+		 cpu, rs6000_builtin_info_x[(size_t) fcode].bifname);
+	  return const0_rtx;
+	}
+
+      rtx platform = gen_reg_rtx (SImode);
+      rtx address = gen_rtx_PLUS (Pmode,
+				  gen_rtx_REG (Pmode, TLS_REGNUM),
+				  GEN_INT (TCB_PLATFORM_OFFSET));
+      rtx tcbmem = gen_const_mem (SImode, address);
+      emit_move_insn (platform, tcbmem);
+      emit_insn (gen_eqsi3 (target, platform, cpuid));
+    }
+  else if (fcode == RS6000_BIF_CPU_SUPPORTS)
+    {
+      const char *hwcap = TREE_STRING_POINTER (arg);
+      rtx mask = NULL_RTX;
+      int hwcap_offset;
+      for (size_t i = 0; i < ARRAY_SIZE (cpu_supports_info); i++)
+	if (strcmp (hwcap, cpu_supports_info[i].hwcap) == 0)
+	  {
+	    mask = GEN_INT (cpu_supports_info[i].mask);
+	    hwcap_offset = TCB_HWCAP_OFFSET (cpu_supports_info[i].id);
+	    break;
+	  }
+      if (mask == NULL_RTX)
+	{
+	  /* Invalid HWCAP argument.  */
+	  error ("%s %qs is an invalid argument to builtin %qs",
+		 "hwcap", hwcap,
+		 rs6000_builtin_info_x[(size_t) fcode].bifname);
+	  return const0_rtx;
+	}
+
+      rtx tcb_hwcap = gen_reg_rtx (SImode);
+      rtx address = gen_rtx_PLUS (Pmode,
+				  gen_rtx_REG (Pmode, TLS_REGNUM),
+				  GEN_INT (hwcap_offset));
+      rtx tcbmem = gen_const_mem (SImode, address);
+      emit_move_insn (tcb_hwcap, tcbmem);
+      rtx scratch1 = gen_reg_rtx (SImode);
+      emit_insn (gen_rtx_SET (scratch1,
+			      gen_rtx_AND (SImode, tcb_hwcap, mask)));
+      rtx scratch2 = gen_reg_rtx (SImode);
+      emit_insn (gen_eqsi3 (scratch2, scratch1, const0_rtx));
+      emit_insn (gen_rtx_SET (target,
+			      gen_rtx_XOR (SImode, scratch2, const1_rtx)));
+    }
+  else
+    gcc_unreachable ();
+
+  /* Record that we have expanded a CPU builtin, so that we can later
+     emit a reference to the special symbol exported by LIBC to ensure we
+     do not link against an old LIBC that doesn't support this feature.  */
+  cpu_builtin_p = true;
+
+#else
+  warning (0, "builtin %qs needs GLIBC (2.23 and newer) that exports hardware "
+	   "capability bits", rs6000_builtin_info_x[(size_t) fcode].bifname);
+
+  /* For old LIBCs, always return FALSE.  */
+  emit_move_insn (target, GEN_INT (0));
+#endif /* TARGET_LIBC_PROVIDES_HWCAP_IN_TCB */
+
+  return target;
+}
+
+/* For the element-reversing load/store built-ins, produce the correct
+   insn_code depending on the target endianness.  */
+static insn_code
+elemrev_icode (rs6000_gen_builtins fcode)
+{
+  switch (fcode)
+    {
+    case RS6000_BIF_ST_ELEMREV_V1TI:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_store_v1ti
+			      : CODE_FOR_vsx_st_elemrev_v1ti;
+
+    case RS6000_BIF_ST_ELEMREV_V2DF:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_store_v2df
+			      : CODE_FOR_vsx_st_elemrev_v2df;
+
+    case RS6000_BIF_ST_ELEMREV_V2DI:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_store_v2di
+			      : CODE_FOR_vsx_st_elemrev_v2di;
+
+    case RS6000_BIF_ST_ELEMREV_V4SF:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_store_v4sf
+			      : CODE_FOR_vsx_st_elemrev_v4sf;
+
+    case RS6000_BIF_ST_ELEMREV_V4SI:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_store_v4si
+			      : CODE_FOR_vsx_st_elemrev_v4si;
+
+    case RS6000_BIF_ST_ELEMREV_V8HI:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_store_v8hi
+			      : CODE_FOR_vsx_st_elemrev_v8hi;
+
+    case RS6000_BIF_ST_ELEMREV_V16QI:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_store_v16qi
+			      : CODE_FOR_vsx_st_elemrev_v16qi;
+
+    case RS6000_BIF_LD_ELEMREV_V2DF:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_load_v2df
+			      : CODE_FOR_vsx_ld_elemrev_v2df;
+
+    case RS6000_BIF_LD_ELEMREV_V1TI:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_load_v1ti
+			      : CODE_FOR_vsx_ld_elemrev_v1ti;
+
+    case RS6000_BIF_LD_ELEMREV_V2DI:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_load_v2di
+			      : CODE_FOR_vsx_ld_elemrev_v2di;
+
+    case RS6000_BIF_LD_ELEMREV_V4SF:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_load_v4sf
+			      : CODE_FOR_vsx_ld_elemrev_v4sf;
+
+    case RS6000_BIF_LD_ELEMREV_V4SI:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_load_v4si
+			      : CODE_FOR_vsx_ld_elemrev_v4si;
+
+    case RS6000_BIF_LD_ELEMREV_V8HI:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_load_v8hi
+			      : CODE_FOR_vsx_ld_elemrev_v8hi;
+
+    case RS6000_BIF_LD_ELEMREV_V16QI:
+      return BYTES_BIG_ENDIAN ? CODE_FOR_vsx_load_v16qi
+			      : CODE_FOR_vsx_ld_elemrev_v16qi;
+    default:
+      ;
+    }
+
+  gcc_unreachable ();
+}
+
+/* Expand an AltiVec vector load builtin, and return the expanded rtx.  */
+static rtx
+ldv_expand_builtin (rtx target, insn_code icode, rtx *op, machine_mode tmode)
+{
+  if (target == 0
+      || GET_MODE (target) != tmode
+      || !insn_data[icode].operand[0].predicate (target, tmode))
+    target = gen_reg_rtx (tmode);
+
+  op[1] = copy_to_mode_reg (Pmode, op[1]);
+
+  /* These CELL built-ins use BLKmode instead of tmode for historical
+     (i.e., unknown) reasons.  TODO: Is this necessary?  */
+  bool blk = (icode == CODE_FOR_altivec_lvlx
+	      || icode == CODE_FOR_altivec_lvlxl
+	      || icode == CODE_FOR_altivec_lvrx
+	      || icode == CODE_FOR_altivec_lvrxl);
+
+  /* For LVX, express the RTL accurately by ANDing the address with -16.
+     LVXL and LVE*X expand to use UNSPECs to hide their special behavior,
+     so the raw address is fine.  */
+  /* TODO: That statement seems wrong, as the UNSPECs don't surround the
+     memory expression, so a latent bug may lie here.  The &-16 is likely
+     needed for all VMX-style loads.  */
+  if (icode == CODE_FOR_altivec_lvx_v1ti
+      || icode == CODE_FOR_altivec_lvx_v2df
+      || icode == CODE_FOR_altivec_lvx_v2di
+      || icode == CODE_FOR_altivec_lvx_v4sf
+      || icode == CODE_FOR_altivec_lvx_v4si
+      || icode == CODE_FOR_altivec_lvx_v8hi
+      || icode == CODE_FOR_altivec_lvx_v16qi)
+    {
+      rtx rawaddr;
+      if (op[0] == const0_rtx)
+	rawaddr = op[1];
+      else
+	{
+	  op[0] = copy_to_mode_reg (Pmode, op[0]);
+	  rawaddr = gen_rtx_PLUS (Pmode, op[1], op[0]);
+	}
+      rtx addr = gen_rtx_AND (Pmode, rawaddr, gen_rtx_CONST_INT (Pmode, -16));
+      addr = gen_rtx_MEM (blk ? BLKmode : tmode, addr);
+
+      emit_insn (gen_rtx_SET (target, addr));
+    }
+  else
+    {
+      rtx addr;
+      if (op[0] == const0_rtx)
+	addr = gen_rtx_MEM (blk ? BLKmode : tmode, op[1]);
+      else
+	{
+	  op[0] = copy_to_mode_reg (Pmode, op[0]);
+	  addr = gen_rtx_MEM (blk ? BLKmode : tmode,
+			      gen_rtx_PLUS (Pmode, op[1], op[0]));
+	}
+
+      rtx pat = GEN_FCN (icode) (target, addr);
+      if (!pat)
+	return 0;
+      emit_insn (pat);
+    }
+
+  return target;
+}
+
+/* Expand a builtin function that loads a scalar into a vector register
+   with sign extension, and return the expanded rtx.  */
+static rtx
+lxvrse_expand_builtin (rtx target, insn_code icode, rtx *op,
+		       machine_mode tmode, machine_mode smode)
+{
+  rtx pat, addr;
+  op[1] = copy_to_mode_reg (Pmode, op[1]);
+
+  if (op[0] == const0_rtx)
+    addr = gen_rtx_MEM (tmode, op[1]);
+  else
+    {
+      op[0] = copy_to_mode_reg (Pmode, op[0]);
+      addr = gen_rtx_MEM (smode,
+			  gen_rtx_PLUS (Pmode, op[1], op[0]));
+    }
+
+  rtx discratch = gen_reg_rtx (V2DImode);
+  rtx tiscratch = gen_reg_rtx (TImode);
+
+  /* Emit the lxvr*x insn.  */
+  pat = GEN_FCN (icode) (tiscratch, addr);
+  if (!pat)
+    return 0;
+  emit_insn (pat);
+
+  /* Emit a sign extension from V16QI,V8HI,V4SI to V2DI.  */
+  rtx temp1;
+  if (icode == CODE_FOR_vsx_lxvrbx)
+    {
+      temp1  = simplify_gen_subreg (V16QImode, tiscratch, TImode, 0);
+      emit_insn (gen_vsx_sign_extend_qi_v2di (discratch, temp1));
+    }
+  else if (icode == CODE_FOR_vsx_lxvrhx)
+    {
+      temp1  = simplify_gen_subreg (V8HImode, tiscratch, TImode, 0);
+      emit_insn (gen_vsx_sign_extend_hi_v2di (discratch, temp1));
+    }
+  else if (icode == CODE_FOR_vsx_lxvrwx)
+    {
+      temp1  = simplify_gen_subreg (V4SImode, tiscratch, TImode, 0);
+      emit_insn (gen_vsx_sign_extend_si_v2di (discratch, temp1));
+    }
+  else if (icode == CODE_FOR_vsx_lxvrdx)
+    discratch = simplify_gen_subreg (V2DImode, tiscratch, TImode, 0);
+  else
+    gcc_unreachable ();
+
+  /* Emit the sign extension from V2DI (double) to TI (quad).  */
+  rtx temp2 = simplify_gen_subreg (TImode, discratch, V2DImode, 0);
+  emit_insn (gen_extendditi2_vector (target, temp2));
+
+  return target;
+}
+
+/* Expand a builtin function that loads a scalar into a vector register
+   with zero extension, and return the expanded rtx.  */
+static rtx
+lxvrze_expand_builtin (rtx target, insn_code icode, rtx *op,
+		       machine_mode tmode, machine_mode smode)
+{
+  rtx pat, addr;
+  op[1] = copy_to_mode_reg (Pmode, op[1]);
+
+  if (op[0] == const0_rtx)
+    addr = gen_rtx_MEM (tmode, op[1]);
+  else
+    {
+      op[0] = copy_to_mode_reg (Pmode, op[0]);
+      addr = gen_rtx_MEM (smode,
+			  gen_rtx_PLUS (Pmode, op[1], op[0]));
+    }
+
+  pat = GEN_FCN (icode) (target, addr);
+  if (!pat)
+    return 0;
+  emit_insn (pat);
+  return target;
+}
+
+/* Expand an AltiVec vector store builtin, and return the expanded rtx.  */
+static rtx
+stv_expand_builtin (insn_code icode, rtx *op,
+		    machine_mode tmode, machine_mode smode)
+{
+  op[2] = copy_to_mode_reg (Pmode, op[2]);
+
+  /* For STVX, express the RTL accurately by ANDing the address with -16.
+     STVXL and STVE*X expand to use UNSPECs to hide their special behavior,
+     so the raw address is fine.  */
+  /* TODO: That statement seems wrong, as the UNSPECs don't surround the
+     memory expression, so a latent bug may lie here.  The &-16 is likely
+     needed for all VMX-style stores.  */
+  if (icode == CODE_FOR_altivec_stvx_v2df
+      || icode == CODE_FOR_altivec_stvx_v2di
+      || icode == CODE_FOR_altivec_stvx_v4sf
+      || icode == CODE_FOR_altivec_stvx_v4si
+      || icode == CODE_FOR_altivec_stvx_v8hi
+      || icode == CODE_FOR_altivec_stvx_v16qi)
+    {
+      rtx rawaddr;
+      if (op[1] == const0_rtx)
+	rawaddr = op[2];
+      else
+	{
+	  op[1] = copy_to_mode_reg (Pmode, op[1]);
+	  rawaddr = gen_rtx_PLUS (Pmode, op[2], op[1]);
+	}
+
+      rtx addr = gen_rtx_AND (Pmode, rawaddr, gen_rtx_CONST_INT (Pmode, -16));
+      addr = gen_rtx_MEM (tmode, addr);
+      op[0] = copy_to_mode_reg (tmode, op[0]);
+      emit_insn (gen_rtx_SET (addr, op[0]));
+    }
+  else if (icode == CODE_FOR_vsx_stxvrbx
+	   || icode == CODE_FOR_vsx_stxvrhx
+	   || icode == CODE_FOR_vsx_stxvrwx
+	   || icode == CODE_FOR_vsx_stxvrdx)
+    {
+      rtx truncrtx = gen_rtx_TRUNCATE (tmode, op[0]);
+      op[0] = copy_to_mode_reg (E_TImode, truncrtx);
+
+      rtx addr;
+      if (op[1] == const0_rtx)
+	addr = gen_rtx_MEM (Pmode, op[2]);
+      else
+	{
+	  op[1] = copy_to_mode_reg (Pmode, op[1]);
+	  addr = gen_rtx_MEM (tmode, gen_rtx_PLUS (Pmode, op[2], op[1]));
+	}
+      rtx pat = GEN_FCN (icode) (addr, op[0]);
+      if (pat)
+	emit_insn (pat);
+    }
+  else
+    {
+      if (!insn_data[icode].operand[1].predicate (op[0], smode))
+	op[0] = copy_to_mode_reg (smode, op[0]);
+
+      rtx addr;
+      if (op[1] == const0_rtx)
+	addr = gen_rtx_MEM (tmode, op[2]);
+      else
+	{
+	  op[1] = copy_to_mode_reg (Pmode, op[1]);
+	  addr = gen_rtx_MEM (tmode, gen_rtx_PLUS (Pmode, op[2], op[1]));
+	}
+
+      rtx pat = GEN_FCN (icode) (addr, op[0]);
+      if (pat)
+	emit_insn (pat);
+    }
+
+  return NULL_RTX;
+}
+
+/* Expand the MMA built-in in EXP, and return it.  */
+static rtx
+new_mma_expand_builtin (tree exp, rtx target, insn_code icode,
+			rs6000_gen_builtins fcode)
+{
+  tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
+  bool void_func = TREE_TYPE (TREE_TYPE (fndecl)) == void_type_node;
+  machine_mode tmode = VOIDmode;
+  rtx op[MAX_MMA_OPERANDS];
+  unsigned nopnds = 0;
+
+  if (!void_func)
+    {
+      tmode = insn_data[icode].operand[0].mode;
+      if (!(target
+	    && GET_MODE (target) == tmode
+	    && insn_data[icode].operand[0].predicate (target, tmode)))
+	target = gen_reg_rtx (tmode);
+      op[nopnds++] = target;
+    }
+  else
+    target = const0_rtx;
+
+  call_expr_arg_iterator iter;
+  tree arg;
+  FOR_EACH_CALL_EXPR_ARG (arg, iter, exp)
+    {
+      if (arg == error_mark_node)
+	return const0_rtx;
+
+      rtx opnd;
+      const struct insn_operand_data *insn_op;
+      insn_op = &insn_data[icode].operand[nopnds];
+      if (TREE_CODE (arg) == ADDR_EXPR
+	  && MEM_P (DECL_RTL (TREE_OPERAND (arg, 0))))
+	opnd = DECL_RTL (TREE_OPERAND (arg, 0));
+      else
+	opnd = expand_normal (arg);
+
+      if (!insn_op->predicate (opnd, insn_op->mode))
+	{
+	  /* TODO: This use of constraints needs explanation.  */
+	  if (!strcmp (insn_op->constraint, "n"))
+	    {
+	      if (!CONST_INT_P (opnd))
+		error ("argument %d must be an unsigned literal", nopnds);
+	      else
+		error ("argument %d is an unsigned literal that is "
+		       "out of range", nopnds);
+	      return const0_rtx;
+	    }
+	  opnd = copy_to_mode_reg (insn_op->mode, opnd);
+	}
+
+      /* Some MMA instructions have INOUT accumulator operands, so force
+	 their target register to be the same as their input register.  */
+      if (!void_func
+	  && nopnds == 1
+	  && !strcmp (insn_op->constraint, "0")
+	  && insn_op->mode == tmode
+	  && REG_P (opnd)
+	  && insn_data[icode].operand[0].predicate (opnd, tmode))
+	target = op[0] = opnd;
+
+      op[nopnds++] = opnd;
+    }
+
+  rtx pat;
+  switch (nopnds)
+    {
+    case 1:
+      pat = GEN_FCN (icode) (op[0]);
+      break;
+    case 2:
+      pat = GEN_FCN (icode) (op[0], op[1]);
+      break;
+    case 3:
+      /* The ASSEMBLE builtin source operands are reversed in little-endian
+	 mode, so reorder them.  */
+      if (fcode == RS6000_BIF_ASSEMBLE_PAIR_V_INTERNAL && !WORDS_BIG_ENDIAN)
+	std::swap (op[1], op[2]);
+      pat = GEN_FCN (icode) (op[0], op[1], op[2]);
+      break;
+    case 4:
+      pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3]);
+      break;
+    case 5:
+      /* The ASSEMBLE builtin source operands are reversed in little-endian
+	 mode, so reorder them.  */
+      if (fcode == RS6000_BIF_ASSEMBLE_ACC_INTERNAL && !WORDS_BIG_ENDIAN)
+	{
+	  std::swap (op[1], op[4]);
+	  std::swap (op[2], op[3]);
+	}
+      pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4]);
+      break;
+    case 6:
+      pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5]);
+      break;
+    case 7:
+      pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5], op[6]);
+      break;
+    default:
+      gcc_unreachable ();
+    }
+
+  if (!pat)
+    return NULL_RTX;
+
+  emit_insn (pat);
+  return target;
+}
+
+/* Return the appropriate SPR number associated with the given builtin.  */
+static inline HOST_WIDE_INT
+new_htm_spr_num (enum rs6000_gen_builtins code)
+{
+  if (code == RS6000_BIF_GET_TFHAR
+      || code == RS6000_BIF_SET_TFHAR)
+    return TFHAR_SPR;
+  else if (code == RS6000_BIF_GET_TFIAR
+	   || code == RS6000_BIF_SET_TFIAR)
+    return TFIAR_SPR;
+  else if (code == RS6000_BIF_GET_TEXASR
+	   || code == RS6000_BIF_SET_TEXASR)
+    return TEXASR_SPR;
+  gcc_assert (code == RS6000_BIF_GET_TEXASRU
+	      || code == RS6000_BIF_SET_TEXASRU);
+  return TEXASRU_SPR;
+}
+
+/* Expand the HTM builtin in EXP and store the result in TARGET.
+   Return the expanded rtx.  */
+static rtx
+new_htm_expand_builtin (bifdata *bifaddr, rs6000_gen_builtins fcode,
+			tree exp, rtx target)
+{
+  if (!TARGET_POWERPC64
+      && (fcode == RS6000_BIF_TABORTDC
+	  || fcode == RS6000_BIF_TABORTDCI))
+    {
+      error ("builtin %qs is only valid in 64-bit mode", bifaddr->bifname);
+      return const0_rtx;
+    }
+
+  tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
+  bool nonvoid = TREE_TYPE (TREE_TYPE (fndecl)) != void_type_node;
+  bool uses_spr = bif_is_htmspr (*bifaddr);
+  insn_code icode = bifaddr->icode;
+
+  if (uses_spr)
+    icode = rs6000_htm_spr_icode (nonvoid);
+
+  rtx op[MAX_HTM_OPERANDS];
+  int nopnds = 0;
+  const insn_operand_data *insn_op = &insn_data[icode].operand[0];
+
+  if (nonvoid)
+    {
+      machine_mode tmode = (uses_spr) ? insn_op->mode : E_SImode;
+      if (!target
+	  || GET_MODE (target) != tmode
+	  || (uses_spr && !insn_op->predicate (target, tmode)))
+	target = gen_reg_rtx (tmode);
+      if (uses_spr)
+	op[nopnds++] = target;
+    }
+
+  tree arg;
+  call_expr_arg_iterator iter;
+
+  FOR_EACH_CALL_EXPR_ARG (arg, iter, exp)
+    {
+      if (arg == error_mark_node || nopnds >= MAX_HTM_OPERANDS)
+	return const0_rtx;
+
+      insn_op = &insn_data[icode].operand[nopnds];
+      op[nopnds] = expand_normal (arg);
+
+      if (!insn_op->predicate (op[nopnds], insn_op->mode))
+	{
+	  /* TODO: This use of constraints could use explanation.
+	     This happens a couple of places, perhaps make that a
+	     function to document what's happening.  */
+	  if (!strcmp (insn_op->constraint, "n"))
+	    {
+	      int arg_num = nonvoid ? nopnds : nopnds + 1;
+	      if (!CONST_INT_P (op[nopnds]))
+		error ("argument %d must be an unsigned literal", arg_num);
+	      else
+		error ("argument %d is an unsigned literal that is "
+		       "out of range", arg_num);
+	      return const0_rtx;
+	    }
+	  op[nopnds] = copy_to_mode_reg (insn_op->mode, op[nopnds]);
+	}
+
+      nopnds++;
+    }
+
+  /* Handle the builtins for extended mnemonics.  These accept
+     no arguments, but map to builtins that take arguments.  */
+  switch (fcode)
+    {
+    case RS6000_BIF_TENDALL:  /* Alias for: tend. 1  */
+    case RS6000_BIF_TRESUME:  /* Alias for: tsr. 1  */
+      op[nopnds++] = GEN_INT (1);
+      break;
+    case RS6000_BIF_TSUSPEND: /* Alias for: tsr. 0  */
+      op[nopnds++] = GEN_INT (0);
+      break;
+    default:
+      break;
+    }
+
+  /* If this builtin accesses SPRs, then pass in the appropriate
+     SPR number and SPR regno as the last two operands.  */
+  rtx cr = NULL_RTX;
+  if (uses_spr)
+    {
+      machine_mode mode = TARGET_POWERPC64 ? DImode : SImode;
+      op[nopnds++] = gen_rtx_CONST_INT (mode, new_htm_spr_num (fcode));
+    }
+  /* If this builtin accesses a CR field, then pass in a scratch
+     CR field as the last operand.  */
+  else if (bif_is_htmcr (*bifaddr))
+    {
+      cr = gen_reg_rtx (CCmode);
+      op[nopnds++] = cr;
+    }
+
+  rtx pat;
+  switch (nopnds)
+    {
+    case 1:
+      pat = GEN_FCN (icode) (op[0]);
+      break;
+    case 2:
+      pat = GEN_FCN (icode) (op[0], op[1]);
+      break;
+    case 3:
+      pat = GEN_FCN (icode) (op[0], op[1], op[2]);
+      break;
+    case 4:
+      pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3]);
+      break;
+    default:
+      gcc_unreachable ();
+    }
+  if (!pat)
+    return NULL_RTX;
+  emit_insn (pat);
+
+  if (bif_is_htmcr (*bifaddr))
+    {
+      if (fcode == RS6000_BIF_TBEGIN)
+	{
+	  /* Emit code to set TARGET to true or false depending on
+	     whether the tbegin. instruction succeeded or failed
+	     to start a transaction.  We do this by placing the 1's
+	     complement of CR's EQ bit into TARGET.  */
+	  rtx scratch = gen_reg_rtx (SImode);
+	  emit_insn (gen_rtx_SET (scratch,
+				  gen_rtx_EQ (SImode, cr,
+					      const0_rtx)));
+	  emit_insn (gen_rtx_SET (target,
+				  gen_rtx_XOR (SImode, scratch,
+					       GEN_INT (1))));
+	}
+      else
+	{
+	  /* Emit code to copy the 4-bit condition register field
+	     CR into the least significant end of register TARGET.  */
+	  rtx scratch1 = gen_reg_rtx (SImode);
+	  rtx scratch2 = gen_reg_rtx (SImode);
+	  rtx subreg = simplify_gen_subreg (CCmode, scratch1, SImode, 0);
+	  emit_insn (gen_movcc (subreg, cr));
+	  emit_insn (gen_lshrsi3 (scratch2, scratch1, GEN_INT (28)));
+	  emit_insn (gen_andsi3 (target, scratch2, GEN_INT (0xf)));
+	}
+    }
+
+  if (nonvoid)
+    return target;
+  return const0_rtx;
+}
+
+/* Expand an expression EXP that calls a built-in function,
+   with result going to TARGET if that's convenient
+   (and in mode MODE if that's convenient).
+   SUBTARGET may be used as the target for computing one of EXP's operands.
+   IGNORE is nonzero if the value is to be ignored.
+   Use the new builtin infrastructure.  */
+static rtx
+rs6000_expand_new_builtin (tree exp, rtx target,
+			   rtx /* subtarget */,
+			   machine_mode /* mode */,
+			   int ignore)
+{
+  tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
+  enum rs6000_gen_builtins fcode
+    = (enum rs6000_gen_builtins) DECL_MD_FUNCTION_CODE (fndecl);
+  size_t uns_fcode = (size_t)fcode;
+  enum insn_code icode = rs6000_builtin_info_x[uns_fcode].icode;
+
+  /* TODO: The following commentary and code is inherited from the original
+     builtin processing code.  The commentary is a bit confusing, with the
+     intent being that KFmode is always IEEE-128, IFmode is always IBM
+     double-double, and TFmode is the current long double.  The code is
+     confusing in that it converts from KFmode to TFmode pattern names,
+     when the other direction is more intuitive.  Try to address this.  */
+
+  /* We have two different modes (KFmode, TFmode) that are the IEEE
+     128-bit floating point type, depending on whether long double is the
+     IBM extended double (KFmode) or long double is IEEE 128-bit (TFmode).
+     It is simpler if we only define one variant of the built-in function,
+     and switch the code when defining it, rather than defining two built-
+     ins and using the overload table in rs6000-c.c to switch between the
+     two.  If we don't have the proper assembler, don't do this switch
+     because CODE_FOR_*kf* and CODE_FOR_*tf* will be CODE_FOR_nothing.  */
+  if (FLOAT128_IEEE_P (TFmode))
+    switch (icode)
+      {
+      case CODE_FOR_sqrtkf2_odd:
+	icode = CODE_FOR_sqrttf2_odd;
+	break;
+      case CODE_FOR_trunckfdf2_odd:
+	icode = CODE_FOR_trunctfdf2_odd;
+	break;
+      case CODE_FOR_addkf3_odd:
+	icode = CODE_FOR_addtf3_odd;
+	break;
+      case CODE_FOR_subkf3_odd:
+	icode = CODE_FOR_subtf3_odd;
+	break;
+      case CODE_FOR_mulkf3_odd:
+	icode = CODE_FOR_multf3_odd;
+	break;
+      case CODE_FOR_divkf3_odd:
+	icode = CODE_FOR_divtf3_odd;
+	break;
+      case CODE_FOR_fmakf4_odd:
+	icode = CODE_FOR_fmatf4_odd;
+	break;
+      case CODE_FOR_xsxexpqp_kf:
+	icode = CODE_FOR_xsxexpqp_tf;
+	break;
+      case CODE_FOR_xsxsigqp_kf:
+	icode = CODE_FOR_xsxsigqp_tf;
+	break;
+      case CODE_FOR_xststdcnegqp_kf:
+	icode = CODE_FOR_xststdcnegqp_tf;
+	break;
+      case CODE_FOR_xsiexpqp_kf:
+	icode = CODE_FOR_xsiexpqp_tf;
+	break;
+      case CODE_FOR_xsiexpqpf_kf:
+	icode = CODE_FOR_xsiexpqpf_tf;
+	break;
+      case CODE_FOR_xststdcqp_kf:
+	icode = CODE_FOR_xststdcqp_tf;
+	break;
+      case CODE_FOR_xscmpexpqp_eq_kf:
+	icode = CODE_FOR_xscmpexpqp_eq_tf;
+	break;
+      case CODE_FOR_xscmpexpqp_lt_kf:
+	icode = CODE_FOR_xscmpexpqp_lt_tf;
+	break;
+      case CODE_FOR_xscmpexpqp_gt_kf:
+	icode = CODE_FOR_xscmpexpqp_gt_tf;
+	break;
+      case CODE_FOR_xscmpexpqp_unordered_kf:
+	icode = CODE_FOR_xscmpexpqp_unordered_tf;
+	break;
+      default:
+	break;
+      }
+
+  /* In case of "#pragma target" changes, we initialize all builtins
+     but check for actual availability now, during expand time.  For
+     invalid builtins, generate a normal call.  */
+  bifdata *bifaddr = &rs6000_builtin_info_x[uns_fcode];
+  bif_enable e = bifaddr->enable;
+
+  if (!(e == ENB_ALWAYS
+	|| (e == ENB_P5         && TARGET_POPCNTB)
+	|| (e == ENB_P6         && TARGET_CMPB)
+	|| (e == ENB_ALTIVEC    && TARGET_ALTIVEC)
+	|| (e == ENB_CELL       && TARGET_ALTIVEC
+				&& rs6000_cpu == PROCESSOR_CELL)
+	|| (e == ENB_VSX        && TARGET_VSX)
+	|| (e == ENB_P7         && TARGET_POPCNTD)
+	|| (e == ENB_P7_64      && TARGET_POPCNTD
+				&& TARGET_POWERPC64)
+	|| (e == ENB_P8         && TARGET_DIRECT_MOVE)
+	|| (e == ENB_P8V        && TARGET_P8_VECTOR)
+	|| (e == ENB_P9         && TARGET_MODULO)
+	|| (e == ENB_P9_64      && TARGET_MODULO
+				&& TARGET_POWERPC64)
+	|| (e == ENB_P9V        && TARGET_P9_VECTOR)
+	|| (e == ENB_IEEE128_HW && TARGET_FLOAT128_HW)
+	|| (e == ENB_DFP        && TARGET_DFP)
+	|| (e == ENB_CRYPTO     && TARGET_CRYPTO)
+	|| (e == ENB_HTM        && TARGET_HTM)
+	|| (e == ENB_P10        && TARGET_POWER10)
+	|| (e == ENB_P10_64     && TARGET_POWER10
+				&& TARGET_POWERPC64)
+	|| (e == ENB_MMA        && TARGET_MMA)))
+    {
+      rs6000_invalid_new_builtin (fcode);
+      return expand_call (exp, target, ignore);
+    }
+
+  if (bif_is_nosoft (*bifaddr)
+      && rs6000_isa_flags & OPTION_MASK_SOFT_FLOAT)
+    {
+      error ("%<%s%> not supported with %<-msoft-float%>",
+	     bifaddr->bifname);
+      return const0_rtx;
+    }
+
+  if (bif_is_no32bit (*bifaddr) && TARGET_32BIT)
+    fatal_error (input_location,
+		 "%<%s%> is not supported in 32-bit mode",
+		 bifaddr->bifname);
+
+  if (bif_is_cpu (*bifaddr))
+    return new_cpu_expand_builtin (fcode, exp, target);
+
+  if (bif_is_init (*bifaddr))
+    return altivec_expand_vec_init_builtin (TREE_TYPE (exp), exp, target);
+
+  if (bif_is_set (*bifaddr))
+    return altivec_expand_vec_set_builtin (exp);
+
+  if (bif_is_extract (*bifaddr))
+    return altivec_expand_vec_ext_builtin (exp, target);
+
+  if (bif_is_predicate (*bifaddr))
+    return altivec_expand_predicate_builtin (icode, exp, target);
+
+  if (bif_is_htm (*bifaddr))
+    return new_htm_expand_builtin (bifaddr, fcode, exp, target);
+
+  if (bif_is_32bit (*bifaddr) && TARGET_32BIT)
+    {
+      if (fcode == RS6000_BIF_MFTB)
+	icode = CODE_FOR_rs6000_mftb_si;
+      else
+	gcc_unreachable ();
+    }
+
+  if (bif_is_endian (*bifaddr) && BYTES_BIG_ENDIAN)
+    {
+      if (fcode == RS6000_BIF_LD_ELEMREV_V1TI)
+	icode = CODE_FOR_vsx_load_v1ti;
+      else if (fcode == RS6000_BIF_LD_ELEMREV_V2DF)
+	icode = CODE_FOR_vsx_load_v2df;
+      else if (fcode == RS6000_BIF_LD_ELEMREV_V2DI)
+	icode = CODE_FOR_vsx_load_v2di;
+      else if (fcode == RS6000_BIF_LD_ELEMREV_V4SF)
+	icode = CODE_FOR_vsx_load_v4sf;
+      else if (fcode == RS6000_BIF_LD_ELEMREV_V4SI)
+	icode = CODE_FOR_vsx_load_v4si;
+      else if (fcode == RS6000_BIF_LD_ELEMREV_V8HI)
+	icode = CODE_FOR_vsx_load_v8hi;
+      else if (fcode == RS6000_BIF_LD_ELEMREV_V16QI)
+	icode = CODE_FOR_vsx_load_v16qi;
+      else if (fcode == RS6000_BIF_ST_ELEMREV_V1TI)
+	icode = CODE_FOR_vsx_store_v1ti;
+      else if (fcode == RS6000_BIF_ST_ELEMREV_V2DF)
+	icode = CODE_FOR_vsx_store_v2df;
+      else if (fcode == RS6000_BIF_ST_ELEMREV_V2DI)
+	icode = CODE_FOR_vsx_store_v2di;
+      else if (fcode == RS6000_BIF_ST_ELEMREV_V4SF)
+	icode = CODE_FOR_vsx_store_v4sf;
+      else if (fcode == RS6000_BIF_ST_ELEMREV_V4SI)
+	icode = CODE_FOR_vsx_store_v4si;
+      else if (fcode == RS6000_BIF_ST_ELEMREV_V8HI)
+	icode = CODE_FOR_vsx_store_v8hi;
+      else if (fcode == RS6000_BIF_ST_ELEMREV_V16QI)
+	icode = CODE_FOR_vsx_store_v16qi;
+      else
+	gcc_unreachable ();
+    }
+
+
+  /* TRUE iff the built-in function returns void.  */
+  bool void_func = TREE_TYPE (TREE_TYPE (fndecl)) == void_type_node;
+  /* Position of first argument (0 for void-returning functions, else 1).  */
+  int k;
+  /* Modes for the return value, if any, and arguments.  */
+  const int MAX_BUILTIN_ARGS = 6;
+  machine_mode mode[MAX_BUILTIN_ARGS + 1];
+
+  if (void_func)
+    k = 0;
+  else
+    {
+      k = 1;
+      mode[0] = insn_data[icode].operand[0].mode;
+    }
+
+  /* Tree expressions for each argument.  */
+  tree arg[MAX_BUILTIN_ARGS];
+  /* RTL expressions for each argument.  */
+  rtx op[MAX_BUILTIN_ARGS];
+
+  int nargs = bifaddr->nargs;
+  gcc_assert (nargs <= MAX_BUILTIN_ARGS);
+
+
+  for (int i = 0; i < nargs; i++)
+    {
+      arg[i] = CALL_EXPR_ARG (exp, i);
+      if (arg[i] == error_mark_node)
+	return const0_rtx;
+      STRIP_NOPS (arg[i]);
+      op[i] = expand_normal (arg[i]);
+      /* We have a couple of pesky patterns that don't specify the mode...  */
+      mode[i+k] = insn_data[icode].operand[i+k].mode;
+      if (!mode[i+k])
+	mode[i+k] = Pmode;
+    }
+
+  /* Check for restricted constant arguments.  */
+  for (int i = 0; i < 2; i++)
+    {
+      switch (bifaddr->restr[i])
+	{
+	case RES_BITS:
+	  {
+	    size_t mask = 1;
+	    mask <<= bifaddr->restr_val1[i];
+	    mask--;
+	    tree restr_arg = arg[bifaddr->restr_opnd[i] - 1];
+	    STRIP_NOPS (restr_arg);
+	    if (!(TREE_CODE (restr_arg) == INTEGER_CST
+		  && (TREE_INT_CST_LOW (restr_arg) & ~mask) == 0))
+	      {
+		error ("argument %d must be a %d-bit unsigned literal",
+		       bifaddr->restr_opnd[i], bifaddr->restr_val1[i]);
+		return CONST0_RTX (mode[0]);
+	      }
+	    break;
+	  }
+	case RES_RANGE:
+	  {
+	    tree restr_arg = arg[bifaddr->restr_opnd[i] - 1];
+	    STRIP_NOPS (restr_arg);
+	    if (!(TREE_CODE (restr_arg) == INTEGER_CST
+		  && IN_RANGE (tree_to_shwi (restr_arg),
+			       bifaddr->restr_val1[i],
+			       bifaddr->restr_val2[i])))
+	      {
+		error ("argument %d must be a literal between %d and %d,"
+		       " inclusive",
+		       bifaddr->restr_opnd[i], bifaddr->restr_val1[i],
+		       bifaddr->restr_val2[i]);
+		return CONST0_RTX (mode[0]);
+	      }
+	    break;
+	  }
+	case RES_VAR_RANGE:
+	  {
+	    tree restr_arg = arg[bifaddr->restr_opnd[i] - 1];
+	    STRIP_NOPS (restr_arg);
+	    if (TREE_CODE (restr_arg) == INTEGER_CST
+		&& !IN_RANGE (tree_to_shwi (restr_arg),
+			      bifaddr->restr_val1[i],
+			      bifaddr->restr_val2[i]))
+	      {
+		error ("argument %d must be a variable or a literal "
+		       "between %d and %d, inclusive",
+		       bifaddr->restr_opnd[i], bifaddr->restr_val1[i],
+		       bifaddr->restr_val2[i]);
+		return CONST0_RTX (mode[0]);
+	      }
+	    break;
+	  }
+	case RES_VALUES:
+	  {
+	    tree restr_arg = arg[bifaddr->restr_opnd[i] - 1];
+	    STRIP_NOPS (restr_arg);
+	    if (!(TREE_CODE (restr_arg) == INTEGER_CST
+		  && (tree_to_shwi (restr_arg) == bifaddr->restr_val1[i]
+		      || tree_to_shwi (restr_arg) == bifaddr->restr_val2[i])))
+	      {
+		error ("argument %d must be either a literal %d or a "
+		       "literal %d",
+		       bifaddr->restr_opnd[i], bifaddr->restr_val1[i],
+		       bifaddr->restr_val2[i]);
+		return CONST0_RTX (mode[0]);
+	      }
+	    break;
+	  }
+	default:
+	case RES_NONE:
+	  break;
+	}
+    }
+
+  if (bif_is_ldstmask (*bifaddr))
+    return rs6000_expand_ldst_mask (target, arg[0]);
+
+  if (bif_is_stvec (*bifaddr))
+    {
+      if (bif_is_reve (*bifaddr))
+	icode = elemrev_icode (fcode);
+      return stv_expand_builtin (icode, op, mode[0], mode[1]);
+    }
+
+  if (bif_is_ldvec (*bifaddr))
+    {
+      if (bif_is_reve (*bifaddr))
+	icode = elemrev_icode (fcode);
+      return ldv_expand_builtin (target, icode, op, mode[0]);
+    }
+
+  if (bif_is_lxvrse (*bifaddr))
+    return lxvrse_expand_builtin (target, icode, op, mode[0], mode[1]);
+
+  if (bif_is_lxvrze (*bifaddr))
+    return lxvrze_expand_builtin (target, icode, op, mode[0], mode[1]);
+
+  if (bif_is_mma (*bifaddr))
+    return new_mma_expand_builtin (exp, target, icode, fcode);
+
+  if (fcode == RS6000_BIF_PACK_IF
+      && TARGET_LONG_DOUBLE_128
+      && !TARGET_IEEEQUAD)
+    {
+      icode = CODE_FOR_packtf;
+      fcode = RS6000_BIF_PACK_TF;
+      uns_fcode = (size_t) fcode;
+    }
+  else if (fcode == RS6000_BIF_UNPACK_IF
+	   && TARGET_LONG_DOUBLE_128
+	   && !TARGET_IEEEQUAD)
+    {
+      icode = CODE_FOR_unpacktf;
+      fcode = RS6000_BIF_UNPACK_TF;
+      uns_fcode = (size_t) fcode;
+    }
+
+  if (TREE_TYPE (TREE_TYPE (fndecl)) == void_type_node)
+    target = NULL_RTX;
+  else if (target == 0
+	   || GET_MODE (target) != mode[0]
+	   || !insn_data[icode].operand[0].predicate (target, mode[0]))
+    target = gen_reg_rtx (mode[0]);
+
+  for (int i = 0; i < nargs; i++)
+    if (!insn_data[icode].operand[i+k].predicate (op[i], mode[i+k]))
+      op[i] = copy_to_mode_reg (mode[i+k], op[i]);
+
+  rtx pat;
+
+  switch (nargs)
+    {
+    case 0:
+      pat = (void_func
+	     ? GEN_FCN (icode) ()
+	     : GEN_FCN (icode) (target));
+      break;
+    case 1:
+      pat = (void_func
+	     ? GEN_FCN (icode) (op[0])
+	     : GEN_FCN (icode) (target, op[0]));
+      break;
+    case 2:
+      pat = (void_func
+	     ? GEN_FCN (icode) (op[0], op[1])
+	     : GEN_FCN (icode) (target, op[0], op[1]));
+      break;
+    case 3:
+      pat = (void_func
+	     ? GEN_FCN (icode) (op[0], op[1], op[2])
+	     : GEN_FCN (icode) (target, op[0], op[1], op[2]));
+      break;
+    case 4:
+      pat = (void_func
+	     ? GEN_FCN (icode) (op[0], op[1], op[2], op[3])
+	     : GEN_FCN (icode) (target, op[0], op[1], op[2], op[3]));
+      break;
+    case 5:
+      pat = (void_func
+	     ? GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4])
+	     : GEN_FCN (icode) (target, op[0], op[1], op[2], op[3], op[4]));
+      break;
+    case 6:
+      pat = (void_func
+	     ? GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5])
+	     : GEN_FCN (icode) (target, op[0], op[1],
+				op[2], op[3], op[4], op[5]));
+      break;
+    default:
+      gcc_assert (MAX_BUILTIN_ARGS == 6);
+      gcc_unreachable ();
+    }
+
+  if (!pat)
+    return 0;
+
+  emit_insn (pat);
+  return target;
+}
+
 /* Create a builtin vector type with a name.  Taking care not to give
    the canonical type a name.  */
 
@@ -14983,8 +16409,72 @@ rs6000_init_builtins (void)
   /* Execute the autogenerated initialization code for builtins.  */
   rs6000_init_generated_builtins ();
 
+  if (TARGET_DEBUG_BUILTIN)
+    {
+      fprintf (stderr, "\nAutogenerated built-in functions:\n\n");
+      for (int i = 1; i < (int) RS6000_BIF_MAX; i++)
+	{
+	  bif_enable e = rs6000_builtin_info_x[i].enable;
+	  if (e == ENB_P5 && !TARGET_POPCNTB)
+	    continue;
+	  if (e == ENB_P6 && !TARGET_CMPB)
+	    continue;
+	  if (e == ENB_ALTIVEC && !TARGET_ALTIVEC)
+	    continue;
+	  if (e == ENB_VSX && !TARGET_VSX)
+	    continue;
+	  if (e == ENB_P7 && !TARGET_POPCNTD)
+	    continue;
+	  if (e == ENB_P7_64 && !(TARGET_POPCNTD && TARGET_POWERPC64))
+	    continue;
+	  if (e == ENB_P8 && !TARGET_DIRECT_MOVE)
+	    continue;
+	  if (e == ENB_P8V && !TARGET_P8_VECTOR)
+	    continue;
+	  if (e == ENB_P9 && !TARGET_MODULO)
+	    continue;
+	  if (e == ENB_P9_64 && !(TARGET_MODULO && TARGET_POWERPC64))
+	    continue;
+	  if (e == ENB_P9V && !TARGET_P9_VECTOR)
+	    continue;
+	  if (e == ENB_IEEE128_HW && !TARGET_FLOAT128_HW)
+	    continue;
+	  if (e == ENB_DFP && !TARGET_DFP)
+	    continue;
+	  if (e == ENB_CRYPTO && !TARGET_CRYPTO)
+	    continue;
+	  if (e == ENB_HTM && !TARGET_HTM)
+	    continue;
+	  if (e == ENB_P10 && !TARGET_POWER10)
+	    continue;
+	  if (e == ENB_P10_64 && !(TARGET_POWER10 && TARGET_POWERPC64))
+	    continue;
+	  if (e == ENB_MMA && !TARGET_MMA)
+	    continue;
+	  tree fntype = rs6000_builtin_info_x[i].fntype;
+	  tree t = TREE_TYPE (fntype);
+	  fprintf (stderr, "%s %s (", rs6000_type_string (t),
+		   rs6000_builtin_info_x[i].bifname);
+	  t = TYPE_ARG_TYPES (fntype);
+	  while (t && TREE_VALUE (t) != void_type_node)
+	    {
+	      fprintf (stderr, "%s",
+		       rs6000_type_string (TREE_VALUE (t)));
+	      t = TREE_CHAIN (t);
+	      if (t && TREE_VALUE (t) != void_type_node)
+		fprintf (stderr, ", ");
+	    }
+	  fprintf (stderr, "); %s [%4d]\n",
+		   rs6000_builtin_info_x[i].attr_string, (int) i);
+	}
+      fprintf (stderr, "\nEnd autogenerated built-in functions.\n\n\n");
+     }
+
   if (new_builtins_are_live)
     {
+      altivec_builtin_mask_for_load
+	= rs6000_builtin_decls_x[RS6000_BIF_MASK_FOR_LOAD];
+
 #ifdef SUBTARGET_INIT_BUILTINS
       SUBTARGET_INIT_BUILTINS;
 #endif
@@ -15119,11 +16609,31 @@ rs6000_init_builtins (void)
     }
 }
 
+static tree
+rs6000_new_builtin_decl (unsigned code, bool /* initialize_p */)
+{
+  rs6000_gen_builtins fcode = (rs6000_gen_builtins) code;
+
+  if (fcode >= RS6000_OVLD_MAX)
+    return error_mark_node;
+
+  if (!rs6000_new_builtin_is_supported (fcode))
+    {
+      rs6000_invalid_new_builtin (fcode);
+      return error_mark_node;
+    }
+
+  return rs6000_builtin_decls_x[code];
+}
+
 /* Returns the rs6000 builtin decl for CODE.  */
 
 tree
 rs6000_builtin_decl (unsigned code, bool initialize_p ATTRIBUTE_UNUSED)
 {
+  if (new_builtins_are_live)
+    return rs6000_new_builtin_decl (code, initialize_p);
+
   HOST_WIDE_INT fnmask;
 
   if (code >= RS6000_BUILTIN_COUNT)
@@ -15626,6 +17136,14 @@ altivec_init_builtins (void)
 			       ALTIVEC_BUILTIN_MASK_FOR_LOAD,
 			       BUILT_IN_MD, NULL, NULL_TREE);
   TREE_READONLY (decl) = 1;
+  if (TARGET_DEBUG_BUILTIN)
+    {
+      tree arg_type = TREE_VALUE (TYPE_ARG_TYPES (v16qi_ftype_pcvoid));
+      fprintf (stderr, "%s __builtin_altivec_mask_for_load (%s); [%4d]\n",
+	       rs6000_type_string (TREE_TYPE (v16qi_ftype_pcvoid)),
+	       rs6000_type_string (arg_type),
+	       (int) ALTIVEC_BUILTIN_MASK_FOR_LOAD);
+    }
   /* Record the decl. Will be used by rs6000_builtin_mask_for_load.  */
   altivec_builtin_mask_for_load = decl;
 
diff --git a/gcc/config/rs6000/rs6000-gen-builtins.c b/gcc/config/rs6000/rs6000-gen-builtins.c
index 7f71121..1655a2f 100644
--- a/gcc/config/rs6000/rs6000-gen-builtins.c
+++ b/gcc/config/rs6000/rs6000-gen-builtins.c
@@ -1,5 +1,5 @@
 /* Generate built-in function initialization and recognition for Power.
-   Copyright (C) 2020-21 Free Software Foundation, Inc.
+   Copyright (C) 2020-2021 Free Software Foundation, Inc.
    Contributed by Bill Schmidt, IBM <wschmidt@linux.ibm.com>
 
 This file is part of GCC.
@@ -183,11 +183,20 @@ static const char *defines_path;
 /* Position information.  Note that "pos" is zero-indexed, but users
    expect one-indexed column information, so representations of "pos"
    as columns in diagnostic messages must be adjusted.  */
+#define MAXLINES 4
 #define LINELEN 1024
-static char linebuf[LINELEN];
+static char linebuf[LINELEN * MAXLINES];
 static int line;
 static int pos;
 
+/* Escape-newline support.  For readability, we prefer to allow developers
+   to use escape-newline to continue long lines to the next one.  We
+   maintain a buffer of "original" lines here, which are concatenated into
+   linebuf, above, and which can be used to convert the virtual line
+   position "line / pos" into actual line and position information.  */
+static char *lines[MAXLINES];
+static int lastline;
+
 /* Used to determine whether a type can be void (only return types).  */
 enum void_status
 {
@@ -568,31 +577,65 @@ static typemap type_map[TYPE_MAP_SIZE] =
     { "vp8hi",		"pixel_V8HI" },
   };
 
+/* From a possibly extended line with a virtual position, calculate
+   the current line and character position.  */
+static void
+real_line_pos (int diagpos, int *real_line, int *real_pos)
+{
+  *real_line = line - lastline;
+  *real_pos = diagpos;
+
+  for (int i = 0; i < MAXLINES; i++)
+    {
+      int len = strlen(lines[i]);
+      if (*real_pos <= len)
+	break;
+
+      (*real_line)++;
+      *real_pos -= len - 2;
+    }
+
+  /* Convert from zero-base to one-base for printing.  */
+  (*real_pos)++;
+}
+
 /* Pointer to a diagnostic function.  */
-static void (*diag) (const char *, ...)
-  __attribute__ ((format (printf, 1, 2)));
+static void (*diag) (int, const char *, ...)
+  __attribute__ ((format (printf, 2, 3)));
 
 /* Custom diagnostics.  */
-static void __attribute__ ((format (printf, 1, 2)))
-bif_diag (const char * fmt, ...)
+static void __attribute__ ((format (printf, 2, 3)))
+bif_diag (int diagpos, const char * fmt, ...)
 {
   va_list args;
-  fprintf (stderr, "%s:%d: ", bif_path, line);
+  int real_line, real_pos;
+  real_line_pos (diagpos, &real_line, &real_pos);
+  fprintf (stderr, "%s:%d:%d: ", bif_path, real_line, real_pos);
   va_start (args, fmt);
   vfprintf (stderr, fmt, args);
   va_end (args);
 }
 
-static void __attribute__ ((format (printf, 1, 2)))
-ovld_diag (const char * fmt, ...)
+static void __attribute__ ((format (printf, 2, 3)))
+ovld_diag (int diagpos, const char * fmt, ...)
 {
   va_list args;
-  fprintf (stderr, "%s:%d: ", ovld_path, line);
+  int real_line, real_pos;
+  real_line_pos (diagpos, &real_line, &real_pos);
+  fprintf (stderr, "%s:%d:%d: ", ovld_path, real_line, real_pos);
   va_start (args, fmt);
   vfprintf (stderr, fmt, args);
   va_end (args);
 }
 
+/* Produce a fatal error message.  */
+static void
+fatal (const char *msg)
+{
+  fprintf (stderr, "FATAL: %s\n", msg);
+  abort ();
+}
+
 /* Pass over whitespace (other than a newline, which terminates the scan).  */
 static void
 consume_whitespace (void)
@@ -602,7 +645,7 @@ consume_whitespace (void)
 
   if (pos >= LINELEN)
     {
-      diag ("line length overrun at %d.\n", pos);
+      diag (pos, "line length overrun.\n");
       exit (1);
     }
 
@@ -620,8 +663,28 @@ advance_line (FILE *file)
 	return 0;
       line++;
       size_t len = strlen (linebuf);
+
+      /* Escape-newline processing.  */
+      lastline = 0;
+      if (len > 1)
+	{
+	  strcpy (lines[0], linebuf);
+	  while (linebuf[len - 2] == '\\'
+		 && linebuf[len - 1] == '\n')
+	    {
+	      lastline++;
+	      if (lastline == MAXLINES)
+		fatal ("number of supported overflow lines exceeded");
+	      line++;
+	      if (!fgets (lines[lastline], LINELEN, file))
+		fatal ("unexpected end of file");
+	      strcpy (&linebuf[len - 2], lines[lastline]);
+	      len += strlen (lines[lastline]) - 2;
+	    }
+	}
+
       if (linebuf[len - 1] != '\n')
-	(*diag) ("line doesn't terminate with newline\n");
+	fatal ("line doesn't terminate with newline");
       pos = 0;
       consume_whitespace ();
       if (linebuf[pos] != '\n' && linebuf[pos] != ';')
@@ -634,7 +697,7 @@ safe_inc_pos (void)
 {
   if (++pos >= LINELEN)
     {
-      (*diag) ("line length overrun.\n");
+      diag (pos, "line length overrun.\n");
       exit (1);
     }
 }
@@ -651,7 +714,7 @@ match_identifier (void)
 
   if (lastpos >= LINELEN - 1)
     {
-      diag ("line length overrun at %d.\n", lastpos);
+      diag (lastpos, "line length overrun.\n");
       exit (1);
     }
 
@@ -681,7 +744,7 @@ match_integer (void)
 
   if (lastpos >= LINELEN - 1)
     {
-      diag ("line length overrun at %d.\n", lastpos);
+      diag (lastpos, "line length overrun.\n");
       exit (1);
     }
 
@@ -705,16 +768,13 @@ match_to_right_bracket (void)
   while (lastpos < LINELEN - 1 && linebuf[lastpos + 1] != ']')
     {
       if (linebuf[lastpos + 1] == '\n')
-	{
-	  (*diag) ("no ']' found before end of line.\n");
-	  exit (1);
-	}
+	fatal ("no ']' found before end of line.\n");
       ++lastpos;
     }
 
   if (lastpos >= LINELEN - 1)
     {
-      diag ("line length overrun at %d.\n", lastpos);
+      diag (lastpos, "line length overrun.\n");
       exit (1);
     }
 
@@ -740,14 +800,6 @@ handle_pointer (typeinfo *typedata)
     }
 }
 
-/* Produce a fatal error message.  */
-static void
-fatal (const char *msg)
-{
-  fprintf (stderr, "FATAL: %s\n", msg);
-  abort ();
-}
-
 static bif_stanza
 stanza_name_to_stanza (const char *stanza_name)
 {
@@ -771,7 +823,7 @@ match_basetype (typeinfo *typedata)
   char *token = match_identifier ();
   if (!token)
     {
-      (*diag) ("missing base type in return type at column %d\n", pos + 1);
+      diag (pos, "missing base type in return type\n");
       return 0;
     }
 
@@ -825,7 +877,7 @@ match_basetype (typeinfo *typedata)
     typedata->base = BT_IBM128;
   else
     {
-      (*diag) ("unrecognized base type at column %d\n", oldpos + 1);
+      diag (oldpos, "unrecognized base type\n");
       return 0;
     }
 
@@ -845,13 +897,13 @@ match_bracketed_pair (typeinfo *typedata, char open, char close,
       char *x = match_integer ();
       if (x == NULL)
 	{
-	  (*diag) ("malformed integer at column %d.\n", oldpos + 1);
+	  diag (oldpos, "malformed integer.\n");
 	  return 0;
 	}
       consume_whitespace ();
       if (linebuf[pos] != ',')
 	{
-	  (*diag) ("missing comma at column %d.\n", pos + 1);
+	  diag (pos, "missing comma.\n");
 	  return 0;
 	}
       safe_inc_pos ();
@@ -860,7 +912,7 @@ match_bracketed_pair (typeinfo *typedata, char open, char close,
       char *y = match_integer ();
       if (y == NULL)
 	{
-	  (*diag) ("malformed integer at column %d.\n", oldpos + 1);
+	  diag (oldpos, "malformed integer.\n");
 	  return 0;
 	}
       typedata->restr = restr;
@@ -870,7 +922,7 @@ match_bracketed_pair (typeinfo *typedata, char open, char close,
       consume_whitespace ();
       if (linebuf[pos] != close)
 	{
-	  (*diag) ("malformed restriction at column %d.\n", pos + 1);
+	  diag (pos, "malformed restriction.\n");
 	  return 0;
 	}
       safe_inc_pos ();
@@ -905,7 +957,7 @@ match_const_restriction (typeinfo *typedata)
       char *x = match_integer ();
       if (x == NULL)
 	{
-	  (*diag) ("malformed integer at column %d.\n", oldpos + 1);
+	  diag (oldpos, "malformed integer.\n");
 	  return 0;
 	}
       consume_whitespace ();
@@ -918,7 +970,7 @@ match_const_restriction (typeinfo *typedata)
 	}
       else if (linebuf[pos] != ',')
 	{
-	  (*diag) ("malformed restriction at column %d.\n", pos + 1);
+	  diag (pos, "malformed restriction.\n");
 	  return 0;
 	}
       safe_inc_pos ();
@@ -926,7 +978,7 @@ match_const_restriction (typeinfo *typedata)
       char *y = match_integer ();
       if (y == NULL)
 	{
-	  (*diag) ("malformed integer at column %d.\n", oldpos + 1);
+	  diag (oldpos, "malformed integer.\n");
 	  return 0;
 	}
       typedata->restr = RES_RANGE;
@@ -936,7 +988,7 @@ match_const_restriction (typeinfo *typedata)
       consume_whitespace ();
       if (linebuf[pos] != '>')
 	{
-	  (*diag) ("malformed restriction at column %d.\n", pos + 1);
+	  diag (pos, "malformed restriction.\n");
 	  return 0;
 	}
       safe_inc_pos ();
@@ -1217,8 +1269,7 @@ match_type (typeinfo *typedata, int voidok)
 	return 1;
       if (typedata->base != BT_INT)
 	{
-	  (*diag)("'const' at %d requires pointer or integer type",
-		  oldpos + 1);
+	  diag (oldpos, "'const' requires pointer or integer type\n");
 	  return 0;
 	}
       consume_whitespace ();
@@ -1248,7 +1299,7 @@ parse_args (prototype *protoptr)
   consume_whitespace ();
   if (linebuf[pos] != '(')
     {
-      (*diag) ("missing '(' at column %d.\n", pos + 1);
+      diag (pos, "missing '('.\n");
       return PC_PARSEFAIL;
     }
   safe_inc_pos ();
@@ -1266,7 +1317,7 @@ parse_args (prototype *protoptr)
 	  {
 	    if (restr_cnt >= MAXRESTROPNDS)
 	      {
-		(*diag) ("More than two %d operands\n", MAXRESTROPNDS);
+		diag (pos, "More than two %d operands\n", MAXRESTROPNDS);
 		return PC_PARSEFAIL;
 	      }
 	    restr_opnd[restr_cnt] = *nargs + 1;
@@ -1283,20 +1334,20 @@ parse_args (prototype *protoptr)
 	  safe_inc_pos ();
 	else if (linebuf[pos] != ')')
 	  {
-	    (*diag) ("arg not followed by ',' or ')' at column %d.\n",
-		     pos + 1);
+	    diag (pos, "arg not followed by ',' or ')'.\n");
 	    return PC_PARSEFAIL;
 	  }
 
 #ifdef DEBUG
-	(*diag) ("argument type: isvoid = %d, isconst = %d, isvector = %d, "
-		 "issigned = %d, isunsigned = %d, isbool = %d, ispixel = %d, "
-		 "ispointer = %d, base = %d, restr = %d, val1 = \"%s\", "
-		 "val2 = \"%s\", pos = %d.\n",
-		 argtype->isvoid, argtype->isconst, argtype->isvector,
-		 argtype->issigned, argtype->isunsigned, argtype->isbool,
-		 argtype->ispixel, argtype->ispointer, argtype->base,
-		 argtype->restr, argtype->val1, argtype->val2, pos + 1);
+	diag (0,
+	      "argument type: isvoid = %d, isconst = %d, isvector = %d, "
+	      "issigned = %d, isunsigned = %d, isbool = %d, ispixel = %d, "
+	      "ispointer = %d, base = %d, restr = %d, val1 = \"%s\", "
+	      "val2 = \"%s\", pos = %d.\n",
+	      argtype->isvoid, argtype->isconst, argtype->isvector,
+	      argtype->issigned, argtype->isunsigned, argtype->isbool,
+	      argtype->ispixel, argtype->ispointer, argtype->base,
+	      argtype->restr, argtype->val1, argtype->val2, pos + 1);
 #endif
       }
     else
@@ -1306,7 +1357,7 @@ parse_args (prototype *protoptr)
 	pos = oldpos;
 	if (linebuf[pos] != ')')
 	  {
-	    (*diag) ("badly terminated arg list at column %d.\n", pos + 1);
+	    diag (pos, "badly terminated arg list.\n");
 	    return PC_PARSEFAIL;
 	  }
 	safe_inc_pos ();
@@ -1323,7 +1374,7 @@ parse_bif_attrs (attrinfo *attrptr)
   consume_whitespace ();
   if (linebuf[pos] != '{')
     {
-      (*diag) ("missing attribute set at column %d.\n", pos + 1);
+      diag (pos, "missing attribute set.\n");
       return PC_PARSEFAIL;
     }
   safe_inc_pos ();
@@ -1383,7 +1434,7 @@ parse_bif_attrs (attrinfo *attrptr)
 	  attrptr->isendian = 1;
 	else
 	  {
-	    (*diag) ("unknown attribute at column %d.\n", oldpos + 1);
+	    diag (oldpos, "unknown attribute.\n");
 	    return PC_PARSEFAIL;
 	  }
 
@@ -1392,8 +1443,7 @@ parse_bif_attrs (attrinfo *attrptr)
 	  safe_inc_pos ();
 	else if (linebuf[pos] != '}')
 	  {
-	    (*diag) ("arg not followed by ',' or '}' at column %d.\n",
-		     pos + 1);
+	    diag (pos, "arg not followed by ',' or '}'.\n");
 	    return PC_PARSEFAIL;
 	  }
       }
@@ -1402,7 +1452,7 @@ parse_bif_attrs (attrinfo *attrptr)
 	pos = oldpos;
 	if (linebuf[pos] != '}')
 	  {
-	    (*diag) ("badly terminated attr set at column %d.\n", pos + 1);
+	    diag (pos, "badly terminated attr set.\n");
 	    return PC_PARSEFAIL;
 	  }
 	safe_inc_pos ();
@@ -1410,18 +1460,19 @@ parse_bif_attrs (attrinfo *attrptr)
   } while (attrname);
 
 #ifdef DEBUG
-  (*diag) ("attribute set: init = %d, set = %d, extract = %d, nosoft = %d, "
-	   "ldvec = %d, stvec = %d, reve = %d, pred = %d, htm = %d, "
-	   "htmspr = %d, htmcr = %d, mma = %d, quad = %d, pair = %d, "
-	   "mmaint = %d, no32bit = %d, 32bit = %d, cpu = %d, ldstmask = %d, "
-	   "lxvrse = %d, lxvrze = %d, endian = %d.\n",
-	   attrptr->isinit, attrptr->isset, attrptr->isextract,
-	   attrptr->isnosoft, attrptr->isldvec, attrptr->isstvec,
-	   attrptr->isreve, attrptr->ispred, attrptr->ishtm, attrptr->ishtmspr,
-	   attrptr->ishtmcr, attrptr->ismma, attrptr->isquad, attrptr->ispair,
-	   attrptr->ismmaint, attrptr->isno32bit, attrptr->is32bit,
-	   attrptr->iscpu, attrptr->isldstmask, attrptr->islxvrse,
-	   attrptr->islxvrze, attrptr->isendian);
+  diag (0,
+	"attribute set: init = %d, set = %d, extract = %d, nosoft = %d, "
+	"ldvec = %d, stvec = %d, reve = %d, pred = %d, htm = %d, "
+	"htmspr = %d, htmcr = %d, mma = %d, quad = %d, pair = %d, "
+	"mmaint = %d, no32bit = %d, 32bit = %d, cpu = %d, ldstmask = %d, "
+	"lxvrse = %d, lxvrze = %d, endian = %d.\n",
+	attrptr->isinit, attrptr->isset, attrptr->isextract,
+	attrptr->isnosoft, attrptr->isldvec, attrptr->isstvec,
+	attrptr->isreve, attrptr->ispred, attrptr->ishtm, attrptr->ishtmspr,
+	attrptr->ishtmcr, attrptr->ismma, attrptr->isquad, attrptr->ispair,
+	attrptr->ismmaint, attrptr->isno32bit, attrptr->is32bit,
+	attrptr->iscpu, attrptr->isldstmask, attrptr->islxvrse,
+	attrptr->islxvrze, attrptr->isendian);
 #endif
 
   return PC_OK;
@@ -1483,7 +1534,7 @@ complete_vector_type (typeinfo *typeptr, char *buf, int *bufi)
       *bufi += 4;
       break;
     default:
-      (*diag) ("unhandled basetype %d.\n", typeptr->base);
+      diag (pos, "unhandled basetype %d.\n", typeptr->base);
       exit (1);
     }
 }
@@ -1543,7 +1594,7 @@ complete_base_type (typeinfo *typeptr, char *buf, int *bufi)
       memcpy (&buf[*bufi], "if", 2);
       break;
     default:
-      (*diag) ("unhandled basetype %d.\n", typeptr->base);
+      diag (pos, "unhandled basetype %d.\n", typeptr->base);
       exit (1);
     }
 
@@ -1664,20 +1715,20 @@ parse_prototype (prototype *protoptr)
   int success = match_type (ret_type, VOID_OK);
   if (!success)
     {
-      (*diag) ("missing or badly formed return type at column %d.\n",
-	       oldpos + 1);
+      diag (oldpos, "missing or badly formed return type.\n");
       return PC_PARSEFAIL;
     }
 
 #ifdef DEBUG
-  (*diag) ("return type: isvoid = %d, isconst = %d, isvector = %d, "
-	   "issigned = %d, isunsigned = %d, isbool = %d, ispixel = %d, "
-	   "ispointer = %d, base = %d, restr = %d, val1 = \"%s\", "
-	   "val2 = \"%s\", pos = %d.\n",
-	   ret_type->isvoid, ret_type->isconst, ret_type->isvector,
-	   ret_type->issigned, ret_type->isunsigned, ret_type->isbool,
-	   ret_type->ispixel, ret_type->ispointer, ret_type->base,
-	   ret_type->restr, ret_type->val1, ret_type->val2, pos + 1);
+  diag (0,
+	"return type: isvoid = %d, isconst = %d, isvector = %d, "
+	"issigned = %d, isunsigned = %d, isbool = %d, ispixel = %d, "
+	"ispointer = %d, base = %d, restr = %d, val1 = \"%s\", "
+	"val2 = \"%s\", pos = %d.\n",
+	ret_type->isvoid, ret_type->isconst, ret_type->isvector,
+	ret_type->issigned, ret_type->isunsigned, ret_type->isbool,
+	ret_type->ispixel, ret_type->ispointer, ret_type->base,
+	ret_type->restr, ret_type->val1, ret_type->val2, pos + 1);
 #endif
 
   /* Get the bif name.  */
@@ -1686,12 +1737,12 @@ parse_prototype (prototype *protoptr)
   *bifname = match_identifier ();
   if (!*bifname)
     {
-      (*diag) ("missing function name at column %d.\n", oldpos + 1);
+      diag (oldpos, "missing function name.\n");
       return PC_PARSEFAIL;
     }
 
 #ifdef DEBUG
-  (*diag) ("function name is '%s'.\n", *bifname);
+  diag (0, "function name is '%s'.\n", *bifname);
 #endif
 
   /* Process arguments.  */
@@ -1702,14 +1753,14 @@ parse_prototype (prototype *protoptr)
   consume_whitespace ();
   if (linebuf[pos] != ';')
     {
-      (*diag) ("missing semicolon at column %d.\n", pos + 1);
+      diag (pos, "missing semicolon.\n");
       return PC_PARSEFAIL;
     }
   safe_inc_pos ();
   consume_whitespace ();
   if (linebuf[pos] != '\n')
     {
-      (*diag) ("garbage at end of line at column %d.\n", pos + 1);
+      diag (pos, "garbage at end of line.\n");
       return PC_PARSEFAIL;
     }
 
@@ -1729,7 +1780,7 @@ parse_bif_entry (void)
   /* Allocate an entry in the bif table.  */
   if (num_bifs >= MAXBIFS - 1)
     {
-      (*diag) ("too many built-in functions.\n");
+      diag (pos, "too many built-in functions.\n");
       return PC_PARSEFAIL;
     }
 
@@ -1742,7 +1793,7 @@ parse_bif_entry (void)
   char *token = match_identifier ();
   if (!token)
     {
-      (*diag) ("malformed entry at column %d\n", oldpos + 1);
+      diag (oldpos, "malformed entry.\n");
       return PC_PARSEFAIL;
     }
 
@@ -1769,7 +1820,7 @@ parse_bif_entry (void)
   /* Now process line 2.  First up is the builtin id.  */
   if (!advance_line (bif_file))
     {
-      (*diag) ("unexpected EOF.\n");
+      diag (pos, "unexpected EOF.\n");
       return PC_PARSEFAIL;
     }
 
@@ -1779,19 +1830,18 @@ parse_bif_entry (void)
   bifs[curr_bif].idname = match_identifier ();
   if (!bifs[curr_bif].idname)
     {
-      (*diag) ("missing builtin id at column %d.\n", pos + 1);
+      diag (pos, "missing builtin id.\n");
       return PC_PARSEFAIL;
     }
 
 #ifdef DEBUG
-  (*diag) ("ID name is '%s'.\n", bifs[curr_bif].idname);
+  diag (0, "ID name is '%s'.\n", bifs[curr_bif].idname);
 #endif
 
   /* Save the ID in a lookup structure.  */
   if (!rbt_insert (&bif_rbt, bifs[curr_bif].idname))
     {
-      (*diag) ("duplicate function ID '%s' at column %d.\n",
-	       bifs[curr_bif].idname, oldpos + 1);
+      diag (oldpos, "duplicate function ID '%s'.\n", bifs[curr_bif].idname);
       return PC_PARSEFAIL;
     }
 
@@ -1804,7 +1854,7 @@ parse_bif_entry (void)
 
   if (!rbt_insert (&bifo_rbt, buf))
     {
-      (*diag) ("internal error inserting '%s' in bifo_rbt\n", buf);
+      diag (pos, "internal error inserting '%s' in bifo_rbt\n", buf);
       return PC_PARSEFAIL;
     }
 
@@ -1813,12 +1863,12 @@ parse_bif_entry (void)
   bifs[curr_bif].patname = match_identifier ();
   if (!bifs[curr_bif].patname)
     {
-      (*diag) ("missing pattern name at column %d.\n", pos + 1);
+      diag (pos, "missing pattern name.\n");
       return PC_PARSEFAIL;
     }
 
 #ifdef DEBUG
-  (*diag) ("pattern name is '%s'.\n", bifs[curr_bif].patname);
+  diag (0, "pattern name is '%s'.\n", bifs[curr_bif].patname);
 #endif
 
   /* Process attributes.  */
@@ -1836,7 +1886,7 @@ parse_bif_stanza (void)
 
   if (linebuf[pos] != '[')
     {
-      (*diag) ("ill-formed stanza header at column %d.\n", pos + 1);
+      diag (pos, "ill-formed stanza header.\n");
       return PC_PARSEFAIL;
     }
   safe_inc_pos ();
@@ -1844,7 +1894,7 @@ parse_bif_stanza (void)
   const char *stanza_name = match_to_right_bracket ();
   if (!stanza_name)
     {
-      (*diag) ("no expression found in stanza header.\n");
+      diag (pos, "no expression found in stanza header.\n");
       return PC_PARSEFAIL;
     }
 
@@ -1852,7 +1902,7 @@ parse_bif_stanza (void)
 
   if (linebuf[pos] != ']')
     {
-      (*diag) ("ill-formed stanza header at column %d.\n", pos + 1);
+      diag (pos, "ill-formed stanza header.\n");
       return PC_PARSEFAIL;
     }
   safe_inc_pos ();
@@ -1860,7 +1910,7 @@ parse_bif_stanza (void)
   consume_whitespace ();
   if (linebuf[pos] != '\n' && pos != LINELEN - 1)
     {
-      (*diag) ("garbage after stanza header.\n");
+      diag (pos, "garbage after stanza header.\n");
       return PC_PARSEFAIL;
     }
 
@@ -1927,7 +1977,7 @@ parse_ovld_entry (void)
   /* Allocate an entry in the overload table.  */
   if (num_ovlds >= MAXOVLDS - 1)
     {
-      (*diag) ("too many overloads.\n");
+      diag (pos, "too many overloads.\n");
       return PC_PARSEFAIL;
     }
 
@@ -1948,7 +1998,7 @@ parse_ovld_entry (void)
      optional overload id.  */
   if (!advance_line (ovld_file))
     {
-      (*diag) ("unexpected EOF.\n");
+      diag (0, "unexpected EOF.\n");
       return PC_EOFILE;
     }
 
@@ -1960,18 +2010,18 @@ parse_ovld_entry (void)
   ovlds[curr_ovld].ovld_id_name = id;
   if (!id)
     {
-      (*diag) ("missing overload id at column %d.\n", pos + 1);
+      diag (pos, "missing overload id.\n");
       return PC_PARSEFAIL;
     }
 
 #ifdef DEBUG
-  (*diag) ("ID name is '%s'.\n", id);
+  diag (pos, "ID name is '%s'.\n", id);
 #endif
 
   /* The builtin id has to match one from the bif file.  */
   if (!rbt_find (&bif_rbt, id))
     {
-      (*diag) ("builtin ID '%s' not found in bif file.\n", id);
+      diag (pos, "builtin ID '%s' not found in bif file.\n", id);
       return PC_PARSEFAIL;
     }
 
@@ -1989,13 +2039,13 @@ parse_ovld_entry (void)
  /* Save the overload ID in a lookup structure.  */
   if (!rbt_insert (&ovld_rbt, id))
     {
-      (*diag) ("duplicate overload ID '%s' at column %d.\n", id, oldpos + 1);
+      diag (oldpos, "duplicate overload ID '%s'.\n", id);
       return PC_PARSEFAIL;
     }
 
   if (linebuf[pos] != '\n')
     {
-      (*diag) ("garbage at end of line at column %d.\n", pos + 1);
+      diag (pos, "garbage at end of line.\n");
       return PC_PARSEFAIL;
     }
   return PC_OK;
@@ -2012,7 +2062,7 @@ parse_ovld_stanza (void)
 
   if (linebuf[pos] != '[')
     {
-      (*diag) ("ill-formed stanza header at column %d.\n", pos + 1);
+      diag (pos, "ill-formed stanza header.\n");
       return PC_PARSEFAIL;
     }
   safe_inc_pos ();
@@ -2020,7 +2070,7 @@ parse_ovld_stanza (void)
   char *stanza_name = match_identifier ();
   if (!stanza_name)
     {
-      (*diag) ("no identifier found in stanza header.\n");
+      diag (pos, "no identifier found in stanza header.\n");
       return PC_PARSEFAIL;
     }
 
@@ -2028,7 +2078,7 @@ parse_ovld_stanza (void)
      with subsequent overload entries.  */
   if (num_ovld_stanzas >= MAXOVLDSTANZAS)
     {
-      (*diag) ("too many stanza headers.\n");
+      diag (pos, "too many stanza headers.\n");
       return PC_PARSEFAIL;
     }
 
@@ -2039,7 +2089,7 @@ parse_ovld_stanza (void)
   consume_whitespace ();
   if (linebuf[pos] != ',')
     {
-      (*diag) ("missing comma at column %d.\n", pos + 1);
+      diag (pos, "missing comma.\n");
       return PC_PARSEFAIL;
     }
   safe_inc_pos ();
@@ -2048,14 +2098,14 @@ parse_ovld_stanza (void)
   stanza->extern_name = match_identifier ();
   if (!stanza->extern_name)
     {
-      (*diag) ("missing external name at column %d.\n", pos + 1);
+      diag (pos, "missing external name.\n");
       return PC_PARSEFAIL;
     }
 
   consume_whitespace ();
   if (linebuf[pos] != ',')
     {
-      (*diag) ("missing comma at column %d.\n", pos + 1);
+      diag (pos, "missing comma.\n");
       return PC_PARSEFAIL;
     }
   safe_inc_pos ();
@@ -2064,7 +2114,7 @@ parse_ovld_stanza (void)
   stanza->intern_name = match_identifier ();
   if (!stanza->intern_name)
     {
-      (*diag) ("missing internal name at column %d.\n", pos + 1);
+      diag (pos, "missing internal name.\n");
       return PC_PARSEFAIL;
     }
 
@@ -2076,7 +2126,7 @@ parse_ovld_stanza (void)
       stanza->ifdef = match_identifier ();
       if (!stanza->ifdef)
 	{
-	  (*diag) ("missing ifdef token at column %d.\n", pos + 1);
+	  diag (pos, "missing ifdef token.\n");
 	  return PC_PARSEFAIL;
 	}
       consume_whitespace ();
@@ -2086,7 +2136,7 @@ parse_ovld_stanza (void)
 
   if (linebuf[pos] != ']')
     {
-      (*diag) ("ill-formed stanza header at column %d.\n", pos + 1);
+      diag (pos, "ill-formed stanza header.\n");
       return PC_PARSEFAIL;
     }
   safe_inc_pos ();
@@ -2094,7 +2144,7 @@ parse_ovld_stanza (void)
   consume_whitespace ();
   if (linebuf[pos] != '\n' && pos != LINELEN - 1)
     {
-      (*diag) ("garbage after stanza header.\n");
+      diag (pos, "garbage after stanza header.\n");
       return PC_PARSEFAIL;
     }
 
@@ -2943,6 +2993,10 @@ main (int argc, const char **argv)
       exit (1);
     }
 
+  /* Allocate some buffers.  */
+  for (int i = 0; i < MAXLINES; i++)
+    lines[i] = (char *) malloc (LINELEN);
+
   /* Initialize the balanced trees containing built-in function ids,
      overload function ids, and function type declaration ids.  */
   rbt_new (&bif_rbt);
diff --git a/gcc/config/rs6000/rs6000-overload.def b/gcc/config/rs6000/rs6000-overload.def
index 4f58331..531a4fc 100644
--- a/gcc/config/rs6000/rs6000-overload.def
+++ b/gcc/config/rs6000/rs6000-overload.def
@@ -1,5 +1,5 @@
 ; Overloaded built-in functions for PowerPC.
-; Copyright (C) 2020-21 Free Software Foundation, Inc.
+; Copyright (C) 2020-2021 Free Software Foundation, Inc.
 ; Contributed by Bill Schmidt, IBM <wschmidt@linux.ibm.com>
 ;
 ; This file is part of GCC.
diff --git a/gcc/config/rs6000/rs6000.c b/gcc/config/rs6000/rs6000.c
index bac959f..e4843eb 100644
--- a/gcc/config/rs6000/rs6000.c
+++ b/gcc/config/rs6000/rs6000.c
@@ -1452,14 +1452,8 @@ static const struct attribute_spec rs6000_attribute_table[] =
 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
   rs6000_preferred_simd_mode
-#undef TARGET_VECTORIZE_INIT_COST
-#define TARGET_VECTORIZE_INIT_COST rs6000_init_cost
-#undef TARGET_VECTORIZE_ADD_STMT_COST
-#define TARGET_VECTORIZE_ADD_STMT_COST rs6000_add_stmt_cost
-#undef TARGET_VECTORIZE_FINISH_COST
-#define TARGET_VECTORIZE_FINISH_COST rs6000_finish_cost
-#undef TARGET_VECTORIZE_DESTROY_COST_DATA
-#define TARGET_VECTORIZE_DESTROY_COST_DATA rs6000_destroy_cost_data
+#undef TARGET_VECTORIZE_CREATE_COSTS
+#define TARGET_VECTORIZE_CREATE_COSTS rs6000_vectorize_create_costs
 
 #undef TARGET_LOOP_UNROLL_ADJUST
 #define TARGET_LOOP_UNROLL_ADJUST rs6000_loop_unroll_adjust
@@ -5146,7 +5140,8 @@ rs6000_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
 	if (TARGET_VSX && TARGET_ALLOW_MOVMISALIGN)
 	  {
 	    elements = TYPE_VECTOR_SUBPARTS (vectype);
-	    if (elements == 2)
+	    /* See PR102767, consider V1TI to keep consistency.  */
+	    if (elements == 2 || elements == 1)
 	      /* Double word aligned.  */
 	      return 4;
 
@@ -5183,39 +5178,40 @@ rs6000_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
 	if (TARGET_EFFICIENT_UNALIGNED_VSX)
 	  return 1;
 
-        if (TARGET_VSX && TARGET_ALLOW_MOVMISALIGN)
-          {
-            elements = TYPE_VECTOR_SUBPARTS (vectype);
-            if (elements == 2)
-              /* Double word aligned.  */
-              return 2;
+	if (TARGET_VSX && TARGET_ALLOW_MOVMISALIGN)
+	  {
+	    elements = TYPE_VECTOR_SUBPARTS (vectype);
+	    /* See PR102767, consider V1TI to keep consistency.  */
+	    if (elements == 2 || elements == 1)
+	      /* Double word aligned.  */
+	      return 2;
 
-            if (elements == 4)
-              {
-                switch (misalign)
-                  {
-                    case 8:
-                      /* Double word aligned.  */
-                      return 2;
-
-                    case -1:
-                      /* Unknown misalignment.  */
-                    case 4:
-                    case 12:
-                      /* Word aligned.  */
-                      return 23;
-
-                    default:
-                      gcc_unreachable ();
-                  }
-              }
-          }
+	    if (elements == 4)
+	      {
+		switch (misalign)
+		  {
+		  case 8:
+		    /* Double word aligned.  */
+		    return 2;
+
+		  case -1:
+		    /* Unknown misalignment.  */
+		  case 4:
+		  case 12:
+		    /* Word aligned.  */
+		    return 23;
+
+		  default:
+		    gcc_unreachable ();
+		  }
+	      }
+	  }
 
-        if (TARGET_ALTIVEC)
-          /* Misaligned stores are not supported.  */
-          gcc_unreachable ();
+	if (TARGET_ALTIVEC)
+	  /* Misaligned stores are not supported.  */
+	  gcc_unreachable ();
 
-        return 2;
+	return 2;
 
       case vec_construct:
 	/* This is a rough approximation assuming non-constant elements
@@ -5263,21 +5259,33 @@ rs6000_preferred_simd_mode (scalar_mode mode)
   return word_mode;
 }
 
-struct rs6000_cost_data
+class rs6000_cost_data : public vector_costs
 {
-  struct loop *loop_info;
-  unsigned cost[3];
+public:
+  using vector_costs::vector_costs;
+
+  unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
+			      stmt_vec_info stmt_info, tree vectype,
+			      int misalign,
+			      vect_cost_model_location where) override;
+  void finish_cost (const vector_costs *) override;
+
+protected:
+  void update_target_cost_per_stmt (vect_cost_for_stmt, stmt_vec_info,
+				    vect_cost_model_location, int,
+				    unsigned int);
+  void density_test (loop_vec_info);
+  void adjust_vect_cost_per_loop (loop_vec_info);
+
   /* Total number of vectorized stmts (loop only).  */
-  unsigned nstmts;
+  unsigned m_nstmts = 0;
   /* Total number of loads (loop only).  */
-  unsigned nloads;
+  unsigned m_nloads = 0;
   /* Possible extra penalized cost on vector construction (loop only).  */
-  unsigned extra_ctor_cost;
+  unsigned m_extra_ctor_cost = 0;
   /* For each vectorized loop, this var holds TRUE iff a non-memory vector
      instruction is needed by the vectorization.  */
-  bool vect_nonmem;
-  /* Indicates this is costing for the scalar version of a loop or block.  */
-  bool costing_for_scalar;
+  bool m_vect_nonmem = false;
 };
 
 /* Test for likely overcommitment of vector hardware resources.  If a
@@ -5286,20 +5294,19 @@ struct rs6000_cost_data
    adequately reflect delays from unavailable vector resources.
    Penalize the loop body cost for this case.  */
 
-static void
-rs6000_density_test (rs6000_cost_data *data)
+void
+rs6000_cost_data::density_test (loop_vec_info loop_vinfo)
 {
   /* This density test only cares about the cost of vector version of the
      loop, so immediately return if we are passed costing for the scalar
      version (namely computing single scalar iteration cost).  */
-  if (data->costing_for_scalar)
+  if (m_costing_for_scalar)
     return;
 
-  struct loop *loop = data->loop_info;
+  struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   basic_block *bbs = get_loop_body (loop);
   int nbbs = loop->num_nodes;
-  loop_vec_info loop_vinfo = loop_vec_info_for_loop (data->loop_info);
-  int vec_cost = data->cost[vect_body], not_vec_cost = 0;
+  int vec_cost = m_costs[vect_body], not_vec_cost = 0;
 
   for (int i = 0; i < nbbs; i++)
     {
@@ -5326,7 +5333,7 @@ rs6000_density_test (rs6000_cost_data *data)
   if (density_pct > rs6000_density_pct_threshold
       && vec_cost + not_vec_cost > rs6000_density_size_threshold)
     {
-      data->cost[vect_body] = vec_cost * (100 + rs6000_density_penalty) / 100;
+      m_costs[vect_body] = vec_cost * (100 + rs6000_density_penalty) / 100;
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_NOTE, vect_location,
 			 "density %d%%, cost %d exceeds threshold, penalizing "
@@ -5336,10 +5343,10 @@ rs6000_density_test (rs6000_cost_data *data)
 
   /* Check whether we need to penalize the body cost to account
      for excess strided or elementwise loads.  */
-  if (data->extra_ctor_cost > 0)
+  if (m_extra_ctor_cost > 0)
     {
-      gcc_assert (data->nloads <= data->nstmts);
-      unsigned int load_pct = (data->nloads * 100) / data->nstmts;
+      gcc_assert (m_nloads <= m_nstmts);
+      unsigned int load_pct = (m_nloads * 100) / m_nstmts;
 
       /* It's likely to be bounded by latency and execution resources
 	 from many scalar loads which are strided or elementwise loads
@@ -5351,10 +5358,10 @@ rs6000_density_test (rs6000_cost_data *data)
 	      the loads.
 	 One typical case is the innermost loop of the hotspot of SPEC2017
 	 503.bwaves_r without loop interchange.  */
-      if (data->nloads > (unsigned int) rs6000_density_load_num_threshold
+      if (m_nloads > (unsigned int) rs6000_density_load_num_threshold
 	  && load_pct > (unsigned int) rs6000_density_load_pct_threshold)
 	{
-	  data->cost[vect_body] += data->extra_ctor_cost;
+	  m_costs[vect_body] += m_extra_ctor_cost;
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location,
 			     "Found %u loads and "
@@ -5363,28 +5370,18 @@ rs6000_density_test (rs6000_cost_data *data)
 			     "penalizing loop body "
 			     "cost by extra cost %u "
 			     "for ctor.\n",
-			     data->nloads, load_pct,
-			     data->extra_ctor_cost);
+			     m_nloads, load_pct,
+			     m_extra_ctor_cost);
 	}
     }
 }
 
-/* Implement targetm.vectorize.init_cost.  */
+/* Implement targetm.vectorize.create_costs.  */
 
-static void *
-rs6000_init_cost (struct loop *loop_info, bool costing_for_scalar)
+static vector_costs *
+rs6000_vectorize_create_costs (vec_info *vinfo, bool costing_for_scalar)
 {
-  rs6000_cost_data *data = XNEW (rs6000_cost_data);
-  data->loop_info = loop_info;
-  data->cost[vect_prologue] = 0;
-  data->cost[vect_body]     = 0;
-  data->cost[vect_epilogue] = 0;
-  data->vect_nonmem = false;
-  data->nstmts = 0;
-  data->nloads = 0;
-  data->extra_ctor_cost = 0;
-  data->costing_for_scalar = costing_for_scalar;
-  return data;
+  return new rs6000_cost_data (vinfo, costing_for_scalar);
 }
 
 /* Adjust vectorization cost after calling rs6000_builtin_vectorization_cost.
@@ -5413,13 +5410,12 @@ rs6000_adjust_vect_cost_per_stmt (enum vect_cost_for_stmt kind,
 /* Helper function for add_stmt_cost.  Check each statement cost
    entry, gather information and update the target_cost fields
    accordingly.  */
-static void
-rs6000_update_target_cost_per_stmt (rs6000_cost_data *data,
-				    enum vect_cost_for_stmt kind,
-				    struct _stmt_vec_info *stmt_info,
-				    enum vect_cost_model_location where,
-				    int stmt_cost,
-				    unsigned int orig_count)
+void
+rs6000_cost_data::update_target_cost_per_stmt (vect_cost_for_stmt kind,
+					       stmt_vec_info stmt_info,
+					       vect_cost_model_location where,
+					       int stmt_cost,
+					       unsigned int orig_count)
 {
 
   /* Check whether we're doing something other than just a copy loop.
@@ -5431,17 +5427,19 @@ rs6000_update_target_cost_per_stmt (rs6000_cost_data *data,
       || kind == vec_construct
       || kind == scalar_to_vec
       || (where == vect_body && kind == vector_stmt))
-    data->vect_nonmem = true;
+    m_vect_nonmem = true;
 
   /* Gather some information when we are costing the vectorized instruction
      for the statements located in a loop body.  */
-  if (!data->costing_for_scalar && data->loop_info && where == vect_body)
+  if (!m_costing_for_scalar
+      && is_a<loop_vec_info> (m_vinfo)
+      && where == vect_body)
     {
-      data->nstmts += orig_count;
+      m_nstmts += orig_count;
 
       if (kind == scalar_load || kind == vector_load
 	  || kind == unaligned_load || kind == vector_gather_load)
-	data->nloads += orig_count;
+	m_nloads += orig_count;
 
       /* Power processors do not currently have instructions for strided
 	 and elementwise loads, and instead we must generate multiple
@@ -5469,20 +5467,16 @@ rs6000_update_target_cost_per_stmt (rs6000_cost_data *data,
 	  const unsigned int MAX_PENALIZED_COST_FOR_CTOR = 12;
 	  if (extra_cost > MAX_PENALIZED_COST_FOR_CTOR)
 	    extra_cost = MAX_PENALIZED_COST_FOR_CTOR;
-	  data->extra_ctor_cost += extra_cost;
+	  m_extra_ctor_cost += extra_cost;
 	}
     }
 }
 
-/* Implement targetm.vectorize.add_stmt_cost.  */
-
-static unsigned
-rs6000_add_stmt_cost (class vec_info *vinfo, void *data, int count,
-		      enum vect_cost_for_stmt kind,
-		      struct _stmt_vec_info *stmt_info, tree vectype,
-		      int misalign, enum vect_cost_model_location where)
+unsigned
+rs6000_cost_data::add_stmt_cost (int count, vect_cost_for_stmt kind,
+				 stmt_vec_info stmt_info, tree vectype,
+				 int misalign, vect_cost_model_location where)
 {
-  rs6000_cost_data *cost_data = (rs6000_cost_data*) data;
   unsigned retval = 0;
 
   if (flag_vect_cost_model)
@@ -5494,19 +5488,11 @@ rs6000_add_stmt_cost (class vec_info *vinfo, void *data, int count,
 	 vectorized are weighted more heavily.  The value here is
 	 arbitrary and could potentially be improved with analysis.  */
       unsigned int orig_count = count;
-      if (where == vect_body && stmt_info
-	  && stmt_in_inner_loop_p (vinfo, stmt_info))
-	{
-	  loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
-	  gcc_assert (loop_vinfo);
-	  count *= LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo); /* FIXME.  */
-	}
+      retval = adjust_cost_for_freq (stmt_info, where, count * stmt_cost);
+      m_costs[where] += retval;
 
-      retval = (unsigned) (count * stmt_cost);
-      cost_data->cost[where] += retval;
-
-      rs6000_update_target_cost_per_stmt (cost_data, kind, stmt_info, where,
-					  stmt_cost, orig_count);
+      update_target_cost_per_stmt (kind, stmt_info, where,
+				   stmt_cost, orig_count);
     }
 
   return retval;
@@ -5518,13 +5504,9 @@ rs6000_add_stmt_cost (class vec_info *vinfo, void *data, int count,
    vector with length by counting number of required lengths under condition
    LOOP_VINFO_FULLY_WITH_LENGTH_P.  */
 
-static void
-rs6000_adjust_vect_cost_per_loop (rs6000_cost_data *data)
+void
+rs6000_cost_data::adjust_vect_cost_per_loop (loop_vec_info loop_vinfo)
 {
-  struct loop *loop = data->loop_info;
-  gcc_assert (loop);
-  loop_vec_info loop_vinfo = loop_vec_info_for_loop (loop);
-
   if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
     {
       rgroup_controls *rgc;
@@ -5535,49 +5517,29 @@ rs6000_adjust_vect_cost_per_loop (rs6000_cost_data *data)
 	  /* Each length needs one shift to fill into bits 0-7.  */
 	  shift_cnt += num_vectors_m1 + 1;
 
-      rs6000_add_stmt_cost (loop_vinfo, (void *) data, shift_cnt, scalar_stmt,
-			    NULL, NULL_TREE, 0, vect_body);
+      add_stmt_cost (shift_cnt, scalar_stmt, NULL, NULL_TREE, 0, vect_body);
     }
 }
 
-/* Implement targetm.vectorize.finish_cost.  */
-
-static void
-rs6000_finish_cost (void *data, unsigned *prologue_cost,
-		    unsigned *body_cost, unsigned *epilogue_cost)
+void
+rs6000_cost_data::finish_cost (const vector_costs *scalar_costs)
 {
-  rs6000_cost_data *cost_data = (rs6000_cost_data*) data;
-
-  if (cost_data->loop_info)
+  if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo))
     {
-      rs6000_adjust_vect_cost_per_loop (cost_data);
-      rs6000_density_test (cost_data);
-    }
+      adjust_vect_cost_per_loop (loop_vinfo);
+      density_test (loop_vinfo);
 
-  /* Don't vectorize minimum-vectorization-factor, simple copy loops
-     that require versioning for any reason.  The vectorization is at
-     best a wash inside the loop, and the versioning checks make
-     profitability highly unlikely and potentially quite harmful.  */
-  if (cost_data->loop_info)
-    {
-      loop_vec_info vec_info = loop_vec_info_for_loop (cost_data->loop_info);
-      if (!cost_data->vect_nonmem
-	  && LOOP_VINFO_VECT_FACTOR (vec_info) == 2
-	  && LOOP_REQUIRES_VERSIONING (vec_info))
-	cost_data->cost[vect_body] += 10000;
+      /* Don't vectorize minimum-vectorization-factor, simple copy loops
+	 that require versioning for any reason.  The vectorization is at
+	 best a wash inside the loop, and the versioning checks make
+	 profitability highly unlikely and potentially quite harmful.  */
+      if (!m_vect_nonmem
+	  && LOOP_VINFO_VECT_FACTOR (loop_vinfo) == 2
+	  && LOOP_REQUIRES_VERSIONING (loop_vinfo))
+	m_costs[vect_body] += 10000;
     }
 
-  *prologue_cost = cost_data->cost[vect_prologue];
-  *body_cost     = cost_data->cost[vect_body];
-  *epilogue_cost = cost_data->cost[vect_epilogue];
-}
-
-/* Implement targetm.vectorize.destroy_cost_data.  */
-
-static void
-rs6000_destroy_cost_data (void *data)
-{
-  free (data);
+  vector_costs::finish_cost (scalar_costs);
 }
 
 /* Implement targetm.loop_unroll_adjust.  */
@@ -16033,9 +15995,7 @@ rs6000_emit_vector_cond_expr (rtx dest, rtx op_true, rtx op_false,
   machine_mode dest_mode = GET_MODE (dest);
   machine_mode mask_mode = GET_MODE (cc_op0);
   enum rtx_code rcode = GET_CODE (cond);
-  machine_mode cc_mode = CCmode;
   rtx mask;
-  rtx cond2;
   bool invert_move = false;
 
   if (VECTOR_UNIT_NONE_P (dest_mode))
@@ -16075,8 +16035,6 @@ rs6000_emit_vector_cond_expr (rtx dest, rtx op_true, rtx op_false,
     case GEU:
     case LTU:
     case LEU:
-      /* Mark unsigned tests with CCUNSmode.  */
-      cc_mode = CCUNSmode;
 
       /* Invert condition to avoid compound test if necessary.  */
       if (rcode == GEU || rcode == LEU)
@@ -16096,6 +16054,9 @@ rs6000_emit_vector_cond_expr (rtx dest, rtx op_true, rtx op_false,
   if (!mask)
     return 0;
 
+  if (mask_mode != dest_mode)
+    mask = simplify_gen_subreg (dest_mode, mask, mask_mode, 0);
+
   if (invert_move)
     std::swap (op_true, op_false);
 
@@ -16135,13 +16096,11 @@ rs6000_emit_vector_cond_expr (rtx dest, rtx op_true, rtx op_false,
   if (!REG_P (op_false) && !SUBREG_P (op_false))
     op_false = force_reg (dest_mode, op_false);
 
-  cond2 = gen_rtx_fmt_ee (NE, cc_mode, gen_lowpart (dest_mode, mask),
-			  CONST0_RTX (dest_mode));
-  emit_insn (gen_rtx_SET (dest,
-			  gen_rtx_IF_THEN_ELSE (dest_mode,
-						cond2,
-						op_true,
-						op_false)));
+  rtx tmp = gen_rtx_IOR (dest_mode,
+			 gen_rtx_AND (dest_mode, gen_rtx_NOT (dest_mode, mask),
+				      op_false),
+			 gen_rtx_AND (dest_mode, mask, op_true));
+  emit_insn (gen_rtx_SET (dest, tmp));
   return 1;
 }
 
@@ -22784,12 +22743,16 @@ rs6000_builtin_reciprocal (tree fndecl)
       if (!RS6000_RECIP_AUTO_RSQRTE_P (V2DFmode))
 	return NULL_TREE;
 
+      if (new_builtins_are_live)
+	return rs6000_builtin_decls_x[RS6000_BIF_RSQRT_2DF];
       return rs6000_builtin_decls[VSX_BUILTIN_RSQRT_2DF];
 
     case VSX_BUILTIN_XVSQRTSP:
       if (!RS6000_RECIP_AUTO_RSQRTE_P (V4SFmode))
 	return NULL_TREE;
 
+      if (new_builtins_are_live)
+	return rs6000_builtin_decls_x[RS6000_BIF_RSQRT_4SF];
       return rs6000_builtin_decls[VSX_BUILTIN_RSQRT_4SF];
 
     default:
@@ -23350,7 +23313,15 @@ altivec_expand_vec_perm_const (rtx target, rtx op0, rtx op1,
     {OPTION_MASK_P8_VECTOR,
      BYTES_BIG_ENDIAN ? CODE_FOR_p8_vmrgow_v4sf_direct
 		      : CODE_FOR_p8_vmrgew_v4sf_direct,
-     {4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31}}};
+     {4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31}},
+    {OPTION_MASK_VSX, CODE_FOR_vsx_xxpermdi_v16qi,
+     {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23}},
+    {OPTION_MASK_VSX, CODE_FOR_vsx_xxpermdi_v16qi,
+     {8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}},
+    {OPTION_MASK_VSX, CODE_FOR_vsx_xxpermdi_v16qi,
+     {0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31}},
+    {OPTION_MASK_VSX, CODE_FOR_vsx_xxpermdi_v16qi,
+     {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31}}};
 
   unsigned int i, j, elt, which;
   unsigned char perm[16];
@@ -23473,6 +23444,27 @@ altivec_expand_vec_perm_const (rtx target, rtx op0, rtx op1,
 	  machine_mode omode = insn_data[icode].operand[0].mode;
 	  machine_mode imode = insn_data[icode].operand[1].mode;
 
+	  rtx perm_idx = GEN_INT (0);
+	  if (icode == CODE_FOR_vsx_xxpermdi_v16qi)
+	    {
+	      int perm_val = 0;
+	      if (one_vec)
+		{
+		  if (perm[0] == 8)
+		    perm_val |= 2;
+		  if (perm[8] == 8)
+		    perm_val |= 1;
+		}
+	      else
+		{
+		  if (perm[0] != 0)
+		    perm_val |= 2;
+		  if (perm[8] != 16)
+		    perm_val |= 1;
+		}
+	      perm_idx = GEN_INT (perm_val);
+	    }
+
 	  /* For little-endian, don't use vpkuwum and vpkuhum if the
 	     underlying vector type is not V4SI and V8HI, respectively.
 	     For example, using vpkuwum with a V8HI picks up the even
@@ -23496,7 +23488,8 @@ altivec_expand_vec_perm_const (rtx target, rtx op0, rtx op1,
           /* For little-endian, the two input operands must be swapped
              (or swapped back) to ensure proper right-to-left numbering
              from 0 to 2N-1.  */
-	  if (swapped ^ !BYTES_BIG_ENDIAN)
+	  if (swapped ^ !BYTES_BIG_ENDIAN
+	      && icode != CODE_FOR_vsx_xxpermdi_v16qi)
 	    std::swap (op0, op1);
 	  if (imode != V16QImode)
 	    {
@@ -23507,7 +23500,10 @@ altivec_expand_vec_perm_const (rtx target, rtx op0, rtx op1,
 	    x = target;
 	  else
 	    x = gen_reg_rtx (omode);
-	  emit_insn (GEN_FCN (icode) (x, op0, op1));
+	  if (icode == CODE_FOR_vsx_xxpermdi_v16qi)
+	    emit_insn (GEN_FCN (icode) (x, op0, op1, perm_idx));
+	  else
+	    emit_insn (GEN_FCN (icode) (x, op0, op1));
 	  if (omode != V16QImode)
 	    emit_move_insn (target, gen_lowpart (V16QImode, x));
 	  return true;
@@ -25379,7 +25375,10 @@ add_condition_to_bb (tree function_decl, tree version_decl,
 
   tree bool_zero = build_int_cst (bool_int_type_node, 0);
   tree cond_var = create_tmp_var (bool_int_type_node);
-  tree predicate_decl = rs6000_builtin_decls [(int) RS6000_BUILTIN_CPU_SUPPORTS];
+  tree predicate_decl
+    = new_builtins_are_live
+	? rs6000_builtin_decls_x[(int) RS6000_BIF_CPU_SUPPORTS]
+	: rs6000_builtin_decls [(int) RS6000_BUILTIN_CPU_SUPPORTS];
   const char *arg_str = rs6000_clone_map[clone_isa].name;
   tree predicate_arg = build_string_literal (strlen (arg_str) + 1, arg_str);
   gimple *call_cond_stmt = gimple_build_call (predicate_decl, 1, predicate_arg);
@@ -28019,8 +28018,12 @@ rs6000_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update)
       return;
     }
 
-  tree mffs = rs6000_builtin_decls[RS6000_BUILTIN_MFFS];
-  tree mtfsf = rs6000_builtin_decls[RS6000_BUILTIN_MTFSF];
+  tree mffs
+    = new_builtins_are_live ? rs6000_builtin_decls_x[RS6000_BIF_MFFS]
+			    : rs6000_builtin_decls[RS6000_BUILTIN_MFFS];
+  tree mtfsf
+    = new_builtins_are_live ? rs6000_builtin_decls_x[RS6000_BIF_MTFSF]
+			    : rs6000_builtin_decls[RS6000_BUILTIN_MTFSF];
   tree call_mffs = build_call_expr (mffs, 0);
 
   /* Generates the equivalent of feholdexcept (&fenv_var)
diff --git a/gcc/config/rs6000/smmintrin.h b/gcc/config/rs6000/smmintrin.h
index b732fbc..0fab308 100644
--- a/gcc/config/rs6000/smmintrin.h
+++ b/gcc/config/rs6000/smmintrin.h
@@ -118,7 +118,7 @@ _mm_blendv_epi8 (__m128i __A, __m128i __B, __m128i __mask)
   return (__m128i) vec_sel ((__v16qu) __A, (__v16qu) __B, __lmask);
 }
 
-__inline __m128
+extern __inline __m128
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_blend_ps (__m128 __A, __m128 __B, const int __imm8)
 {
@@ -145,7 +145,7 @@ _mm_blend_ps (__m128 __A, __m128 __B, const int __imm8)
   return (__m128) __r;
 }
 
-__inline __m128
+extern __inline __m128
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_blendv_ps (__m128 __A, __m128 __B, __m128 __mask)
 {
@@ -154,7 +154,7 @@ _mm_blendv_ps (__m128 __A, __m128 __B, __m128 __mask)
   return (__m128) vec_sel ((__v4su) __A, (__v4su) __B, (__v4su) __boolmask);
 }
 
-__inline __m128d
+extern __inline __m128d
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_blend_pd (__m128d __A, __m128d __B, const int __imm8)
 {
@@ -170,7 +170,7 @@ _mm_blend_pd (__m128d __A, __m128d __B, const int __imm8)
 }
 
 #ifdef _ARCH_PWR8
-__inline __m128d
+extern __inline __m128d
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_blendv_pd (__m128d __A, __m128d __B, __m128d __mask)
 {
@@ -180,7 +180,7 @@ _mm_blendv_pd (__m128d __A, __m128d __B, __m128d __mask)
 }
 #endif
 
-__inline int
+extern __inline int
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_testz_si128 (__m128i __A, __m128i __B)
 {
@@ -189,7 +189,7 @@ _mm_testz_si128 (__m128i __A, __m128i __B)
   return vec_all_eq (vec_and ((__v16qu) __A, (__v16qu) __B), __zero);
 }
 
-__inline int
+extern __inline int
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_testc_si128 (__m128i __A, __m128i __B)
 {
@@ -199,7 +199,7 @@ _mm_testc_si128 (__m128i __A, __m128i __B)
   return vec_all_eq (vec_and ((__v16qu) __notA, (__v16qu) __B), __zero);
 }
 
-__inline int
+extern __inline int
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_testnzc_si128 (__m128i __A, __m128i __B)
 {
@@ -214,14 +214,14 @@ _mm_testnzc_si128 (__m128i __A, __m128i __B)
 
 #define _mm_test_mix_ones_zeros(M, V) _mm_testnzc_si128 ((M), (V))
 
-__inline __m128d
+extern __inline __m128d
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_ceil_pd (__m128d __A)
 {
   return (__m128d) vec_ceil ((__v2df) __A);
 }
 
-__inline __m128d
+extern __inline __m128d
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_ceil_sd (__m128d __A, __m128d __B)
 {
@@ -230,14 +230,14 @@ _mm_ceil_sd (__m128d __A, __m128d __B)
   return (__m128d) __r;
 }
 
-__inline __m128d
+extern __inline __m128d
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_floor_pd (__m128d __A)
 {
   return (__m128d) vec_floor ((__v2df) __A);
 }
 
-__inline __m128d
+extern __inline __m128d
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_floor_sd (__m128d __A, __m128d __B)
 {
@@ -246,14 +246,14 @@ _mm_floor_sd (__m128d __A, __m128d __B)
   return (__m128d) __r;
 }
 
-__inline __m128
+extern __inline __m128
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_ceil_ps (__m128 __A)
 {
   return (__m128) vec_ceil ((__v4sf) __A);
 }
 
-__inline __m128
+extern __inline __m128
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_ceil_ss (__m128 __A, __m128 __B)
 {
@@ -262,14 +262,14 @@ _mm_ceil_ss (__m128 __A, __m128 __B)
   return __r;
 }
 
-__inline __m128
+extern __inline __m128
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_floor_ps (__m128 __A)
 {
   return (__m128) vec_floor ((__v4sf) __A);
 }
 
-__inline __m128
+extern __inline __m128
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_floor_ss (__m128 __A, __m128 __B)
 {
@@ -351,7 +351,7 @@ _mm_mullo_epi32 (__m128i __X, __m128i __Y)
 }
 
 #ifdef _ARCH_PWR8
-__inline __m128i
+extern __inline __m128i
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_mul_epi32 (__m128i __X, __m128i __Y)
 {
@@ -359,7 +359,7 @@ _mm_mul_epi32 (__m128i __X, __m128i __Y)
 }
 #endif
 
-__inline __m128i
+extern __inline __m128i
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_cvtepi8_epi16 (__m128i __A)
 {
@@ -499,7 +499,7 @@ _mm_cvtepu32_epi64 (__m128i __A)
 
 /* Return horizontal packed word minimum and its index in bits [15:0]
    and bits [18:16] respectively.  */
-__inline __m128i
+extern __inline __m128i
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_minpos_epu16 (__m128i __A)
 {
@@ -524,7 +524,7 @@ _mm_minpos_epu16 (__m128i __A)
   return __r.__m;
 }
 
-__inline __m128i
+extern __inline __m128i
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_packus_epi32 (__m128i __X, __m128i __Y)
 {
@@ -532,7 +532,7 @@ _mm_packus_epi32 (__m128i __X, __m128i __Y)
 }
 
 #ifdef _ARCH_PWR8
-__inline __m128i
+extern __inline __m128i
 __attribute__ ((__gnu_inline__, __always_inline__, __artificial__))
 _mm_cmpgt_epi64 (__m128i __X, __m128i __Y)
 {
diff --git a/gcc/config/rs6000/vector.md b/gcc/config/rs6000/vector.md
index 7e36c788..062aef7 100644
--- a/gcc/config/rs6000/vector.md
+++ b/gcc/config/rs6000/vector.md
@@ -916,23 +916,21 @@
 ;; which is in the reverse order that we want
 (define_expand "vector_select_<mode>"
   [(set (match_operand:VEC_L 0 "vlogical_operand")
-	(if_then_else:VEC_L
-	 (ne:CC (match_operand:VEC_L 3 "vlogical_operand")
-		(match_dup 4))
-	 (match_operand:VEC_L 2 "vlogical_operand")
-	 (match_operand:VEC_L 1 "vlogical_operand")))]
-  "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
-  "operands[4] = CONST0_RTX (<MODE>mode);")
+	(ior:VEC_L
+	  (and:VEC_L (not:VEC_L (match_operand:VEC_L 3 "vlogical_operand"))
+		     (match_operand:VEC_L 1 "vlogical_operand"))
+	  (and:VEC_L (match_dup 3)
+		     (match_operand:VEC_L 2 "vlogical_operand"))))]
+  "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)")
 
 (define_expand "vector_select_<mode>_uns"
   [(set (match_operand:VEC_L 0 "vlogical_operand")
-	(if_then_else:VEC_L
-	 (ne:CCUNS (match_operand:VEC_L 3 "vlogical_operand")
-		   (match_dup 4))
-	 (match_operand:VEC_L 2 "vlogical_operand")
-	 (match_operand:VEC_L 1 "vlogical_operand")))]
-  "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
-  "operands[4] = CONST0_RTX (<MODE>mode);")
+	(ior:VEC_L
+	  (and:VEC_L (not:VEC_L (match_operand:VEC_L 3 "vlogical_operand"))
+		     (match_operand:VEC_L 1 "vlogical_operand"))
+	  (and:VEC_L (match_dup 3)
+		     (match_operand:VEC_L 2 "vlogical_operand"))))]
+  "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)")
 
 ;; Expansions that compare vectors producing a vector result and a predicate,
 ;; setting CR6 to indicate a combined status
diff --git a/gcc/config/rs6000/vsx.md b/gcc/config/rs6000/vsx.md
index 346f1a9..83d6c7b 100644
--- a/gcc/config/rs6000/vsx.md
+++ b/gcc/config/rs6000/vsx.md
@@ -2184,31 +2184,6 @@
   "xvcmpge<sd>p. %x0,%x1,%x2"
   [(set_attr "type" "<VStype_simple>")])
 
-;; Vector select
-(define_insn "*vsx_xxsel<mode>"
-  [(set (match_operand:VSX_L 0 "vsx_register_operand" "=<VSr>,?wa")
-	(if_then_else:VSX_L
-	 (ne:CC (match_operand:VSX_L 1 "vsx_register_operand" "<VSr>,wa")
-		(match_operand:VSX_L 4 "zero_constant" ""))
-	 (match_operand:VSX_L 2 "vsx_register_operand" "<VSr>,wa")
-	 (match_operand:VSX_L 3 "vsx_register_operand" "<VSr>,wa")))]
-  "VECTOR_MEM_VSX_P (<MODE>mode)"
-  "xxsel %x0,%x3,%x2,%x1"
-  [(set_attr "type" "vecmove")
-   (set_attr "isa" "<VSisa>")])
-
-(define_insn "*vsx_xxsel<mode>_uns"
-  [(set (match_operand:VSX_L 0 "vsx_register_operand" "=<VSr>,?wa")
-	(if_then_else:VSX_L
-	 (ne:CCUNS (match_operand:VSX_L 1 "vsx_register_operand" "<VSr>,wa")
-		   (match_operand:VSX_L 4 "zero_constant" ""))
-	 (match_operand:VSX_L 2 "vsx_register_operand" "<VSr>,wa")
-	 (match_operand:VSX_L 3 "vsx_register_operand" "<VSr>,wa")))]
-  "VECTOR_MEM_VSX_P (<MODE>mode)"
-  "xxsel %x0,%x3,%x2,%x1"
-  [(set_attr "type" "vecmove")
-   (set_attr "isa" "<VSisa>")])
-
 ;; Copy sign
 (define_insn "vsx_copysign<mode>3"
   [(set (match_operand:VSX_F 0 "vsx_register_operand" "=wa")
@@ -4590,7 +4565,7 @@
 
 ;; V4SI splat support
 (define_insn "vsx_splat_v4si"
-  [(set (match_operand:V4SI 0 "vsx_register_operand" "=we,we")
+  [(set (match_operand:V4SI 0 "vsx_register_operand" "=wa,wa")
 	(vec_duplicate:V4SI
 	 (match_operand:SI 1 "splat_input_operand" "r,Z")))]
   "TARGET_P9_VECTOR"
diff --git a/gcc/config/rs6000/x86gprintrin.h b/gcc/config/rs6000/x86gprintrin.h
new file mode 100644
index 0000000..57ef120
--- /dev/null
+++ b/gcc/config/rs6000/x86gprintrin.h
@@ -0,0 +1,31 @@
+/* Copyright (C) 2021 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef _X86GPRINTRIN_H_INCLUDED
+#define _X86GPRINTRIN_H_INCLUDED
+
+#include <bmiintrin.h>
+
+#include <bmi2intrin.h>
+
+#endif /* _X86GPRINTRIN_H_INCLUDED.  */
diff --git a/gcc/config/rs6000/x86intrin.h b/gcc/config/rs6000/x86intrin.h
index 6ad2bfc..e8aa922 100644
--- a/gcc/config/rs6000/x86intrin.h
+++ b/gcc/config/rs6000/x86intrin.h
@@ -36,15 +36,7 @@
 #define _X86INTRIN_H_INCLUDED
 
 #ifdef __ALTIVEC__
-#include <mmintrin.h>
-
-#include <xmmintrin.h>
-
-#include <emmintrin.h>
+#include <immintrin.h>
 #endif /* __ALTIVEC__ */
 
-#include <bmiintrin.h>
-
-#include <bmi2intrin.h>
-
 #endif /* _X86INTRIN_H_INCLUDED */
diff --git a/gcc/config/s390/s390.c b/gcc/config/s390/s390.c
index b2f2f64..510e7f5 100644
--- a/gcc/config/s390/s390.c
+++ b/gcc/config/s390/s390.c
@@ -15400,7 +15400,10 @@ s390_loop_unroll_adjust (unsigned nunroll, struct loop *loop)
 		  || (GET_CODE (SET_SRC (set)) == COMPARE
 		      && GET_MODE (XEXP (SET_SRC (set), 0)) == BLKmode
 		      && GET_MODE (XEXP (SET_SRC (set), 1)) == BLKmode)))
-	    return 1;
+	    {
+	      free (bbs);
+	      return 1;
+	    }
 
 	  FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
 	    if (MEM_P (*iter))
diff --git a/gcc/config/s390/s390.h b/gcc/config/s390/s390.h
index fb16a45..186c5c6 100644
--- a/gcc/config/s390/s390.h
+++ b/gcc/config/s390/s390.h
@@ -332,6 +332,11 @@ extern const char *s390_host_detect_local_cpu (int argc, const char **argv);
 
 #define STACK_SIZE_MODE (Pmode)
 
+/* Make the stack pointer to be moved downwards while issuing stack probes with
+   -fstack-check.  We need this to prevent memory below the stack pointer from
+   being accessed.  */
+#define STACK_CHECK_MOVING_SP 1
+
 #ifndef IN_LIBGCC2
 
 /* Width of a word, in units (bytes).  */
diff --git a/gcc/config/s390/s390.md b/gcc/config/s390/s390.md
index b8bdbae..4debdcd 100644
--- a/gcc/config/s390/s390.md
+++ b/gcc/config/s390/s390.md
@@ -3533,7 +3533,7 @@
   [(set (match_operand:SI 0 "nonimmediate_operand"     "=d")
         (unspec:SI [(match_operand 1 "register_operand" "0")]
                    UNSPEC_CC_TO_INT))]
-  "operands != NULL"
+  ""
   "#"
   "reload_completed"
   [(set (match_dup 0) (lshiftrt:SI (match_dup 0) (const_int 28)))])
@@ -12062,7 +12062,7 @@
 (define_expand "tabort"
   [(unspec_volatile [(match_operand:SI 0 "nonmemory_operand" "")]
 		    UNSPECV_TABORT)]
-  "TARGET_HTM && operands != NULL"
+  "TARGET_HTM"
 {
   if (CONST_INT_P (operands[0])
       && INTVAL (operands[0]) >= 0 && INTVAL (operands[0]) <= 255)
@@ -12076,7 +12076,7 @@
 (define_insn "*tabort_1"
   [(unspec_volatile [(match_operand:SI 0 "nonmemory_operand" "aJ")]
 		    UNSPECV_TABORT)]
-  "TARGET_HTM && operands != NULL"
+  "TARGET_HTM"
   "tabort\t%Y0"
   [(set_attr "op_type" "S")])
 
@@ -12084,8 +12084,7 @@
   [(unspec_volatile [(plus:SI (match_operand:SI 0 "register_operand"  "a")
 			      (match_operand:SI 1 "const_int_operand" "J"))]
 		    UNSPECV_TABORT)]
-  "TARGET_HTM && operands != NULL
-   && CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[1]), 'J', \"J\")"
+  "TARGET_HTM && CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[1]), 'J', \"J\")"
   "tabort\t%1(%0)"
   [(set_attr "op_type" "S")])
 
diff --git a/gcc/config/vax/vax.c b/gcc/config/vax/vax.c
index e26ab3b..cf77db3 100644
--- a/gcc/config/vax/vax.c
+++ b/gcc/config/vax/vax.c
@@ -62,6 +62,7 @@ static rtx vax_function_arg (cumulative_args_t, const function_arg_info &);
 static void vax_function_arg_advance (cumulative_args_t,
 				      const function_arg_info &);
 static rtx vax_struct_value_rtx (tree, int);
+static bool vax_lra_p (void);
 static void vax_asm_trampoline_template (FILE *);
 static void vax_trampoline_init (rtx, tree, rtx);
 static poly_int64 vax_return_pops_args (tree, tree, poly_int64);
@@ -114,7 +115,7 @@ static HOST_WIDE_INT vax_starting_frame_offset (void);
 #define TARGET_STRUCT_VALUE_RTX vax_struct_value_rtx
 
 #undef TARGET_LRA_P
-#define TARGET_LRA_P hook_bool_void_false
+#define TARGET_LRA_P vax_lra_p
 
 #undef TARGET_LEGITIMATE_ADDRESS_P
 #define TARGET_LEGITIMATE_ADDRESS_P vax_legitimate_address_p
@@ -1221,6 +1222,14 @@ vax_struct_value_rtx (tree fntype ATTRIBUTE_UNUSED,
   return gen_rtx_REG (Pmode, VAX_STRUCT_VALUE_REGNUM);
 }
 
+/* Return true if we use LRA instead of reload pass.  */
+
+static bool
+vax_lra_p (void)
+{
+  return TARGET_LRA;
+}
+
 /* Output integer move instructions.  */
 
 bool
diff --git a/gcc/config/vax/vax.h b/gcc/config/vax/vax.h
index b67d668..006c0ab 100644
--- a/gcc/config/vax/vax.h
+++ b/gcc/config/vax/vax.h
@@ -433,6 +433,7 @@ enum reg_class { NO_REGS, ALL_REGS, LIM_REG_CLASSES };
    move-instruction pairs, we will do a cpymem or libcall instead.  */
 #define MOVE_RATIO(speed) ((speed) ? 6 : 3)
 #define CLEAR_RATIO(speed) ((speed) ? 6 : 2)
+#define SET_RATIO(speed) ((speed) ? 6 : 2)
 
 /* Nonzero if access to memory by bytes is slow and undesirable.  */
 #define SLOW_BYTE_ACCESS 0
diff --git a/gcc/config/vax/vax.md b/gcc/config/vax/vax.md
index 5b1b392..977dc80 100644
--- a/gcc/config/vax/vax.md
+++ b/gcc/config/vax/vax.md
@@ -32,6 +32,12 @@
   VUNSPEC_PEM		    ; 'procedure_entry_mask' insn.
 ])
 
+;; UNSPEC usage:
+
+(define_c_enum "unspec" [
+  UNSPEC_SETMEM_FILL	    ; 'fill' operand to 'setmem' insn.
+])
+
 (define_constants
   [(VAX_AP_REGNUM 12)	    ; Register 12 contains the argument pointer
    (VAX_FP_REGNUM 13)	    ; Register 13 contains the frame pointer
@@ -438,6 +444,64 @@
    (clobber (reg:CC VAX_PSL_REGNUM))]
   "reload_completed"
   "movc3 %2,%1,%0")
+
+;; This is here to accept 4 arguments and pass the first 3 along
+;; to the setmemhi1 pattern that really does the work.
+(define_expand "setmemhi"
+  [(set (match_operand:BLK 0 "memory_operand" "")
+	(match_operand:QI 2 "general_operand" ""))
+   (use (match_operand:HI 1 "general_operand" ""))
+   (match_operand 3 "" "")]
+  ""
+  "
+{
+  emit_insn (gen_setmemhi1 (operands[0], operands[1], operands[2]));
+  DONE;
+}")
+
+;; The srcaddr operand of MOVC5 is not dereferenced if srclen is zero, so we
+;; set it to (%ap) somewhat arbitrarily chosen for the shortest encoding.
+(define_insn_and_split "setmemhi1"
+  [(set (match_operand:BLK 0 "memory_operand" "=o")
+	(unspec:BLK [(use (match_operand:QI 2 "general_operand" "g"))]
+		    UNSPEC_SETMEM_FILL))
+   (use (match_operand:HI 1 "general_operand" "g"))
+   (clobber (reg:SI 0))
+   (clobber (reg:SI 1))
+   (clobber (reg:SI 2))
+   (clobber (reg:SI 3))
+   (clobber (reg:SI 4))
+   (clobber (reg:SI 5))]
+  ""
+  "#"
+  "reload_completed"
+  [(parallel
+     [(set (match_dup 0)
+	   (unspec:BLK [(use (match_dup 2))] UNSPEC_SETMEM_FILL))
+      (use (match_dup 1))
+      (clobber (reg:SI 0))
+      (clobber (reg:SI 1))
+      (clobber (reg:SI 2))
+      (clobber (reg:SI 3))
+      (clobber (reg:SI 4))
+      (clobber (reg:SI 5))
+      (clobber (reg:CC VAX_PSL_REGNUM))])]
+  "")
+
+(define_insn "*setmemhi1"
+  [(set (match_operand:BLK 0 "memory_operand" "=o")
+	(unspec:BLK [(use (match_operand:QI 2 "general_operand" "g"))]
+		    UNSPEC_SETMEM_FILL))
+   (use (match_operand:HI 1 "general_operand" "g"))
+   (clobber (reg:SI 0))
+   (clobber (reg:SI 1))
+   (clobber (reg:SI 2))
+   (clobber (reg:SI 3))
+   (clobber (reg:SI 4))
+   (clobber (reg:SI 5))
+   (clobber (reg:CC VAX_PSL_REGNUM))]
+  "reload_completed"
+  "movc5 $0,(%%ap),%2,%1,%0")
 
 ;; Extension and truncation insns.
 
diff --git a/gcc/config/vax/vax.opt b/gcc/config/vax/vax.opt
index 470e8b1..712ba3a 100644
--- a/gcc/config/vax/vax.opt
+++ b/gcc/config/vax/vax.opt
@@ -42,6 +42,10 @@ munix
 Target RejectNegative Mask(UNIX_ASM)
 Generate code for UNIX assembler.
 
+mlra
+Target Mask(LRA)
+Enable Local Register Allocation.
+
 mvaxc-alignment
 Target RejectNegative Mask(VAXC_ALIGNMENT)
 Use VAXC structure conventions.
diff --git a/gcc/config/vx-common.h b/gcc/config/vx-common.h
index 7dd4dee..a436bf1 100644
--- a/gcc/config/vx-common.h
+++ b/gcc/config/vx-common.h
@@ -97,7 +97,7 @@ along with GCC; see the file COPYING3.  If not see
 
 /* ------------------------ Misc configuration bits ----------------------  */
 
-#ifndef TARGET_VXWORKS7
+#if !TARGET_VXWORKS7
 /* VxWorks, prior to version 7, could not have dots in constructor
    labels, because it used a mutant variation of collect2 that
    generates C code instead of assembly.  Thus each constructor label
diff --git a/gcc/configure b/gcc/configure
index 8790153..74b9d9b 100755
--- a/gcc/configure
+++ b/gcc/configure
@@ -12769,7 +12769,7 @@ else
 	CC="${CC_FOR_BUILD}" CFLAGS="${CFLAGS_FOR_BUILD}" \
 	CXX="${CXX_FOR_BUILD}" CXXFLAGS="${CXXFLAGS_FOR_BUILD}" \
 	LD="${LD_FOR_BUILD}" LDFLAGS="${LDFLAGS_FOR_BUILD}" \
-	GMPINC="" CPPFLAGS="${CPPFLAGS} -DGENERATOR_FILE" \
+	GMPINC="" CPPFLAGS="${CPPFLAGS_FOR_BUILD} -DGENERATOR_FILE" \
 	${realsrcdir}/configure \
 		--enable-languages=${enable_languages-all} \
 		${enable_obsolete+--enable-obsolete="$enable_obsolete"} \
@@ -30398,6 +30398,7 @@ if test x"$ld64_flag" = x"yes"; then
 
   # Set defaults for possibly untestable items.
   gcc_cv_ld64_export_dynamic=0
+  gcc_cv_ld64_platform_version=0
 
   if test "$build" = "$host"; then
     darwin_try_test=1
@@ -30421,9 +30422,12 @@ $as_echo_n "checking ld64 specified version... " >&6; }
     gcc_cv_ld64_major=`echo "$gcc_cv_ld64_version" | sed -e 's/\..*//'`
     { $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_ld64_major" >&5
 $as_echo "$gcc_cv_ld64_major" >&6; }
-   if test "$gcc_cv_ld64_major" -ge 236; then
+    if test "$gcc_cv_ld64_major" -ge 236; then
       gcc_cv_ld64_export_dynamic=1
     fi
+    if test "$gcc_cv_ld64_major" -ge 512; then
+      gcc_cv_ld64_platform_version=1
+    fi
   elif test -x "$gcc_cv_ld" -a "$darwin_try_test" -eq 1; then
     # If the version was not specified, try to find it.
     { $as_echo "$as_me:${as_lineno-$LINENO}: checking linker version" >&5
@@ -30442,6 +30446,15 @@ $as_echo_n "checking linker for -export_dynamic support... " >&6; }
     fi
     { $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_ld64_export_dynamic" >&5
 $as_echo "$gcc_cv_ld64_export_dynamic" >&6; }
+
+    { $as_echo "$as_me:${as_lineno-$LINENO}: checking linker for -platform_version support" >&5
+$as_echo_n "checking linker for -platform_version support... " >&6; }
+    gcc_cv_ld64_platform_version=1
+    if $gcc_cv_ld -platform_version macos 10.5 0.0 < /dev/null 2>&1 | grep 'unknown option' > /dev/null; then
+      gcc_cv_ld64_platform_version=0
+    fi
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_ld64_platform_version" >&5
+$as_echo "$gcc_cv_ld64_platform_version" >&6; }
   fi
 
   if test x"${gcc_cv_ld64_version}" != x; then
@@ -30457,6 +30470,12 @@ cat >>confdefs.h <<_ACEOF
 #define LD64_HAS_EXPORT_DYNAMIC $gcc_cv_ld64_export_dynamic
 _ACEOF
 
+
+
+cat >>confdefs.h <<_ACEOF
+#define LD64_HAS_PLATFORM_VERSION $gcc_cv_ld64_platform_version
+_ACEOF
+
 fi
 
 if test x"$dsymutil_flag" = x"yes"; then
@@ -31555,8 +31574,8 @@ done
 rm -f Make-hooks
 touch Make-hooks
 target_list="all.cross start.encap rest.encap tags \
-	install-common install-man install-info install-pdf install-html dvi \
-	pdf html uninstall info man srcextra srcman srcinfo \
+	install-common install-man install-info install-dvi install-pdf \
+	install-html dvi pdf html uninstall info man srcextra srcman srcinfo \
 	mostlyclean clean distclean maintainer-clean install-plugin"
 
 for t in $target_list
diff --git a/gcc/configure.ac b/gcc/configure.ac
index c2cad0a..c9ee1fb 100644
--- a/gcc/configure.ac
+++ b/gcc/configure.ac
@@ -2065,7 +2065,7 @@ else
 	CC="${CC_FOR_BUILD}" CFLAGS="${CFLAGS_FOR_BUILD}" \
 	CXX="${CXX_FOR_BUILD}" CXXFLAGS="${CXXFLAGS_FOR_BUILD}" \
 	LD="${LD_FOR_BUILD}" LDFLAGS="${LDFLAGS_FOR_BUILD}" \
-	GMPINC="" CPPFLAGS="${CPPFLAGS} -DGENERATOR_FILE" \
+	GMPINC="" CPPFLAGS="${CPPFLAGS_FOR_BUILD} -DGENERATOR_FILE" \
 	${realsrcdir}/configure \
 		--enable-languages=${enable_languages-all} \
 		${enable_obsolete+--enable-obsolete="$enable_obsolete"} \
@@ -6253,6 +6253,7 @@ if test x"$ld64_flag" = x"yes"; then
 
   # Set defaults for possibly untestable items.
   gcc_cv_ld64_export_dynamic=0
+  gcc_cv_ld64_platform_version=0
 
   if test "$build" = "$host"; then
     darwin_try_test=1
@@ -6274,9 +6275,12 @@ if test x"$ld64_flag" = x"yes"; then
     AC_MSG_CHECKING(ld64 specified version)
     gcc_cv_ld64_major=`echo "$gcc_cv_ld64_version" | sed -e 's/\..*//'`
     AC_MSG_RESULT($gcc_cv_ld64_major)
-   if test "$gcc_cv_ld64_major" -ge 236; then
+    if test "$gcc_cv_ld64_major" -ge 236; then
       gcc_cv_ld64_export_dynamic=1
     fi
+    if test "$gcc_cv_ld64_major" -ge 512; then
+      gcc_cv_ld64_platform_version=1
+    fi
   elif test -x "$gcc_cv_ld" -a "$darwin_try_test" -eq 1; then
     # If the version was not specified, try to find it.
     AC_MSG_CHECKING(linker version)
@@ -6291,6 +6295,13 @@ if test x"$ld64_flag" = x"yes"; then
       gcc_cv_ld64_export_dynamic=0
     fi
     AC_MSG_RESULT($gcc_cv_ld64_export_dynamic)
+
+    AC_MSG_CHECKING(linker for -platform_version support)
+    gcc_cv_ld64_platform_version=1
+    if $gcc_cv_ld -platform_version macos 10.5 0.0 < /dev/null 2>&1 | grep 'unknown option' > /dev/null; then
+      gcc_cv_ld64_platform_version=0
+    fi
+    AC_MSG_RESULT($gcc_cv_ld64_platform_version)
   fi
 
   if test x"${gcc_cv_ld64_version}" != x; then
@@ -6300,6 +6311,9 @@ if test x"$ld64_flag" = x"yes"; then
 
   AC_DEFINE_UNQUOTED(LD64_HAS_EXPORT_DYNAMIC, $gcc_cv_ld64_export_dynamic,
   [Define to 1 if ld64 supports '-export_dynamic'.])
+
+  AC_DEFINE_UNQUOTED(LD64_HAS_PLATFORM_VERSION, $gcc_cv_ld64_platform_version,
+  [Define to 1 if ld64 supports '-platform_version'.])
 fi
 
 if test x"$dsymutil_flag" = x"yes"; then
@@ -7218,8 +7232,8 @@ done
 rm -f Make-hooks
 touch Make-hooks
 target_list="all.cross start.encap rest.encap tags \
-	install-common install-man install-info install-pdf install-html dvi \
-	pdf html uninstall info man srcextra srcman srcinfo \
+	install-common install-man install-info install-dvi install-pdf \
+	install-html dvi pdf html uninstall info man srcextra srcman srcinfo \
 	mostlyclean clean distclean maintainer-clean install-plugin"
 
 for t in $target_list
diff --git a/gcc/coverage.c b/gcc/coverage.c
index 4467f1e..4daa3f9 100644
--- a/gcc/coverage.c
+++ b/gcc/coverage.c
@@ -571,8 +571,11 @@ coverage_compute_profile_id (struct cgraph_node *n)
       if (!use_name_only && first_global_object_name)
 	chksum = coverage_checksum_string
 	  (chksum, first_global_object_name);
-      chksum = coverage_checksum_string
-	(chksum, aux_base_name);
+      char *base_name = xstrdup (aux_base_name);
+      if (endswith (base_name, ".gk"))
+	base_name[strlen (base_name) - 3] = '\0';
+      chksum = coverage_checksum_string (chksum, base_name);
+      free (base_name);
     }
 
   /* Non-negative integers are hopefully small enough to fit in all targets.
diff --git a/gcc/cp/ChangeLog b/gcc/cp/ChangeLog
index 08ba25e1..61ce629 100644
--- a/gcc/cp/ChangeLog
+++ b/gcc/cp/ChangeLog
@@ -1,3 +1,223 @@
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* cp-tree.h (struct lang_decl_fn): Add implicit_constexpr.
+	(decl_implicit_constexpr_p): New.
+	* class.c (type_maybe_constexpr_destructor): Use
+	TYPE_HAS_TRIVIAL_DESTRUCTOR and maybe_constexpr_fn.
+	(finalize_literal_type_property): Simplify.
+	* constexpr.c (is_valid_constexpr_fn): Check for dtor.
+	(maybe_save_constexpr_fundef): Try to set DECL_DECLARED_CONSTEXPR_P
+	on inlines.
+	(cxx_eval_call_expression): Use maybe_constexpr_fn.
+	(maybe_constexpr_fn): Handle flag_implicit_constexpr.
+	(var_in_maybe_constexpr_fn): Use maybe_constexpr_fn.
+	(potential_constant_expression_1): Likewise.
+	(decl_implicit_constexpr_p): New.
+	* decl.c (validate_constexpr_redeclaration): Allow change with
+	-fimplicit-constexpr.
+	(grok_special_member_properties): Use maybe_constexpr_fn.
+	* error.c (dump_function_decl): Don't print 'constexpr'
+	if it's implicit.
+	* Make-lang.in (check-c++-all): Update.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* typeck2.c (split_nonconstant_init_1): Handle flexarrays better.
+
+2021-11-15  Jakub Jelinek  <jakub@redhat.com>
+
+	* parser.c (OMP_TARGET_CLAUSE_MASK): Add
+	PRAGMA_OMP_CLAUSE_THREAD_LIMIT.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* call.c (perform_implicit_conversion_flags): Use the location of
+	the argument.
+	* lambda.c (build_lambda_object): Set location on the TARGET_EXPR.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* constexpr.c (maybe_save_constexpr_fundef): Also check whether the
+	body of a constructor is potentially constant.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* semantics.c (is_this_parameter): Check DECL_HAS_VALUE_EXPR_P
+	instead of is_capture_proxy.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	PR c++/70690
+	* init.c (build_vec_init): Check default_init_uninitialized_part in
+	C++20.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* constexpr.c (maybe_constant_value): Don't evaluate
+	in an unevaluated operand unless manifestly const-evaluated.
+	(fold_non_dependent_expr_template): Likewise.
+	* decl.c (compute_array_index_type_loc): This context is
+	manifestly constant-evaluated.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* constexpr.c (cxx_eval_thunk_call): Error instead of ICE
+	on vbase thunk to constexpr function.
+
+2021-11-11  Patrick Palka  <ppalka@redhat.com>
+
+	* parser.c (cp_parser_template_argument_list): Use auto_vec
+	instead of manual memory management.
+
+2021-11-11  Jakub Jelinek  <jakub@redhat.com>
+
+	* parser.c (cp_parser_omp_clause_num_teams): Parse optional
+	lower-bound and store it into OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR.
+	Use OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR instead of
+	OMP_CLAUSE_NUM_TEAMS_EXPR.
+	(cp_parser_omp_target): For OMP_CLAUSE_NUM_TEAMS evaluate before
+	combined target teams even lower-bound expression.
+	* semantics.c (finish_omp_clauses): Handle
+	OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR of OMP_CLAUSE_NUM_TEAMS clause.
+	* pt.c (tsubst_omp_clauses): Likewise.
+	(tsubst_expr): For OMP_CLAUSE_NUM_TEAMS evaluate before
+	combined target teams even lower-bound expression.
+
+2021-11-09  Marek Polacek  <polacek@redhat.com>
+
+	* class.c (default_init_uninitialized_part): Use
+	next_initializable_field.
+	* method.c (walk_field_subobs): Skip unnamed bit-fields.
+
+2021-11-09  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/103114
+	* parser.c (cp_parser_userdef_numeric_literal): Use fold_build2
+	with COMPLEX_EXPR arg instead of build_complex, use build_zero_cst
+	instead of fold_convert from integer_zero_node.
+
+2021-11-09  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/98394
+	PR c++/85846
+	* parser.c (cp_parser_placeholder_type_specifier): Declare
+	static.  Don't override tentative to false when tmpl is a
+	concept-id with empty argument list.  Don't emit a "does not
+	constrain a type" error when tentative.
+
+2021-11-09  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/100652
+	* pt.c (push_template_decl): Check for bare parameter packs in
+	the argument list of a variable template partial specialization.
+
+2021-11-09  Aldy Hernandez  <aldyh@redhat.com>
+
+	* ptree.c (cxx_print_xnode): Add more space to pfx array.
+
+2021-11-04  Jason Merrill  <jason@redhat.com>
+
+	* call.c (build_array_conv): Use range-for.
+	(build_complex_conv): Likewise.
+	* constexpr.c (clear_no_implicit_zero)
+	(reduced_constant_expression_p): Likewise.
+	* decl.c (cp_complete_array_type): Likewise.
+	* decl2.c (mark_vtable_entries): Likewise.
+	* pt.c (iterative_hash_template_arg):
+	(invalid_tparm_referent_p, unify)
+	(type_dependent_expression_p): Likewise.
+	* typeck.c (build_ptrmemfunc_access_expr): Likewise.
+
+2021-11-03  Joseph Myers  <joseph@codesourcery.com>
+
+	PR c/103031
+	* cvt.c (convert_init): New function.
+
+2021-11-02  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* decl.c: Include langhooks-def.h.
+	(cxx_simulate_record_decl): New function.
+	* cp-objcp-common.h (cxx_simulate_record_decl): Declare.
+	(LANG_HOOKS_SIMULATE_RECORD_DECL): Override.
+
+2021-10-29  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/102820
+	* semantics.c (maybe_zero_constructor_nelts): New function.
+	(finish_compound_literal): Implement DR2351 - void{}.
+	If type is cv void and compound_literal has no elements, return
+	void_node.  If type is cv void and compound_literal might have no
+	elements after expansion, handle it like other dependent compound
+	literals.
+
+2021-10-28  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/102933
+	* parser.c (cp_parser_simple_type_specifier): Adjust diagnostic
+	for using auto in parameter declaration.
+	* pt.c (extract_autos_r): Ignore CTAD placeholders.
+	(extract_autos): Use range-based for.
+	(do_auto_deduction): Use extract_autos only for the concepts TS
+	and not also for standard concepts.
+	(type_uses_auto): Likewise with for_each_template_parm.
+	(check_auto_in_tmpl_args): Just return false outside of the
+	concepts TS.  Simplify.
+
+2021-10-28  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/102780
+	* constexpr.c (potential_constant_expression_1) <case TRUTH_*_EXPR>:
+	When tf_error isn't set, preemptively check potentiality of the
+	second operand before performing trial evaluation of the first
+	operand.
+	(potential_constant_expression_1): When tf_error is set, first check
+	potentiality quietly and return true if successful, otherwise
+	proceed noisily to give errors.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/102854
+	* semantics.c (handle_omp_for_class_iterator,
+	finish_omp_for): Adjust c_omp_check_loop_iv_exprs caller.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/102753
+	* constexpr.c (cxx_eval_outermost_constant_expr): Perform
+	find_immediate_fndecl discovery if is_consteval or
+	in_immediate_context () rather than if is_consteval, t != r
+	and not in immediate function's body.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/102753
+	* cp-tree.h (saved_scope): Document that consteval_if_p member
+	is also set while processing immediate invocation.
+	(in_immediate_context): Declare.
+	* call.c (in_immediate_context): New function.
+	(immediate_invocation_p): Use it.
+	(struct in_consteval_if_p_temp_override): New class.
+	(build_over_call): Temporarily set in_consteval_if_p for processing
+	immediate invocation arguments.
+	* typeck.c (cp_build_addr_expr_1): Diagnose taking address of
+	an immediate method.  Use t instead of TREE_OPERAND (arg, 1).
+	Use in_immediate_context function.
+	* constexpr.c (find_immediate_fndecl): Handle PTRMEM_CST
+	which refers to immediate function decl.
+
+2021-10-26  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/102617
+	* parser.c (cp_parser_for): Maybe call cp_parser_init_statement
+	twice.  Warn about range-based for loops with initializer here.
+	(cp_parser_init_statement): Don't duplicate code.  Allow
+	alias-declaration in init-statement.
+
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Make-lang.in: Add dummy c++.install-dvi target.
+
 2021-10-20  Jason Merrill  <jason@redhat.com>
 
 	* parser.c (cp_parser_parse_and_diagnose_invalid_type_name):
diff --git a/gcc/cp/Make-lang.in b/gcc/cp/Make-lang.in
index 155be74..de1fc02 100644
--- a/gcc/cp/Make-lang.in
+++ b/gcc/cp/Make-lang.in
@@ -165,6 +165,7 @@ c++.rest.encap:
 c++.info:
 c++.install-info:
 c++.dvi:
+c++.install-dvi:
 c++.pdf:
 c++.install-pdf:
 c++.install-html:
@@ -220,7 +221,7 @@ check-c++17:
 
 # Run the testsuite in all standard conformance levels.
 check-c++-all:
-	$(MAKE) RUNTESTFLAGS="$(RUNTESTFLAGS) --stds=98,11,14,17,2a,concepts" check-g++
+	$(MAKE) RUNTESTFLAGS="$(RUNTESTFLAGS) --stds=98,11,14,17,20,23,impcx" check-g++
 
 # Run the testsuite with garbage collection at every opportunity.
 check-g++-strict-gc:
diff --git a/gcc/cp/call.c b/gcc/cp/call.c
index c5601d9..4ee21c7 100644
--- a/gcc/cp/call.c
+++ b/gcc/cp/call.c
@@ -1070,8 +1070,6 @@ build_array_conv (tree type, tree ctor, int flags, tsubst_flags_t complain)
   conversion *c;
   unsigned HOST_WIDE_INT len = CONSTRUCTOR_NELTS (ctor);
   tree elttype = TREE_TYPE (type);
-  unsigned i;
-  tree val;
   bool bad = false;
   bool user = false;
   enum conversion_rank rank = cr_exact;
@@ -1089,10 +1087,10 @@ build_array_conv (tree type, tree ctor, int flags, tsubst_flags_t complain)
 
   flags = LOOKUP_IMPLICIT|LOOKUP_NO_NARROWING;
 
-  FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (ctor), i, val)
+  for (auto &e: CONSTRUCTOR_ELTS (ctor))
     {
       conversion *sub
-	= implicit_conversion (elttype, TREE_TYPE (val), val,
+	= implicit_conversion (elttype, TREE_TYPE (e.value), e.value,
 			       false, flags, complain);
       if (sub == NULL)
 	return NULL;
@@ -1124,8 +1122,6 @@ build_complex_conv (tree type, tree ctor, int flags,
   conversion *c;
   unsigned HOST_WIDE_INT len = CONSTRUCTOR_NELTS (ctor);
   tree elttype = TREE_TYPE (type);
-  unsigned i;
-  tree val;
   bool bad = false;
   bool user = false;
   enum conversion_rank rank = cr_exact;
@@ -1135,10 +1131,10 @@ build_complex_conv (tree type, tree ctor, int flags,
 
   flags = LOOKUP_IMPLICIT|LOOKUP_NO_NARROWING;
 
-  FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (ctor), i, val)
+  for (auto &e: CONSTRUCTOR_ELTS (ctor))
     {
       conversion *sub
-	= implicit_conversion (elttype, TREE_TYPE (val), val,
+	= implicit_conversion (elttype, TREE_TYPE (e.value), e.value,
 			       false, flags, complain);
       if (sub == NULL)
 	return NULL;
@@ -9025,6 +9021,20 @@ build_trivial_dtor_call (tree instance, bool no_ptr_deref)
 		 instance, clobber);
 }
 
+/* Return true if in an immediate function context, or an unevaluated operand,
+   or a subexpression of an immediate invocation.  */
+
+bool
+in_immediate_context ()
+{
+  return (cp_unevaluated_operand != 0
+	  || (current_function_decl != NULL_TREE
+	      && DECL_IMMEDIATE_FUNCTION_P (current_function_decl))
+	  || (current_binding_level->kind == sk_function_parms
+	      && current_binding_level->immediate_fn_ctx_p)
+	  || in_consteval_if_p);
+}
+
 /* Return true if a call to FN with number of arguments NARGS
    is an immediate invocation.  */
 
@@ -9033,18 +9043,25 @@ immediate_invocation_p (tree fn, int nargs)
 {
   return (TREE_CODE (fn) == FUNCTION_DECL
 	  && DECL_IMMEDIATE_FUNCTION_P (fn)
-	  && cp_unevaluated_operand == 0
-	  && (current_function_decl == NULL_TREE
-	      || !DECL_IMMEDIATE_FUNCTION_P (current_function_decl))
-	  && (current_binding_level->kind != sk_function_parms
-	      || !current_binding_level->immediate_fn_ctx_p)
-	  && !in_consteval_if_p
+	  && !in_immediate_context ()
 	  /* As an exception, we defer std::source_location::current ()
 	     invocations until genericization because LWG3396 mandates
 	     special behavior for it.  */
 	  && (nargs > 1 || !source_location_current_p (fn)));
 }
 
+/* temp_override for in_consteval_if_p, which can't use make_temp_override
+   because it is a bitfield.  */
+
+struct in_consteval_if_p_temp_override {
+  bool save_in_consteval_if_p;
+  in_consteval_if_p_temp_override ()
+    : save_in_consteval_if_p (in_consteval_if_p) {}
+  void reset () { in_consteval_if_p = save_in_consteval_if_p; }
+  ~in_consteval_if_p_temp_override ()
+  { reset (); }
+};
+
 /* Subroutine of the various build_*_call functions.  Overload resolution
    has chosen a winning candidate CAND; build up a CALL_EXPR accordingly.
    ARGS is a TREE_LIST of the unconverted arguments to the call.  FLAGS is a
@@ -9254,6 +9271,12 @@ build_over_call (struct z_candidate *cand, int flags, tsubst_flags_t complain)
     nargs = parmlen;
   argarray = XALLOCAVEC (tree, nargs);
 
+  in_consteval_if_p_temp_override icip;
+  /* If the call is immediate function invocation, make sure
+     taking address of immediate functions is allowed in its arguments.  */
+  if (immediate_invocation_p (STRIP_TEMPLATE (fn), nargs))
+    in_consteval_if_p = true;
+
   /* The implicit parameters to a constructor are not considered by overload
      resolution, and must be of the proper type.  */
   if (DECL_CONSTRUCTOR_P (fn))
@@ -9498,6 +9521,7 @@ build_over_call (struct z_candidate *cand, int flags, tsubst_flags_t complain)
 
   gcc_assert (j <= nargs);
   nargs = j;
+  icip.reset ();
 
   /* Avoid performing argument transformation if warnings are disabled.
      When tf_warning is set and at least one of the warnings is active
@@ -12525,7 +12549,11 @@ perform_implicit_conversion_flags (tree type, tree expr,
 	IMPLICIT_CONV_EXPR_BRACED_INIT (expr) = true;
     }
   else
-    expr = convert_like (conv, expr, complain);
+    {
+      /* Give a conversion call the same location as expr.  */
+      iloc_sentinel il (loc);
+      expr = convert_like (conv, expr, complain);
+    }
 
   /* Free all the conversions we allocated.  */
   obstack_free (&conversion_obstack, p);
diff --git a/gcc/cp/class.c b/gcc/cp/class.c
index f16e50b..c30a44f 100644
--- a/gcc/cp/class.c
+++ b/gcc/cp/class.c
@@ -5455,10 +5455,9 @@ default_init_uninitialized_part (tree type)
       if (r)
 	return r;
     }
-  for (t = TYPE_FIELDS (type); t; t = DECL_CHAIN (t))
-    if (TREE_CODE (t) == FIELD_DECL
-	&& !DECL_ARTIFICIAL (t)
-	&& !DECL_INITIAL (t))
+  for (t = next_initializable_field (TYPE_FIELDS (type)); t;
+       t = next_initializable_field (DECL_CHAIN (t)))
+    if (!DECL_INITIAL (t) && !DECL_ARTIFICIAL (t))
       {
 	r = default_init_uninitialized_part (TREE_TYPE (t));
 	if (r)
@@ -5548,10 +5547,16 @@ type_has_constexpr_destructor (tree t)
 static bool
 type_maybe_constexpr_destructor (tree t)
 {
+  /* Until C++20, only trivial destruction is constexpr.  */
+  if (TYPE_HAS_TRIVIAL_DESTRUCTOR (t))
+    return true;
+  if (cxx_dialect < cxx20)
+    return false;
   if (CLASS_TYPE_P (t) && CLASSTYPE_LAZY_DESTRUCTOR (t))
     /* Assume it's constexpr.  */
     return true;
-  return type_has_constexpr_destructor (t);
+  tree fn = CLASSTYPE_DESTRUCTOR (t);
+  return (fn && maybe_constexpr_fn (fn));
 }
 
 /* Returns true iff class TYPE has a virtual destructor.  */
@@ -5824,8 +5829,7 @@ finalize_literal_type_property (tree t)
   if (cxx_dialect < cxx11)
     CLASSTYPE_LITERAL_P (t) = false;
   else if (CLASSTYPE_LITERAL_P (t)
-	   && TYPE_HAS_NONTRIVIAL_DESTRUCTOR (t)
-	   && (cxx_dialect < cxx20 || !type_maybe_constexpr_destructor (t)))
+	   && !type_maybe_constexpr_destructor (t))
     CLASSTYPE_LITERAL_P (t) = false;
   else if (CLASSTYPE_LITERAL_P (t) && LAMBDA_TYPE_P (t))
     CLASSTYPE_LITERAL_P (t) = (cxx_dialect >= cxx17);
diff --git a/gcc/cp/constexpr.c b/gcc/cp/constexpr.c
index 6f83d30..69d9d3d 100644
--- a/gcc/cp/constexpr.c
+++ b/gcc/cp/constexpr.c
@@ -37,6 +37,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "stor-layout.h"
 #include "cgraph.h"
 #include "opts.h"
+#include "stringpool.h"
+#include "attribs.h"
 
 static bool verify_constant (tree, bool, bool *, bool *);
 #define VERIFY_CONSTANT(X)						\
@@ -220,6 +222,17 @@ is_valid_constexpr_fn (tree fun, bool complain)
 	inform (DECL_SOURCE_LOCATION (fun),
 		"lambdas are implicitly %<constexpr%> only in C++17 and later");
     }
+  else if (DECL_DESTRUCTOR_P (fun))
+    {
+      if (cxx_dialect < cxx20)
+	{
+	  ret = false;
+	  if (complain)
+	    error_at (DECL_SOURCE_LOCATION (fun),
+		      "%<constexpr%> destructors only available"
+		      " with %<-std=c++20%> or %<-std=gnu++20%>");
+	}
+    }
   else if (!DECL_CONSTRUCTOR_P (fun))
     {
       tree rettype = TREE_TYPE (TREE_TYPE (fun));
@@ -865,35 +878,82 @@ void
 maybe_save_constexpr_fundef (tree fun)
 {
   if (processing_template_decl
-      || !DECL_DECLARED_CONSTEXPR_P (fun)
       || cp_function_chain->invalid_constexpr
       || (DECL_CLONED_FUNCTION_P (fun) && !DECL_DELETING_DESTRUCTOR_P (fun)))
     return;
 
-  if (!is_valid_constexpr_fn (fun, !DECL_GENERATED_P (fun)))
+  /* With -fimplicit-constexpr, try to make inlines constexpr.  We'll
+     actually set DECL_DECLARED_CONSTEXPR_P below if the checks pass.  */
+  bool implicit = false;
+  if (flag_implicit_constexpr)
+    {
+      if (DECL_DELETING_DESTRUCTOR_P (fun)
+	  && decl_implicit_constexpr_p (DECL_CLONED_FUNCTION (fun)))
+	/* Don't inherit implicit constexpr from the non-deleting
+	   destructor.  */
+	DECL_DECLARED_CONSTEXPR_P (fun) = false;
+
+      if (!DECL_DECLARED_CONSTEXPR_P (fun)
+	  && DECL_DECLARED_INLINE_P (fun)
+	  && !lookup_attribute ("noinline", DECL_ATTRIBUTES (fun)))
+	implicit = true;
+    }
+
+  if (!DECL_DECLARED_CONSTEXPR_P (fun) && !implicit)
+    return;
+
+  bool complain = !DECL_GENERATED_P (fun) && !implicit;
+
+  if (!is_valid_constexpr_fn (fun, complain))
     return;
 
   tree massaged = massage_constexpr_body (fun, DECL_SAVED_TREE (fun));
   if (massaged == NULL_TREE || massaged == error_mark_node)
     {
-      if (!DECL_CONSTRUCTOR_P (fun))
+      if (!DECL_CONSTRUCTOR_P (fun) && complain)
 	error ("body of %<constexpr%> function %qD not a return-statement",
 	       fun);
       return;
     }
 
   bool potential = potential_rvalue_constant_expression (massaged);
-  if (!potential && !DECL_GENERATED_P (fun))
+  if (!potential && complain)
     require_potential_rvalue_constant_expression (massaged);
 
-  if (DECL_CONSTRUCTOR_P (fun)
-      && cx_check_missing_mem_inits (DECL_CONTEXT (fun),
-				     massaged, !DECL_GENERATED_P (fun)))
-    potential = false;
+  if (DECL_CONSTRUCTOR_P (fun) && potential)
+    {
+      if (cx_check_missing_mem_inits (DECL_CONTEXT (fun),
+				      massaged, complain))
+	potential = false;
+      else if (cxx_dialect > cxx11)
+	{
+	  /* What we got from massage_constexpr_body is pretty much just the
+	     ctor-initializer, also check the body.  */
+	  massaged = DECL_SAVED_TREE (fun);
+	  potential = potential_rvalue_constant_expression (massaged);
+	  if (!potential && complain)
+	    require_potential_rvalue_constant_expression (massaged);
+	}
+    }
 
-  if (!potential && !DECL_GENERATED_P (fun))
+  if (!potential && complain)
     return;
 
+  if (implicit)
+    {
+      if (potential)
+	{
+	  DECL_DECLARED_CONSTEXPR_P (fun) = true;
+	  DECL_LANG_SPECIFIC (fun)->u.fn.implicit_constexpr = true;
+	  if (DECL_CONSTRUCTOR_P (fun))
+	    TYPE_HAS_CONSTEXPR_CTOR (DECL_CONTEXT (fun)) = true;
+	}
+      else
+	/* Don't bother keeping the pre-generic body of unsuitable functions
+	   not explicitly declared constexpr.  */
+	return;
+    }
+
   constexpr_fundef entry = {fun, NULL_TREE, NULL_TREE, NULL_TREE};
   bool clear_ctx = false;
   if (DECL_RESULT (fun) && DECL_CONTEXT (DECL_RESULT (fun)) == NULL_TREE)
@@ -1831,10 +1891,9 @@ clear_no_implicit_zero (tree ctor)
   if (CONSTRUCTOR_NO_CLEARING (ctor))
     {
       CONSTRUCTOR_NO_CLEARING (ctor) = false;
-      tree elt; unsigned HOST_WIDE_INT idx;
-      FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (ctor), idx, elt)
-	if (TREE_CODE (elt) == CONSTRUCTOR)
-	  clear_no_implicit_zero (elt);
+      for (auto &e: CONSTRUCTOR_ELTS (ctor))
+	if (TREE_CODE (e.value) == CONSTRUCTOR)
+	  clear_no_implicit_zero (e.value);
     }
 }
 
@@ -2247,15 +2306,20 @@ cxx_eval_thunk_call (const constexpr_ctx *ctx, tree t, tree thunk_fndecl,
 {
   tree function = THUNK_TARGET (thunk_fndecl);
 
-  /* virtual_offset is only set in the presence of virtual bases, which make
-     the class non-literal, so we don't need to handle it here.  */
   if (THUNK_VIRTUAL_OFFSET (thunk_fndecl))
     {
-      gcc_assert (!DECL_DECLARED_CONSTEXPR_P (function));
       if (!ctx->quiet)
 	{
-	  error ("call to non-%<constexpr%> function %qD", function);
-	  explain_invalid_constexpr_fn (function);
+	  if (!DECL_DECLARED_CONSTEXPR_P (function))
+	    {
+	      error ("call to non-%<constexpr%> function %qD", function);
+	      explain_invalid_constexpr_fn (function);
+	    }
+	  else
+	    /* virtual_offset is only set for virtual bases, which make the
+	       class non-literal, so we don't need to handle it here.  */
+	    error ("calling constexpr member function %qD through virtual "
+		   "base subobject", function);
 	}
       *non_constant_p = true;
       return t;
@@ -2387,7 +2451,7 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 					   lval, non_constant_p, overflow_p);
   if (DECL_THUNK_P (fun))
     return cxx_eval_thunk_call (ctx, t, fun, lval, non_constant_p, overflow_p);
-  if (!DECL_DECLARED_CONSTEXPR_P (fun))
+  if (!maybe_constexpr_fn (fun))
     {
       if (TREE_CODE (t) == CALL_EXPR
 	  && cxx_replaceable_global_alloc_fn (fun)
@@ -2950,7 +3014,7 @@ reduced_constant_expression_p (tree t)
 
     case CONSTRUCTOR:
       /* And we need to handle PTRMEM_CST wrapped in a CONSTRUCTOR.  */
-      tree idx, val, field; unsigned HOST_WIDE_INT i;
+      tree field;
       if (CONSTRUCTOR_NO_CLEARING (t))
 	{
 	  if (TREE_CODE (TREE_TYPE (t)) == VECTOR_TYPE)
@@ -2964,14 +3028,14 @@ reduced_constant_expression_p (tree t)
 	      tree min = TYPE_MIN_VALUE (TYPE_DOMAIN (TREE_TYPE (t)));
 	      tree max = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (t)));
 	      tree cursor = min;
-	      FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (t), i, idx, val)
+	      for (auto &e: CONSTRUCTOR_ELTS (t))
 		{
-		  if (!reduced_constant_expression_p (val))
+		  if (!reduced_constant_expression_p (e.value))
 		    return false;
-		  if (array_index_cmp (cursor, idx) != 0)
+		  if (array_index_cmp (cursor, e.index) != 0)
 		    return false;
-		  if (TREE_CODE (idx) == RANGE_EXPR)
-		    cursor = TREE_OPERAND (idx, 1);
+		  if (TREE_CODE (e.index) == RANGE_EXPR)
+		    cursor = TREE_OPERAND (e.index, 1);
 		  cursor = int_const_binop (PLUS_EXPR, cursor, size_one_node);
 		}
 	      if (find_array_ctor_elt (t, max) == -1)
@@ -2992,14 +3056,14 @@ reduced_constant_expression_p (tree t)
 	}
       else
 	field = NULL_TREE;
-      FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (t), i, idx, val)
+      for (auto &e: CONSTRUCTOR_ELTS (t))
 	{
 	  /* If VAL is null, we're in the middle of initializing this
 	     element.  */
-	  if (!reduced_constant_expression_p (val))
+	  if (!reduced_constant_expression_p (e.value))
 	    return false;
 	  /* Empty class field may or may not have an initializer.  */
-	  for (; field && idx != field;
+	  for (; field && e.index != field;
 	       field = next_initializable_field (DECL_CHAIN (field)))
 	    if (!is_really_empty_class (TREE_TYPE (field),
 					/*ignore_vptr*/false))
@@ -5212,7 +5276,9 @@ bool
 maybe_constexpr_fn (tree t)
 {
   return (DECL_DECLARED_CONSTEXPR_P (t)
-	  || (cxx_dialect >= cxx17 && LAMBDA_FUNCTION_P (t)));
+	  || (cxx_dialect >= cxx17 && LAMBDA_FUNCTION_P (t))
+	  || (flag_implicit_constexpr
+	      && DECL_DECLARED_INLINE_P (STRIP_TEMPLATE (t))));
 }
 
 /* True if T was declared in a function that might be constexpr: either a
@@ -5221,11 +5287,8 @@ maybe_constexpr_fn (tree t)
 bool
 var_in_maybe_constexpr_fn (tree t)
 {
-  if (cxx_dialect >= cxx17
-      && DECL_FUNCTION_SCOPE_P (t)
-      && LAMBDA_FUNCTION_P (DECL_CONTEXT (t)))
-    return true;
-  return var_in_constexpr_fn (t);
+  return (DECL_FUNCTION_SCOPE_P (t)
+	  && maybe_constexpr_fn (DECL_CONTEXT (t)));
 }
 
 /* We're assigning INIT to TARGET.  In do_build_copy_constructor and
@@ -7276,6 +7339,10 @@ find_immediate_fndecl (tree *tp, int */*walk_subtrees*/, void */*data*/)
 {
   if (TREE_CODE (*tp) == FUNCTION_DECL && DECL_IMMEDIATE_FUNCTION_P (*tp))
     return *tp;
+  if (TREE_CODE (*tp) == PTRMEM_CST
+      && TREE_CODE (PTRMEM_CST_MEMBER (*tp)) == FUNCTION_DECL
+      && DECL_IMMEDIATE_FUNCTION_P (PTRMEM_CST_MEMBER (*tp)))
+    return PTRMEM_CST_MEMBER (*tp);
   return NULL_TREE;
 }
 
@@ -7468,12 +7535,8 @@ cxx_eval_outermost_constant_expr (tree t, bool allow_non_constant,
     }
 
   /* Check that immediate invocation does not return an expression referencing
-     any immediate function decls.  They need to be allowed while parsing
-     immediate functions, but can't leak outside of them.  */
-  if (is_consteval
-      && t != r
-      && (current_function_decl == NULL_TREE
-	  || !DECL_IMMEDIATE_FUNCTION_P (current_function_decl)))
+     any immediate function decls.  */
+  if (is_consteval || in_immediate_context ())
     if (tree immediate_fndecl
 	= cp_walk_tree_without_duplicates (&r, find_immediate_fndecl,
 					   NULL))
@@ -7692,6 +7755,10 @@ maybe_constant_value (tree t, tree decl, bool manifestly_const_eval)
       return r;
     }
 
+  /* Don't evaluate an unevaluated operand.  */
+  if (cp_unevaluated_operand)
+    return t;
+
   uid_sensitive_constexpr_evaluation_checker c;
   r = cxx_eval_outermost_constant_expr (t, true, true, false, false, decl);
   gcc_checking_assert (r == t
@@ -7755,6 +7822,9 @@ fold_non_dependent_expr_template (tree t, tsubst_flags_t complain,
 	  return t;
 	}
 
+      if (cp_unevaluated_operand && !manifestly_const_eval)
+	return t;
+
       tree r = cxx_eval_outermost_constant_expr (t, true, true,
 						 manifestly_const_eval,
 						 false, object);
@@ -8195,7 +8265,7 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 	      {
 		if (builtin_valid_in_constant_expr_p (fun))
 		  return true;
-		if (!DECL_DECLARED_CONSTEXPR_P (fun)
+		if (!maybe_constexpr_fn (fun)
 		    /* Allow any built-in function; if the expansion
 		       isn't constant, we'll deal with that then.  */
 		    && !fndecl_built_in_p (fun)
@@ -8892,13 +8962,18 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
       tmp = boolean_false_node;
     truth:
       {
-	tree op = TREE_OPERAND (t, 0);
-	if (!RECUR (op, rval))
+	tree op0 = TREE_OPERAND (t, 0);
+	tree op1 = TREE_OPERAND (t, 1);
+	if (!RECUR (op0, rval))
 	  return false;
+	if (!(flags & tf_error) && RECUR (op1, rval))
+	  /* When quiet, try to avoid expensive trial evaluation by first
+	     checking potentiality of the second operand.  */
+	  return true;
 	if (!processing_template_decl)
-	  op = cxx_eval_outermost_constant_expr (op, true);
-	if (tree_int_cst_equal (op, tmp))
-	  return RECUR (TREE_OPERAND (t, 1), rval);
+	  op0 = cxx_eval_outermost_constant_expr (op0, true);
+	if (tree_int_cst_equal (op0, tmp))
+	  return (flags & tf_error) ? RECUR (op1, rval) : false;
 	else
 	  return true;
       }
@@ -9107,6 +9182,17 @@ bool
 potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 				 tsubst_flags_t flags)
 {
+  if (flags & tf_error)
+    {
+      /* Check potentiality quietly first, as that could be performed more
+	 efficiently in some cases (currently only for TRUTH_*_EXPR).  If
+	 that fails, replay the check noisily to give errors.  */
+      flags &= ~tf_error;
+      if (potential_constant_expression_1 (t, want_rval, strict, now, flags))
+	return true;
+      flags |= tf_error;
+    }
+
   tree target = NULL_TREE;
   return potential_constant_expression_1 (t, want_rval, strict, now,
 					  flags, &target);
@@ -9217,6 +9303,23 @@ is_nondependent_static_init_expression (tree t)
 	  && !instantiation_dependent_expression_p (t));
 }
 
+/* True iff FN is an implicitly constexpr function.  */
+
+bool
+decl_implicit_constexpr_p (tree fn)
+{
+  if (!(flag_implicit_constexpr
+	&& TREE_CODE (fn) == FUNCTION_DECL
+	&& DECL_DECLARED_CONSTEXPR_P (fn)))
+    return false;
+
+  if (DECL_CLONED_FUNCTION_P (fn))
+    fn = DECL_CLONED_FUNCTION (fn);
+
+  return (DECL_LANG_SPECIFIC (fn)
+	  && DECL_LANG_SPECIFIC (fn)->u.fn.implicit_constexpr);
+}
+
 /* Finalize constexpr processing after parsing.  */
 
 void
diff --git a/gcc/cp/cp-objcp-common.h b/gcc/cp/cp-objcp-common.h
index f1704aa..d585940 100644
--- a/gcc/cp/cp-objcp-common.h
+++ b/gcc/cp/cp-objcp-common.h
@@ -39,6 +39,8 @@ extern bool cp_handle_option (size_t, const char *, HOST_WIDE_INT, int,
 extern tree cxx_make_type_hook			(tree_code);
 extern tree cxx_simulate_enum_decl (location_t, const char *,
 				    vec<string_int_pair> *);
+extern tree cxx_simulate_record_decl (location_t, const char *,
+				      array_slice<const tree>);
 
 /* Lang hooks that are shared between C++ and ObjC++ are defined here.  Hooks
    specific to C++ or ObjC++ go in cp/cp-lang.c and objcp/objcp-lang.c,
@@ -139,6 +141,8 @@ extern tree cxx_simulate_enum_decl (location_t, const char *,
 #define LANG_HOOKS_MAKE_TYPE cxx_make_type_hook
 #undef LANG_HOOKS_SIMULATE_ENUM_DECL
 #define LANG_HOOKS_SIMULATE_ENUM_DECL cxx_simulate_enum_decl
+#undef LANG_HOOKS_SIMULATE_RECORD_DECL
+#define LANG_HOOKS_SIMULATE_RECORD_DECL cxx_simulate_record_decl
 #undef LANG_HOOKS_TYPE_FOR_MODE
 #define LANG_HOOKS_TYPE_FOR_MODE c_common_type_for_mode
 #undef LANG_HOOKS_TYPE_FOR_SIZE
diff --git a/gcc/cp/cp-tree.h b/gcc/cp/cp-tree.h
index 5248ecd..acc98c9 100644
--- a/gcc/cp/cp-tree.h
+++ b/gcc/cp/cp-tree.h
@@ -1825,7 +1825,8 @@ struct GTY(()) saved_scope {
      if-statement.  */
   BOOL_BITFIELD discarded_stmt : 1;
   /* Nonzero if we are parsing or instantiating the compound-statement
-     of consteval if statement.  */
+     of consteval if statement.  Also set while processing an immediate
+     invocation.  */
   BOOL_BITFIELD consteval_if_p : 1;
 
   int unevaluated_operand;
@@ -2867,8 +2868,9 @@ struct GTY(()) lang_decl_fn {
   unsigned immediate_fn_p : 1;
   unsigned maybe_deleted : 1;
   unsigned coroutine_p : 1;
+  unsigned implicit_constexpr : 1;
 
-  unsigned spare : 10;
+  unsigned spare : 9;
 
   /* 32-bits padding on 64-bit host.  */
 
@@ -6547,6 +6549,7 @@ extern tree perform_direct_initialization_if_possible (tree, tree, bool,
                                                        tsubst_flags_t);
 extern vec<tree,va_gc> *resolve_args (vec<tree,va_gc>*, tsubst_flags_t);
 extern tree in_charge_arg_for_name		(tree);
+extern bool in_immediate_context		();
 extern tree build_cxx_call			(tree, int, tree *,
 						 tsubst_flags_t,
 						 tree = NULL_TREE);
@@ -8313,6 +8316,7 @@ extern vec<tree> cx_error_context               (void);
 extern tree fold_sizeof_expr			(tree);
 extern void clear_cv_and_fold_caches		(void);
 extern tree unshare_constructor			(tree CXX_MEM_STAT_INFO);
+extern bool decl_implicit_constexpr_p		(tree);
 
 /* An RAII sentinel used to restrict constexpr evaluation so that it
    doesn't do anything that causes extra DECL_UID generation.  */
diff --git a/gcc/cp/cvt.c b/gcc/cp/cvt.c
index d035e61..7e6752a 100644
--- a/gcc/cp/cvt.c
+++ b/gcc/cp/cvt.c
@@ -1685,6 +1685,15 @@ convert (tree type, tree expr)
 		      tf_warning_or_error);
 }
 
+/* Like convert, but in a static initializer (called from
+   convert_and_check).  */
+
+tree
+convert_init (tree type, tree expr)
+{
+  return convert (type, expr);
+}
+
 /* Like cp_convert, except permit conversions to take place which
    are not normally allowed due to access restrictions
    (such as conversion from sub-type to private super-type).  */
diff --git a/gcc/cp/decl.c b/gcc/cp/decl.c
index 8a45411..2ddf0e4 100644
--- a/gcc/cp/decl.c
+++ b/gcc/cp/decl.c
@@ -57,6 +57,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "omp-general.h"
 #include "omp-offload.h"  /* For offload_vars.  */
 #include "opts.h"
+#include "langhooks-def.h"  /* For lhd_simulate_record_decl  */
 
 /* Possible cases of bad specifiers type used by bad_specifiers. */
 enum bad_spec_place {
@@ -1289,6 +1290,12 @@ validate_constexpr_redeclaration (tree old_decl, tree new_decl)
     }
   if (TREE_CODE (old_decl) == FUNCTION_DECL)
     {
+      /* With -fimplicit-constexpr, ignore changes in the constexpr
+	 keyword.  */
+      if (flag_implicit_constexpr
+	  && (DECL_IMMEDIATE_FUNCTION_P (new_decl)
+	      == DECL_IMMEDIATE_FUNCTION_P (old_decl)))
+	return true;
       if (fndecl_built_in_p (old_decl))
 	{
 	  /* Hide a built-in declaration.  */
@@ -9537,9 +9544,6 @@ cp_complete_array_type (tree *ptype, tree initial_value, bool do_default)
 
   if (initial_value)
     {
-      unsigned HOST_WIDE_INT i;
-      tree value;
-
       /* An array of character type can be initialized from a
 	 brace-enclosed string constant.
 
@@ -9561,14 +9565,9 @@ cp_complete_array_type (tree *ptype, tree initial_value, bool do_default)
       /* If any of the elements are parameter packs, we can't actually
 	 complete this type now because the array size is dependent.  */
       if (TREE_CODE (initial_value) == CONSTRUCTOR)
-	{
-	  FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (initial_value), 
-				      i, value)
-	    {
-	      if (PACK_EXPANSION_P (value))
-		return 0;
-	    }
-	}
+	for (auto &e: CONSTRUCTOR_ELTS (initial_value))
+	  if (PACK_EXPANSION_P (e.value))
+	    return 0;
     }
 
   failure = complete_array_type (ptype, initial_value, do_default);
@@ -11007,7 +11006,7 @@ compute_array_index_type_loc (location_t name_loc, tree name, tree size,
 				    cp_convert (ssizetype, integer_one_node,
 						complain),
 				    complain);
-	itype = maybe_constant_value (itype);
+	itype = maybe_constant_value (itype, NULL_TREE, true);
       }
 
       if (!TREE_CONSTANT (itype))
@@ -14870,7 +14869,7 @@ grok_special_member_properties (tree decl)
       if (is_list_ctor (decl))
 	TYPE_HAS_LIST_CTOR (class_type) = 1;
 
-      if (DECL_DECLARED_CONSTEXPR_P (decl)
+      if (maybe_constexpr_fn (decl)
 	  && !ctor && !move_fn_p (decl))
 	TYPE_HAS_CONSTEXPR_CTOR (class_type) = 1;
     }
@@ -16602,6 +16601,42 @@ cxx_simulate_enum_decl (location_t loc, const char *name,
   input_location = saved_loc;
   return enumtype;
 }
+
+/* Implement LANG_HOOKS_SIMULATE_RECORD_DECL.  */
+
+tree
+cxx_simulate_record_decl (location_t loc, const char *name,
+			  array_slice<const tree> fields)
+{
+  iloc_sentinel ils (loc);
+
+  tree ident = get_identifier (name);
+  tree type = xref_tag (/*tag_code=*/record_type, ident);
+  if (type != error_mark_node
+      && (TREE_CODE (type) != RECORD_TYPE || COMPLETE_TYPE_P (type)))
+    {
+      error ("redefinition of %q#T", type);
+      type = error_mark_node;
+    }
+  if (type == error_mark_node)
+    return lhd_simulate_record_decl (loc, name, fields);
+
+  xref_basetypes (type, NULL_TREE);
+  type = begin_class_definition (type);
+  if (type == error_mark_node)
+    return lhd_simulate_record_decl (loc, name, fields);
+
+  for (tree field : fields)
+    finish_member_declaration (field);
+
+  type = finish_struct (type, NULL_TREE);
+
+  tree decl = build_decl (loc, TYPE_DECL, ident, type);
+  set_underlying_type (decl);
+  lang_hooks.decls.pushdecl (decl);
+
+  return type;
+}
 
 /* We're defining DECL.  Make sure that its type is OK.  */
 
diff --git a/gcc/cp/decl2.c b/gcc/cp/decl2.c
index a79a70b..32d3fe3 100644
--- a/gcc/cp/decl2.c
+++ b/gcc/cp/decl2.c
@@ -1913,18 +1913,14 @@ coerce_delete_type (tree decl, location_t loc)
 static void
 mark_vtable_entries (tree decl, vec<tree> &consteval_vtables)
 {
-  tree fnaddr;
-  unsigned HOST_WIDE_INT idx;
-
   /* It's OK for the vtable to refer to deprecated virtual functions.  */
   warning_sentinel w(warn_deprecated_decl);
 
   bool consteval_seen = false;
 
-  FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (DECL_INITIAL (decl)),
-			      idx, fnaddr)
+  for (auto &e: CONSTRUCTOR_ELTS (DECL_INITIAL (decl)))
     {
-      tree fn;
+      tree fnaddr = e.value;
 
       STRIP_NOPS (fnaddr);
 
@@ -1934,7 +1930,7 @@ mark_vtable_entries (tree decl, vec<tree> &consteval_vtables)
 	   virtual call offset, an RTTI offset, etc.  */
 	continue;
 
-      fn = TREE_OPERAND (fnaddr, 0);
+      tree fn = TREE_OPERAND (fnaddr, 0);
       if (TREE_CODE (fn) == FUNCTION_DECL && DECL_IMMEDIATE_FUNCTION_P (fn))
 	{
 	  if (!consteval_seen)
diff --git a/gcc/cp/error.c b/gcc/cp/error.c
index 012a4ec..8724793 100644
--- a/gcc/cp/error.c
+++ b/gcc/cp/error.c
@@ -1686,7 +1686,8 @@ dump_function_decl (cxx_pretty_printer *pp, tree t, int flags)
   exceptions = TYPE_RAISES_EXCEPTIONS (TREE_TYPE (t));
 
   /* Likewise for the constexpr specifier, in case t is a specialization.  */
-  constexpr_p = DECL_DECLARED_CONSTEXPR_P (t);
+  constexpr_p = (DECL_DECLARED_CONSTEXPR_P (t)
+		 && !decl_implicit_constexpr_p (t));
 
   /* Pretty print template instantiations only.  */
   if (DECL_USE_TEMPLATE (t) && DECL_TEMPLATE_INFO (t)
diff --git a/gcc/cp/init.c b/gcc/cp/init.c
index 771a19b..3ba2e3b 100644
--- a/gcc/cp/init.c
+++ b/gcc/cp/init.c
@@ -4470,11 +4470,14 @@ build_vec_init (tree base, tree maxindex, tree init,
 
      We do need to keep going if we're copying an array.  */
 
-  if (try_const && !init)
+  if (try_const && !init
+      && (cxx_dialect < cxx20
+	  || !default_init_uninitialized_part (inner_elt_type)))
     /* With a constexpr default constructor, which we checked for when
        setting try_const above, default-initialization is equivalent to
        value-initialization, and build_value_init gives us something more
-       friendly to maybe_constant_init.  */
+       friendly to maybe_constant_init.  Except in C++20 and up a constexpr
+       constructor need not initialize all the members.  */
     explicit_value_init_p = true;
   if (from_array
       || ((type_build_ctor_call (type) || init || explicit_value_init_p)
diff --git a/gcc/cp/lambda.c b/gcc/cp/lambda.c
index 2e9d38b..f68c68c 100644
--- a/gcc/cp/lambda.c
+++ b/gcc/cp/lambda.c
@@ -57,14 +57,13 @@ build_lambda_object (tree lambda_expr)
      - cp_parser_functional_cast  */
   vec<constructor_elt, va_gc> *elts = NULL;
   tree node, expr, type;
-  location_t saved_loc;
 
   if (processing_template_decl || lambda_expr == error_mark_node)
     return lambda_expr;
 
   /* Make sure any error messages refer to the lambda-introducer.  */
-  saved_loc = input_location;
-  input_location = LAMBDA_EXPR_LOCATION (lambda_expr);
+  location_t loc = LAMBDA_EXPR_LOCATION (lambda_expr);
+  iloc_sentinel il (loc);
 
   for (node = LAMBDA_EXPR_CAPTURE_LIST (lambda_expr);
        node;
@@ -117,10 +116,10 @@ build_lambda_object (tree lambda_expr)
   type = LAMBDA_EXPR_CLOSURE (lambda_expr);
   CLASSTYPE_NON_AGGREGATE (type) = 0;
   expr = finish_compound_literal (type, expr, tf_warning_or_error);
+  protected_set_expr_location (expr, loc);
   CLASSTYPE_NON_AGGREGATE (type) = 1;
 
  out:
-  input_location = saved_loc;
   return expr;
 }
 
diff --git a/gcc/cp/method.c b/gcc/cp/method.c
index 1023aef..935946f 100644
--- a/gcc/cp/method.c
+++ b/gcc/cp/method.c
@@ -2295,7 +2295,9 @@ walk_field_subobs (tree fields, special_function_kind sfk, tree fnname,
     {
       tree mem_type, argtype, rval;
 
-      if (TREE_CODE (field) != FIELD_DECL || DECL_ARTIFICIAL (field))
+      if (TREE_CODE (field) != FIELD_DECL
+	  || DECL_ARTIFICIAL (field)
+	  || DECL_UNNAMED_BIT_FIELD (field))
 	continue;
 
       /* Variant members only affect deletedness.  In particular, they don't
diff --git a/gcc/cp/parser.c b/gcc/cp/parser.c
index 49d951c..e86e2b5 100644
--- a/gcc/cp/parser.c
+++ b/gcc/cp/parser.c
@@ -4804,9 +4804,8 @@ cp_parser_userdef_numeric_literal (cp_parser *parser)
       else /* if (id_equal (suffix_id, "il")) */
 	type = long_double_type_node;
 
-      value = build_complex (build_complex_type (type),
-			     fold_convert (type, integer_zero_node),
-			     fold_convert (type, value));
+      value = fold_build2 (COMPLEX_EXPR, build_complex_type (type),
+			   build_zero_cst (type), fold_convert (type, value));
     }
 
   if (cp_parser_uncommitted_to_tentative_parse_p (parser))
@@ -12040,6 +12039,7 @@ cp_parser_handle_directive_omp_attributes (cp_parser *parser, tree *pattrs,
   init-statement:
     expression-statement
     simple-declaration
+    alias-declaration
 
   TM Extension:
 
@@ -13327,6 +13327,23 @@ cp_parser_for (cp_parser *parser, bool ivdep, unsigned short unroll)
   /* Begin the for-statement.  */
   scope = begin_for_scope (&init);
 
+  /* Maybe parse the optional init-statement in a range-based for loop.  */
+  if (cp_parser_range_based_for_with_init_p (parser)
+      /* Checked for diagnostic purposes only.  */
+      && cp_lexer_next_token_is_not (parser->lexer, CPP_SEMICOLON))
+    {
+      tree dummy;
+      cp_parser_init_statement (parser, &dummy);
+      if (cxx_dialect < cxx20)
+	{
+	  pedwarn (cp_lexer_peek_token (parser->lexer)->location,
+		   OPT_Wc__20_extensions,
+		   "range-based %<for%> loops with initializer only "
+		   "available with %<-std=c++20%> or %<-std=gnu++20%>");
+	  decl = error_mark_node;
+	}
+    }
+
   /* Parse the initialization.  */
   is_range_for = cp_parser_init_statement (parser, &decl);
 
@@ -13987,12 +14004,13 @@ cp_parser_iteration_statement (cp_parser* parser, bool *if_p, bool ivdep,
   return statement;
 }
 
-/* Parse a init-statement or the declarator of a range-based-for.
+/* Parse an init-statement or the declarator of a range-based-for.
    Returns true if a range-based-for declaration is seen.
 
    init-statement:
      expression-statement
-     simple-declaration  */
+     simple-declaration
+     alias-declaration  */
 
 static bool
 cp_parser_init_statement (cp_parser *parser, tree *decl)
@@ -14008,40 +14026,29 @@ cp_parser_init_statement (cp_parser *parser, tree *decl)
       bool is_range_for = false;
       bool saved_colon_corrects_to_scope_p = parser->colon_corrects_to_scope_p;
 
-      /* Try to parse the init-statement.  */
-      if (cp_parser_range_based_for_with_init_p (parser))
-	{
-	  tree dummy;
-	  cp_parser_parse_tentatively (parser);
-	  /* Parse the declaration.  */
-	  cp_parser_simple_declaration (parser,
-					/*function_definition_allowed_p=*/false,
-					&dummy);
-	  cp_parser_require (parser, CPP_SEMICOLON, RT_SEMICOLON);
-	  if (!cp_parser_parse_definitely (parser))
-	    /* That didn't work, try to parse it as an expression-statement.  */
-	    cp_parser_expression_statement (parser, NULL_TREE);
-
-	  if (cxx_dialect < cxx20)
-	    {
-	      pedwarn (cp_lexer_peek_token (parser->lexer)->location,
-		       OPT_Wc__20_extensions,
-		       "range-based %<for%> loops with initializer only "
-		       "available with %<-std=c++20%> or %<-std=gnu++20%>");
-	      *decl = error_mark_node;
-	    }
-	}
-
       /* A colon is used in range-based for.  */
       parser->colon_corrects_to_scope_p = false;
 
       /* We're going to speculatively look for a declaration, falling back
 	 to an expression, if necessary.  */
       cp_parser_parse_tentatively (parser);
-      /* Parse the declaration.  */
-      cp_parser_simple_declaration (parser,
-				    /*function_definition_allowed_p=*/false,
-				    decl);
+      bool expect_semicolon_p = true;
+      if (cp_lexer_next_token_is_keyword (parser->lexer, RID_USING))
+	{
+	  cp_parser_alias_declaration (parser);
+	  expect_semicolon_p = false;
+	  if (cxx_dialect < cxx23
+	      && !cp_parser_uncommitted_to_tentative_parse_p (parser))
+	    pedwarn (cp_lexer_peek_token (parser->lexer)->location,
+		     OPT_Wc__23_extensions,
+		     "alias-declaration in init-statement only "
+		     "available with %<-std=c++23%> or %<-std=gnu++23%>");
+	}
+      else
+	/* Parse the declaration.  */
+	cp_parser_simple_declaration (parser,
+				      /*function_definition_allowed_p=*/false,
+				      decl);
       parser->colon_corrects_to_scope_p = saved_colon_corrects_to_scope_p;
       if (cp_lexer_next_token_is (parser->lexer, CPP_COLON))
 	{
@@ -14054,7 +14061,7 @@ cp_parser_init_statement (cp_parser *parser, tree *decl)
 		     "range-based %<for%> loops only available with "
 		     "%<-std=c++11%> or %<-std=gnu++11%>");
 	}
-      else
+      else if (expect_semicolon_p)
 	/* The ';' is not consumed yet because we told
 	   cp_parser_simple_declaration not to.  */
 	cp_parser_require (parser, CPP_SEMICOLON, RT_SEMICOLON);
@@ -18551,11 +18558,6 @@ cp_parser_template_name (cp_parser* parser,
 static tree
 cp_parser_template_argument_list (cp_parser* parser)
 {
-  tree fixed_args[10];
-  unsigned n_args = 0;
-  unsigned alloced = 10;
-  tree *arg_ary = fixed_args;
-  tree vec;
   bool saved_in_template_argument_list_p;
   bool saved_ice_p;
   bool saved_non_ice_p;
@@ -18574,16 +18576,15 @@ cp_parser_template_argument_list (cp_parser* parser)
   parser->non_integral_constant_expression_p = false;
 
   /* Parse the arguments.  */
+  auto_vec<tree, 10> args;
   do
     {
-      tree argument;
-
-      if (n_args)
+      if (!args.is_empty ())
 	/* Consume the comma.  */
 	cp_lexer_consume_token (parser->lexer);
 
       /* Parse the template-argument.  */
-      argument = cp_parser_template_argument (parser);
+      tree argument = cp_parser_template_argument (parser);
 
       /* If the next token is an ellipsis, we're expanding a template
          argument pack. */
@@ -18603,29 +18604,16 @@ cp_parser_template_argument_list (cp_parser* parser)
           argument = make_pack_expansion (argument);
         }
 
-      if (n_args == alloced)
-	{
-	  alloced *= 2;
-
-	  if (arg_ary == fixed_args)
-	    {
-	      arg_ary = XNEWVEC (tree, alloced);
-	      memcpy (arg_ary, fixed_args, sizeof (tree) * n_args);
-	    }
-	  else
-	    arg_ary = XRESIZEVEC (tree, arg_ary, alloced);
-	}
-      arg_ary[n_args++] = argument;
+      args.safe_push (argument);
     }
   while (cp_lexer_next_token_is (parser->lexer, CPP_COMMA));
 
-  vec = make_tree_vec (n_args);
+  int n_args = args.length ();
+  tree vec = make_tree_vec (n_args);
 
-  while (n_args--)
-    TREE_VEC_ELT (vec, n_args) = arg_ary[n_args];
+  for (int i = 0; i < n_args; i++)
+    TREE_VEC_ELT (vec, i) = args[i];
 
-  if (arg_ary != fixed_args)
-    free (arg_ary);
   parser->non_integral_constant_expression_p = saved_non_ice_p;
   parser->integral_constant_expression_p = saved_ice_p;
   parser->in_template_argument_list_p = saved_in_template_argument_list_p;
@@ -19505,7 +19493,7 @@ cp_parser_simple_type_specifier (cp_parser* parser,
 	  else if (!flag_concepts)
 	    pedwarn (token->location, 0,
 		     "use of %<auto%> in parameter declaration "
-		     "only available with %<-fconcepts-ts%>");
+		     "only available with %<-std=c++20%> or %<-fconcepts%>");
 	}
       else
 	type = make_auto ();
@@ -19847,7 +19835,7 @@ cp_parser_simple_type_specifier (cp_parser* parser,
   Note that the Concepts TS allows the auto or decltype(auto) to be
   omitted in a constrained-type-specifier.  */
 
-tree
+static tree
 cp_parser_placeholder_type_specifier (cp_parser *parser, location_t loc,
 				      tree tmpl, bool tentative)
 {
@@ -19863,7 +19851,7 @@ cp_parser_placeholder_type_specifier (cp_parser *parser, location_t loc,
       args = TREE_OPERAND (tmpl, 1);
       tmpl = TREE_OPERAND (tmpl, 0);
     }
-  if (args == NULL_TREE)
+  else
     /* A concept-name with no arguments can't be an expression.  */
     tentative = false;
 
@@ -19901,8 +19889,11 @@ cp_parser_placeholder_type_specifier (cp_parser *parser, location_t loc,
       if (!flag_concepts_ts
 	  || !processing_template_parmlist)
 	{
-	  error_at (loc, "%qE does not constrain a type", DECL_NAME (con));
-	  inform (DECL_SOURCE_LOCATION (con), "concept defined here");
+	  if (!tentative)
+	    {
+	      error_at (loc, "%qE does not constrain a type", DECL_NAME (con));
+	      inform (DECL_SOURCE_LOCATION (con), "concept defined here");
+	    }
 	  return error_mark_node;
 	}
     }
@@ -38265,21 +38256,35 @@ cp_parser_omp_clause_orderedkind (cp_parser * /*parser*/,
 }
 
 /* OpenMP 4.0:
-   num_teams ( expression ) */
+   num_teams ( expression )
+
+   OpenMP 5.1:
+   num_teams ( expression : expression ) */
 
 static tree
 cp_parser_omp_clause_num_teams (cp_parser *parser, tree list,
 				location_t location)
 {
-  tree t, c;
+  tree upper, lower = NULL_TREE, c;
 
   matching_parens parens;
   if (!parens.require_open (parser))
     return list;
 
-  t = cp_parser_assignment_expression (parser);
+  bool saved_colon_corrects_to_scope_p = parser->colon_corrects_to_scope_p;
+  parser->colon_corrects_to_scope_p = false;
+  upper = cp_parser_assignment_expression (parser);
+  parser->colon_corrects_to_scope_p = saved_colon_corrects_to_scope_p;
 
-  if (t == error_mark_node
+  if (upper != error_mark_node
+      && cp_lexer_next_token_is (parser->lexer, CPP_COLON))
+    {
+      lower = upper;
+      cp_lexer_consume_token (parser->lexer);
+      upper = cp_parser_assignment_expression (parser);
+    }
+
+  if (upper == error_mark_node
       || !parens.require_close (parser))
     cp_parser_skip_to_closing_parenthesis (parser, /*recovering=*/true,
 					   /*or_comma=*/false,
@@ -38289,7 +38294,8 @@ cp_parser_omp_clause_num_teams (cp_parser *parser, tree list,
 			     "num_teams", location);
 
   c = build_omp_clause (location, OMP_CLAUSE_NUM_TEAMS);
-  OMP_CLAUSE_NUM_TEAMS_EXPR (c) = t;
+  OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (c) = upper;
+  OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c) = lower;
   OMP_CLAUSE_CHAIN (c) = list;
 
   return c;
@@ -44009,6 +44015,7 @@ cp_parser_omp_target_update (cp_parser *parser, cp_token *pragma_tok,
 	| (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_DEFAULTMAP)	\
 	| (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_ALLOCATE)	\
 	| (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_IN_REDUCTION)	\
+	| (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_THREAD_LIMIT)	\
 	| (OMP_CLAUSE_MASK_1 << PRAGMA_OMP_CLAUSE_IS_DEVICE_PTR))
 
 static bool
@@ -44094,32 +44101,33 @@ cp_parser_omp_target (cp_parser *parser, cp_token *pragma_tok,
 	  if (ret == NULL_TREE)
 	    return false;
 	  if (ccode == OMP_TEAMS && !processing_template_decl)
-	    {
-	      /* For combined target teams, ensure the num_teams and
-		 thread_limit clause expressions are evaluated on the host,
-		 before entering the target construct.  */
-	      tree c;
-	      for (c = cclauses[C_OMP_CLAUSE_SPLIT_TEAMS];
-		   c; c = OMP_CLAUSE_CHAIN (c))
-		if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS
-		     || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREAD_LIMIT)
-		    && TREE_CODE (OMP_CLAUSE_OPERAND (c, 0)) != INTEGER_CST)
-		  {
-		    tree expr = OMP_CLAUSE_OPERAND (c, 0);
-		    expr = force_target_expr (TREE_TYPE (expr), expr, tf_none);
-		    if (expr == error_mark_node)
-		      continue;
-		    tree tmp = TARGET_EXPR_SLOT (expr);
-		    add_stmt (expr);
-		    OMP_CLAUSE_OPERAND (c, 0) = expr;
-		    tree tc = build_omp_clause (OMP_CLAUSE_LOCATION (c),
-						OMP_CLAUSE_FIRSTPRIVATE);
-		    OMP_CLAUSE_DECL (tc) = tmp;
-		    OMP_CLAUSE_CHAIN (tc)
-		      = cclauses[C_OMP_CLAUSE_SPLIT_TARGET];
-		    cclauses[C_OMP_CLAUSE_SPLIT_TARGET] = tc;
-		  }
-	    }
+	    /* For combined target teams, ensure the num_teams and
+	       thread_limit clause expressions are evaluated on the host,
+	       before entering the target construct.  */
+	    for (tree c = cclauses[C_OMP_CLAUSE_SPLIT_TEAMS];
+		 c; c = OMP_CLAUSE_CHAIN (c))
+	      if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS
+		  || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREAD_LIMIT)
+		for (int i = 0;
+		     i <= (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS); ++i)
+		  if (OMP_CLAUSE_OPERAND (c, i)
+		      && TREE_CODE (OMP_CLAUSE_OPERAND (c, i)) != INTEGER_CST)
+		    {
+		      tree expr = OMP_CLAUSE_OPERAND (c, i);
+		      expr = force_target_expr (TREE_TYPE (expr), expr,
+						tf_none);
+		      if (expr == error_mark_node)
+			continue;
+		      tree tmp = TARGET_EXPR_SLOT (expr);
+		      add_stmt (expr);
+		      OMP_CLAUSE_OPERAND (c, i) = expr;
+		      tree tc = build_omp_clause (OMP_CLAUSE_LOCATION (c),
+						  OMP_CLAUSE_FIRSTPRIVATE);
+		      OMP_CLAUSE_DECL (tc) = tmp;
+		      OMP_CLAUSE_CHAIN (tc)
+			= cclauses[C_OMP_CLAUSE_SPLIT_TARGET];
+		      cclauses[C_OMP_CLAUSE_SPLIT_TARGET] = tc;
+		    }
 	  tree stmt = make_node (OMP_TARGET);
 	  TREE_TYPE (stmt) = void_type_node;
 	  OMP_TARGET_CLAUSES (stmt) = cclauses[C_OMP_CLAUSE_SPLIT_TARGET];
diff --git a/gcc/cp/pt.c b/gcc/cp/pt.c
index 287cf4c..82bf7dc 100644
--- a/gcc/cp/pt.c
+++ b/gcc/cp/pt.c
@@ -1831,13 +1831,11 @@ iterative_hash_template_arg (tree arg, hashval_t val)
 
     case CONSTRUCTOR:
       {
-	tree field, value;
-	unsigned i;
 	iterative_hash_template_arg (TREE_TYPE (arg), val);
-	FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (arg), i, field, value)
+	for (auto &e: CONSTRUCTOR_ELTS (arg))
 	  {
-	    val = iterative_hash_template_arg (field, val);
-	    val = iterative_hash_template_arg (value, val);
+	    val = iterative_hash_template_arg (e.index, val);
+	    val = iterative_hash_template_arg (e.value, val);
 	  }
 	return val;
       }
@@ -5879,12 +5877,19 @@ push_template_decl (tree decl, bool is_friend)
       if (check_for_bare_parameter_packs (TYPE_RAISES_EXCEPTIONS (type)))
 	TYPE_RAISES_EXCEPTIONS (type) = NULL_TREE;
     }
-  else if (check_for_bare_parameter_packs (is_typedef_decl (decl)
-					   ? DECL_ORIGINAL_TYPE (decl)
-					   : TREE_TYPE (decl)))
+  else
     {
-      TREE_TYPE (decl) = error_mark_node;
-      return error_mark_node;
+      if (check_for_bare_parameter_packs (is_typedef_decl (decl)
+					  ? DECL_ORIGINAL_TYPE (decl)
+					  : TREE_TYPE (decl)))
+	{
+	  TREE_TYPE (decl) = error_mark_node;
+	  return error_mark_node;
+	}
+
+      if (is_partial && VAR_P (decl)
+	  && check_for_bare_parameter_packs (DECL_TI_ARGS (decl)))
+	return error_mark_node;
     }
 
   if (is_partial)
@@ -7004,9 +7009,8 @@ invalid_tparm_referent_p (tree type, tree expr, tsubst_flags_t complain)
 
     case CONSTRUCTOR:
       {
-	unsigned i; tree elt;
-	FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (expr), i, elt)
-	  if (invalid_tparm_referent_p (TREE_TYPE (elt), elt, complain))
+	for (auto &e: CONSTRUCTOR_ELTS (expr))
+	  if (invalid_tparm_referent_p (TREE_TYPE (e.value), e.value, complain))
 	    return true;
       }
       break;
@@ -17433,6 +17437,13 @@ tsubst_omp_clauses (tree clauses, enum c_omp_region_type ort,
 	    = tsubst_omp_clause_decl (OMP_CLAUSE_DECL (oc), args, complain,
 				      in_decl, iterator_cache);
 	  break;
+	case OMP_CLAUSE_NUM_TEAMS:
+	  if (OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (oc))
+	    OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (nc)
+	      = tsubst_expr (OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (oc), args,
+			     complain, in_decl,
+			     /*integral_constant_expression_p=*/false);
+	  /* FALLTHRU */
 	case OMP_CLAUSE_TILE:
 	case OMP_CLAUSE_IF:
 	case OMP_CLAUSE_NUM_THREADS:
@@ -17441,7 +17452,6 @@ tsubst_omp_clauses (tree clauses, enum c_omp_region_type ort,
 	case OMP_CLAUSE_FINAL:
 	case OMP_CLAUSE_DEVICE:
 	case OMP_CLAUSE_DIST_SCHEDULE:
-	case OMP_CLAUSE_NUM_TEAMS:
 	case OMP_CLAUSE_THREAD_LIMIT:
 	case OMP_CLAUSE_SAFELEN:
 	case OMP_CLAUSE_SIMDLEN:
@@ -18944,31 +18954,32 @@ tsubst_expr (tree t, tree args, tsubst_flags_t complain, tree in_decl,
 	{
 	  tree teams = cp_walk_tree (&stmt, tsubst_find_omp_teams, NULL, NULL);
 	  if (teams)
-	    {
-	      /* For combined target teams, ensure the num_teams and
-		 thread_limit clause expressions are evaluated on the host,
-		 before entering the target construct.  */
-	      tree c;
-	      for (c = OMP_TEAMS_CLAUSES (teams);
-		   c; c = OMP_CLAUSE_CHAIN (c))
-		if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS
-		     || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREAD_LIMIT)
-		    && TREE_CODE (OMP_CLAUSE_OPERAND (c, 0)) != INTEGER_CST)
-		  {
-		    tree expr = OMP_CLAUSE_OPERAND (c, 0);
-		    expr = force_target_expr (TREE_TYPE (expr), expr, tf_none);
-		    if (expr == error_mark_node)
-		      continue;
-		    tmp = TARGET_EXPR_SLOT (expr);
-		    add_stmt (expr);
-		    OMP_CLAUSE_OPERAND (c, 0) = expr;
-		    tree tc = build_omp_clause (OMP_CLAUSE_LOCATION (c),
-						OMP_CLAUSE_FIRSTPRIVATE);
-		    OMP_CLAUSE_DECL (tc) = tmp;
-		    OMP_CLAUSE_CHAIN (tc) = OMP_TARGET_CLAUSES (t);
-		    OMP_TARGET_CLAUSES (t) = tc;
-		  }
-	    }
+	    /* For combined target teams, ensure the num_teams and
+	       thread_limit clause expressions are evaluated on the host,
+	       before entering the target construct.  */
+	    for (tree c = OMP_TEAMS_CLAUSES (teams);
+		 c; c = OMP_CLAUSE_CHAIN (c))
+	      if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS
+		  || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREAD_LIMIT)
+		for (int i = 0;
+		     i <= (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS); ++i)
+		  if (OMP_CLAUSE_OPERAND (c, i)
+		      && TREE_CODE (OMP_CLAUSE_OPERAND (c, i)) != INTEGER_CST)
+		    {
+		      tree expr = OMP_CLAUSE_OPERAND (c, i);
+		      expr = force_target_expr (TREE_TYPE (expr), expr,
+						tf_none);
+		      if (expr == error_mark_node)
+			continue;
+		      tmp = TARGET_EXPR_SLOT (expr);
+		      add_stmt (expr);
+		      OMP_CLAUSE_OPERAND (c, i) = expr;
+		      tree tc = build_omp_clause (OMP_CLAUSE_LOCATION (c),
+						  OMP_CLAUSE_FIRSTPRIVATE);
+		      OMP_CLAUSE_DECL (tc) = tmp;
+		      OMP_CLAUSE_CHAIN (tc) = OMP_TARGET_CLAUSES (t);
+		      OMP_TARGET_CLAUSES (t) = tc;
+		    }
 	}
       add_stmt (t);
       break;
@@ -23605,8 +23616,7 @@ unify (tree tparms, tree targs, tree parm, tree arg, int strict,
      we're dealing with a type. */
   if (BRACE_ENCLOSED_INITIALIZER_P (arg))
     {
-      tree elt, elttype;
-      unsigned i;
+      tree elttype;
       tree orig_parm = parm;
 
       if (!is_std_init_list (parm)
@@ -23633,8 +23643,9 @@ unify (tree tparms, tree targs, tree parm, tree arg, int strict,
 	/* If ELTTYPE has no deducible template parms, skip deduction from
 	   the list elements.  */;
       else
-	FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (arg), i, elt)
+	for (auto &e: CONSTRUCTOR_ELTS (arg))
 	  {
+	    tree elt = e.value;
 	    int elt_strict = strict;
 
 	    if (elt == error_mark_node)
@@ -27420,14 +27431,9 @@ type_dependent_expression_p (tree expression)
 
   if (BRACE_ENCLOSED_INITIALIZER_P (expression))
     {
-      tree elt;
-      unsigned i;
-
-      FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (expression), i, elt)
-	{
-	  if (type_dependent_expression_p (elt))
-	    return true;
-	}
+      for (auto &elt : CONSTRUCTOR_ELTS (expression))
+	if (type_dependent_expression_p (elt.value))
+	  return true;
       return false;
     }
 
@@ -28560,7 +28566,7 @@ static int
 extract_autos_r (tree t, void *data)
 {
   hash_table<auto_hash> &hash = *(hash_table<auto_hash>*)data;
-  if (is_auto (t))
+  if (is_auto (t) && !template_placeholder_p (t))
     {
       /* All the autos were built with index 0; fix that up now.  */
       tree *p = hash.find_slot (t, INSERT);
@@ -28594,10 +28600,8 @@ extract_autos (tree type)
   for_each_template_parm (type, extract_autos_r, &hash, &visited, true);
 
   tree tree_vec = make_tree_vec (hash.elements());
-  for (hash_table<auto_hash>::iterator iter = hash.begin();
-       iter != hash.end(); ++iter)
+  for (tree elt : hash)
     {
-      tree elt = *iter;
       unsigned i = TEMPLATE_PARM_IDX (TEMPLATE_TYPE_PARM_INDEX (elt));
       TREE_VEC_ELT (tree_vec, i)
 	= build_tree_list (NULL_TREE, TYPE_NAME (elt));
@@ -29837,7 +29841,7 @@ do_auto_deduction (tree type, tree init, tree auto_node,
       tree parms = build_tree_list (NULL_TREE, type);
       tree tparms;
 
-      if (flag_concepts)
+      if (flag_concepts_ts)
 	tparms = extract_autos (type);
       else
 	{
@@ -30025,7 +30029,7 @@ type_uses_auto (tree type)
 {
   if (type == NULL_TREE)
     return NULL_TREE;
-  else if (flag_concepts)
+  else if (flag_concepts_ts)
     {
       /* The Concepts TS allows multiple autos in one type-specifier; just
 	 return the first one we find, do_auto_deduction will collect all of
@@ -30048,6 +30052,11 @@ type_uses_auto (tree type)
 bool
 check_auto_in_tmpl_args (tree tmpl, tree args)
 {
+  if (!flag_concepts_ts)
+    /* Only the concepts TS allows 'auto' as a type-id; it'd otherwise
+       have already been rejected by the parser more generally.  */
+    return false;
+
   /* If there were previous errors, nevermind.  */
   if (!args || TREE_CODE (args) != TREE_VEC)
     return false;
@@ -30057,11 +30066,10 @@ check_auto_in_tmpl_args (tree tmpl, tree args)
      We'll only be able to tell during template substitution, so we
      expect to be called again then.  If concepts are enabled and we
      know we have a type, we're ok.  */
-  if (flag_concepts
-      && (identifier_p (tmpl)
-	  || (DECL_P (tmpl)
-	      &&  (DECL_TYPE_TEMPLATE_P (tmpl)
-		   || DECL_TEMPLATE_TEMPLATE_PARM_P (tmpl)))))
+  if (identifier_p (tmpl)
+      || (DECL_P (tmpl)
+	  &&  (DECL_TYPE_TEMPLATE_P (tmpl)
+	       || DECL_TEMPLATE_TEMPLATE_PARM_P (tmpl))))
     return false;
 
   /* Quickly search for any occurrences of auto; usually there won't
diff --git a/gcc/cp/ptree.c b/gcc/cp/ptree.c
index 1dcd764..ca7884d 100644
--- a/gcc/cp/ptree.c
+++ b/gcc/cp/ptree.c
@@ -292,7 +292,7 @@ cxx_print_xnode (FILE *file, tree node, int indent)
 	for (unsigned ix = 0; ix != len; ix++)
 	  {
 	    binding_cluster *cluster = &BINDING_VECTOR_CLUSTER (node, ix);
-	    char pfx[24];
+	    char pfx[32];
 	    for (unsigned jx = 0; jx != BINDING_VECTOR_SLOTS_PER_CLUSTER; jx++)
 	      if (cluster->indices[jx].span)
 		{
diff --git a/gcc/cp/semantics.c b/gcc/cp/semantics.c
index 0d8e5fd..1540442 100644
--- a/gcc/cp/semantics.c
+++ b/gcc/cp/semantics.c
@@ -3079,6 +3079,22 @@ finish_unary_op_expr (location_t op_loc, enum tree_code code, cp_expr expr,
   return result;
 }
 
+/* Return true if CONSTRUCTOR EXPR after pack expansion could have no
+   elements.  */
+
+static bool
+maybe_zero_constructor_nelts (tree expr)
+{
+  if (CONSTRUCTOR_NELTS (expr) == 0)
+    return true;
+  if (!processing_template_decl)
+    return false;
+  for (constructor_elt &elt : CONSTRUCTOR_ELTS (expr))
+    if (!PACK_EXPANSION_P (elt.value))
+      return false;
+  return true;
+}
+
 /* Finish a compound-literal expression or C++11 functional cast with aggregate
    initializer.  TYPE is the type to which the CONSTRUCTOR in COMPOUND_LITERAL
    is being cast.  */
@@ -3104,9 +3120,20 @@ finish_compound_literal (tree type, tree compound_literal,
 
   if (!TYPE_OBJ_P (type))
     {
-      if (complain & tf_error)
-	error ("compound literal of non-object type %qT", type);
-      return error_mark_node;
+      /* DR2351 */
+      if (VOID_TYPE_P (type) && CONSTRUCTOR_NELTS (compound_literal) == 0)
+	return void_node;
+      else if (VOID_TYPE_P (type)
+	       && processing_template_decl
+	       && maybe_zero_constructor_nelts (compound_literal))
+	/* If there are only packs in compound_literal, it could
+	   be void{} after pack expansion.  */;
+      else
+	{
+	  if (complain & tf_error)
+	    error ("compound literal of non-object type %qT", type);
+	  return error_mark_node;
+	}
     }
 
   if (template_placeholder_p (type))
@@ -7201,6 +7228,53 @@ finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 		}
 	      OMP_CLAUSE_OPERAND (c, 0) = t;
 	    }
+	  if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS
+	      && OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c)
+	      && !remove)
+	    {
+	      t = OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c);
+	      if (t == error_mark_node)
+		remove = true;
+	      else if (!type_dependent_expression_p (t)
+		       && !INTEGRAL_TYPE_P (TREE_TYPE (t)))
+		{
+		  error_at (OMP_CLAUSE_LOCATION (c),
+			    "%qs expression must be integral",
+			    omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
+		  remove = true;
+		}
+	      else
+		{
+		  t = mark_rvalue_use (t);
+		  if (!processing_template_decl)
+		    {
+		      t = maybe_constant_value (t);
+		      if (TREE_CODE (t) == INTEGER_CST
+			  && tree_int_cst_sgn (t) != 1)
+			{
+			  warning_at (OMP_CLAUSE_LOCATION (c), 0,
+				      "%qs value must be positive",
+				      omp_clause_code_name
+				      [OMP_CLAUSE_CODE (c)]);
+			  t = NULL_TREE;
+			}
+		      else
+			t = fold_build_cleanup_point_expr (TREE_TYPE (t), t);
+		      tree upper = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (c);
+		      if (t
+			  && TREE_CODE (t) == INTEGER_CST
+			  && TREE_CODE (upper) == INTEGER_CST
+			  && tree_int_cst_lt (upper, t))
+			{
+			  warning_at (OMP_CLAUSE_LOCATION (c), 0,
+				      "%<num_teams%> lower bound %qE bigger "
+				      "than upper bound %qE", t, upper);
+			  t = NULL_TREE;
+			}
+		    }
+		  OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c) = t;
+		}
+	    }
 	  break;
 
 	case OMP_CLAUSE_SCHEDULE:
@@ -9211,7 +9285,7 @@ handle_omp_for_class_iterator (int i, location_t locus, enum tree_code code,
 		TREE_OPERAND (cond, 1), iter);
       return true;
     }
-  if (!c_omp_check_loop_iv_exprs (locus, orig_declv, i,
+  if (!c_omp_check_loop_iv_exprs (locus, code, orig_declv, i,
 				  TREE_VEC_ELT (declv, i), NULL_TREE,
 				  cond, cp_walk_subtrees))
     return true;
@@ -9597,7 +9671,7 @@ finish_omp_for (location_t locus, enum tree_code code, tree declv,
       tree orig_init;
       FOR_EACH_VEC_ELT (*orig_inits, i, orig_init)
 	if (orig_init
-	    && !c_omp_check_loop_iv_exprs (locus,
+	    && !c_omp_check_loop_iv_exprs (locus, code,
 					   orig_declv ? orig_declv : declv, i,
 					   TREE_VEC_ELT (declv, i), orig_init,
 					   NULL_TREE, cp_walk_subtrees))
@@ -11308,7 +11382,8 @@ is_this_parameter (tree t)
 {
   if (!DECL_P (t) || DECL_NAME (t) != this_identifier)
     return false;
-  gcc_assert (TREE_CODE (t) == PARM_DECL || is_capture_proxy (t)
+  gcc_assert (TREE_CODE (t) == PARM_DECL
+	      || (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t))
 	      || (cp_binding_oracle && TREE_CODE (t) == VAR_DECL));
   return true;
 }
diff --git a/gcc/cp/typeck.c b/gcc/cp/typeck.c
index ab0f9da..cb20329 100644
--- a/gcc/cp/typeck.c
+++ b/gcc/cp/typeck.c
@@ -3460,12 +3460,9 @@ build_ptrmemfunc_access_expr (tree ptrmem, tree member_name)
 
   if (TREE_CODE (ptrmem) == CONSTRUCTOR)
     {
-      unsigned int ix;
-      tree index, value;
-      FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (ptrmem),
-				ix, index, value)
-	if (index && DECL_P (index) && DECL_NAME (index) == member_name)
-	  return value;
+      for (auto &e: CONSTRUCTOR_ELTS (ptrmem))
+	if (e.index && DECL_P (e.index) && DECL_NAME (e.index) == member_name)
+	  return e.value;
       gcc_unreachable ();
     }
 
@@ -6773,9 +6770,19 @@ cp_build_addr_expr_1 (tree arg, bool strict_lvalue, tsubst_flags_t complain)
 	    return error_mark_node;
 	  }
 
+	if (TREE_CODE (t) == FUNCTION_DECL
+	    && DECL_IMMEDIATE_FUNCTION_P (t)
+	    && !in_immediate_context ())
+	  {
+	    if (complain & tf_error)
+	      error_at (loc, "taking address of an immediate function %qD",
+			t);
+	    return error_mark_node;
+	  }
+
 	type = build_ptrmem_type (context_for_name_lookup (t),
 				  TREE_TYPE (t));
-	t = make_ptrmem_cst (type, TREE_OPERAND (arg, 1));
+	t = make_ptrmem_cst (type, t);
 	return t;
       }
 
@@ -6800,9 +6807,7 @@ cp_build_addr_expr_1 (tree arg, bool strict_lvalue, tsubst_flags_t complain)
       tree stripped_arg = tree_strip_any_location_wrapper (arg);
       if (TREE_CODE (stripped_arg) == FUNCTION_DECL
 	  && DECL_IMMEDIATE_FUNCTION_P (stripped_arg)
-	  && cp_unevaluated_operand == 0
-	  && (current_function_decl == NULL_TREE
-	      || !DECL_IMMEDIATE_FUNCTION_P (current_function_decl)))
+	  && !in_immediate_context ())
 	{
 	  if (complain & tf_error)
 	    error_at (loc, "taking address of an immediate function %qD",
diff --git a/gcc/cp/typeck2.c b/gcc/cp/typeck2.c
index c01f2f8..e98fbf7 100644
--- a/gcc/cp/typeck2.c
+++ b/gcc/cp/typeck2.c
@@ -484,6 +484,15 @@ split_nonconstant_init_1 (tree dest, tree init, bool nested)
 	   && TYPE_HAS_NONTRIVIAL_DESTRUCTOR (type))
 	  || vla_type_p (type))
 	{
+	  if (!TYPE_DOMAIN (type)
+	      && TREE_CODE (init) == CONSTRUCTOR
+	      && CONSTRUCTOR_NELTS (init))
+	    {
+	      /* Flexible array.  */
+	      cp_complete_array_type (&type, init, /*default*/true);
+	      dest = build1 (VIEW_CONVERT_EXPR, type, dest);
+	    }
+
 	  /* For an array, we only need/want a single cleanup region rather
 	     than one per element.  */
 	  tree code = build_vec_init (dest, NULL_TREE, init, false, 1,
diff --git a/gcc/cse.c b/gcc/cse.c
index 4c3988e..c1c7d0c 100644
--- a/gcc/cse.c
+++ b/gcc/cse.c
@@ -44,6 +44,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "regs.h"
 #include "function-abi.h"
 #include "rtlanal.h"
+#include "expr.h"
 
 /* The basic idea of common subexpression elimination is to go
    through the code, keeping a record of expressions that would
@@ -4239,14 +4240,21 @@ try_back_substitute_reg (rtx set, rtx_insn *insn)
 	}
     }
 }
-
+
+/* Add an entry containing RTL X into SETS.  */
+static inline void
+add_to_set (vec<struct set> *sets, rtx x)
+{
+  struct set entry = {};
+  entry.rtl = x;
+  sets->safe_push (entry);
+}
+
 /* Record all the SETs in this instruction into SETS_PTR,
    and return the number of recorded sets.  */
 static int
-find_sets_in_insn (rtx_insn *insn, struct set **psets)
+find_sets_in_insn (rtx_insn *insn, vec<struct set> *psets)
 {
-  struct set *sets = *psets;
-  int n_sets = 0;
   rtx x = PATTERN (insn);
 
   if (GET_CODE (x) == SET)
@@ -4266,8 +4274,25 @@ find_sets_in_insn (rtx_insn *insn, struct set **psets)
 	 someplace else, so it isn't worth cse'ing.  */
       else if (GET_CODE (SET_SRC (x)) == CALL)
 	;
+      else if (GET_CODE (SET_SRC (x)) == CONST_VECTOR
+	       && GET_MODE_CLASS (GET_MODE (SET_SRC (x))) != MODE_VECTOR_BOOL)
+	{
+	  /* First register the vector itself.  */
+	  add_to_set (psets, x);
+	  rtx src = SET_SRC (x);
+	  /* Go over the constants of the CONST_VECTOR in forward order, to
+	     put them in the same order in the SETS array.  */
+	  for (unsigned i = 0; i < const_vector_encoded_nelts (src) ; i++)
+	    {
+	      /* These are templates and don't actually get emitted but are
+		 used to tell CSE how to get to a particular constant.  */
+	      rtx y = simplify_gen_vec_select (SET_DEST (x), i);
+	      gcc_assert (y);
+	      add_to_set (psets, gen_rtx_SET (y, CONST_VECTOR_ELT (src, i)));
+	    }
+	}
       else
-	sets[n_sets++].rtl = x;
+	add_to_set (psets, x);
     }
   else if (GET_CODE (x) == PARALLEL)
     {
@@ -4288,12 +4313,12 @@ find_sets_in_insn (rtx_insn *insn, struct set **psets)
 	      else if (GET_CODE (SET_SRC (y)) == CALL)
 		;
 	      else
-		sets[n_sets++].rtl = y;
+		add_to_set (psets, y);
 	    }
 	}
     }
 
-  return n_sets;
+  return psets->length ();
 }
 
 /* Subroutine of canonicalize_insn.  X is an ASM_OPERANDS in INSN.  */
@@ -4341,9 +4366,10 @@ canon_asm_operands (rtx x, rtx_insn *insn)
    see canon_reg.  */
 
 static void
-canonicalize_insn (rtx_insn *insn, struct set **psets, int n_sets)
+canonicalize_insn (rtx_insn *insn, vec<struct set> *psets)
 {
-  struct set *sets = *psets;
+  vec<struct set> sets = *psets;
+  int n_sets = sets.length ();
   rtx tem;
   rtx x = PATTERN (insn);
   int i;
@@ -4502,13 +4528,6 @@ cse_insn (rtx_insn *insn)
   int src_eqv_in_memory = 0;
   unsigned src_eqv_hash = 0;
 
-  struct set *sets = (struct set *) 0;
-
-  if (GET_CODE (x) == SET)
-    sets = XALLOCA (struct set);
-  else if (GET_CODE (x) == PARALLEL)
-    sets = XALLOCAVEC (struct set, XVECLEN (x, 0));
-
   this_insn = insn;
 
   /* Find all regs explicitly clobbered in this insn,
@@ -4517,10 +4536,11 @@ cse_insn (rtx_insn *insn)
   invalidate_from_sets_and_clobbers (insn);
 
   /* Record all the SETs in this instruction.  */
-  n_sets = find_sets_in_insn (insn, &sets);
+  auto_vec<struct set, 8> sets;
+  n_sets = find_sets_in_insn (insn, (vec<struct set>*)&sets);
 
   /* Substitute the canonical register where possible.  */
-  canonicalize_insn (insn, &sets, n_sets);
+  canonicalize_insn (insn, (vec<struct set>*)&sets);
 
   /* If this insn has a REG_EQUAL note, store the equivalent value in SRC_EQV,
      if different, or if the DEST is a STRICT_LOW_PART/ZERO_EXTRACT.  The
@@ -4986,6 +5006,30 @@ cse_insn (rtx_insn *insn)
 	  src_related_is_const_anchor = src_related != NULL_RTX;
 	}
 
+      /* Try to re-materialize a vec_dup with an existing constant.   */
+      rtx src_elt;
+      if ((!src_eqv_here || CONSTANT_P (src_eqv_here))
+	  && const_vec_duplicate_p (src, &src_elt))
+	{
+	   machine_mode const_mode = GET_MODE_INNER (GET_MODE (src));
+	   struct table_elt *related_elt
+		= lookup (src_elt, HASH (src_elt, const_mode), const_mode);
+	   if (related_elt)
+	    {
+	      for (related_elt = related_elt->first_same_value;
+		   related_elt; related_elt = related_elt->next_same_value)
+		if (REG_P (related_elt->exp))
+		  {
+		   /* We don't need to compare costs with an existing (constant)
+		      src_eqv_here, since any such src_eqv_here should already be
+		      available in src_const.  */
+		    src_eqv_here
+			= gen_rtx_VEC_DUPLICATE (GET_MODE (src),
+						 related_elt->exp);
+		    break;
+		  }
+	    }
+	}
 
       if (src == src_folded)
 	src_folded = 0;
diff --git a/gcc/d/ChangeLog b/gcc/d/ChangeLog
index c045cc9..b752dd9 100644
--- a/gcc/d/ChangeLog
+++ b/gcc/d/ChangeLog
@@ -1,3 +1,9 @@
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Make-lang.in: Allow dvi-formatted documentation
+	to be installed.
+
 2021-10-08  Martin Liska  <mliska@suse.cz>
 
 	* d-lang.cc (d_post_options): Use new macro
diff --git a/gcc/d/Make-lang.in b/gcc/d/Make-lang.in
index b3c77a0..554a26e 100644
--- a/gcc/d/Make-lang.in
+++ b/gcc/d/Make-lang.in
@@ -268,6 +268,16 @@ d.install-pdf: doc/gdc.pdf
 	  $(INSTALL_DATA) "$$d$$p" "$(DESTDIR)$(pdfdir)/gcc/$$f"; \
 	done
 
+d.install-dvi: doc/gdc.dvi
+	@$(NORMAL_INSTALL)
+	test -z "$(dvidir)" || $(mkinstalldirs) "$(DESTDIR)$(dvidir)/gcc"
+	@for p in doc/gdc.dvi; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  f=$(dvi__strip_dir) \
+	  echo " $(INSTALL_DATA) '$$d$$p' '$(DESTDIR)$(dvidir)/gcc/$$f'"; \
+	  $(INSTALL_DATA) "$$d$$p" "$(DESTDIR)$(dvidir)/gcc/$$f"; \
+	done
+
 d.install-html: $(build_htmldir)/d
 	@$(NORMAL_INSTALL)
 	test -z "$(htmldir)" || $(mkinstalldirs) "$(DESTDIR)$(htmldir)"
diff --git a/gcc/dbgcnt.def b/gcc/dbgcnt.def
index c2bcc4e..f8a15f3 100644
--- a/gcc/dbgcnt.def
+++ b/gcc/dbgcnt.def
@@ -144,6 +144,10 @@ echo ubound: $ub
    Please keep the list sorted in alphabetic order.  */
 DEBUG_COUNTER (asan_use_after_scope)
 DEBUG_COUNTER (auto_inc_dec)
+DEBUG_COUNTER (back_thread1)
+DEBUG_COUNTER (back_thread2)
+DEBUG_COUNTER (back_threadfull1)
+DEBUG_COUNTER (back_threadfull2)
 DEBUG_COUNTER (ccp)
 DEBUG_COUNTER (cfg_cleanup)
 DEBUG_COUNTER (cprop)
@@ -171,6 +175,7 @@ DEBUG_COUNTER (if_after_reload)
 DEBUG_COUNTER (if_conversion)
 DEBUG_COUNTER (if_conversion_tree)
 DEBUG_COUNTER (if_to_switch)
+DEBUG_COUNTER (ipa_attr)
 DEBUG_COUNTER (ipa_cp_bits)
 DEBUG_COUNTER (ipa_cp_values)
 DEBUG_COUNTER (ipa_cp_vr)
diff --git a/gcc/diagnostic-format-json.cc b/gcc/diagnostic-format-json.cc
index 6da96f7..eff1907 100644
--- a/gcc/diagnostic-format-json.cc
+++ b/gcc/diagnostic-format-json.cc
@@ -264,6 +264,9 @@ json_end_diagnostic (diagnostic_context *context, diagnostic_info *diagnostic,
       json::value *path_value = context->make_json_for_path (context, path);
       diag_obj->set ("path", path_value);
     }
+
+  diag_obj->set ("escape-source",
+		 new json::literal (richloc->escape_on_output_p ()));
 }
 
 /* No-op implementation of "begin_group_cb" for JSON output.  */
diff --git a/gcc/diagnostic-show-locus.c b/gcc/diagnostic-show-locus.c
index 24bd031..67be213 100644
--- a/gcc/diagnostic-show-locus.c
+++ b/gcc/diagnostic-show-locus.c
@@ -175,10 +175,26 @@ enum column_unit {
 class exploc_with_display_col : public expanded_location
 {
  public:
-  exploc_with_display_col (const expanded_location &exploc, int tabstop)
-    : expanded_location (exploc),
-      m_display_col (location_compute_display_column (exploc, tabstop))
-  {}
+  exploc_with_display_col (const expanded_location &exploc,
+			   const cpp_char_column_policy &policy,
+			   enum location_aspect aspect)
+  : expanded_location (exploc),
+    m_display_col (location_compute_display_column (exploc, policy))
+  {
+    if (exploc.column > 0)
+      {
+	/* m_display_col is now the final column of the byte.
+	   If escaping has happened, we may want the first column instead.  */
+	if (aspect != LOCATION_ASPECT_FINISH)
+	  {
+	    expanded_location prev_exploc (exploc);
+	    prev_exploc.column--;
+	    int prev_display_col
+	      = (location_compute_display_column (prev_exploc, policy));
+	    m_display_col = prev_display_col + 1;
+	  }
+      }
+  }
 
   int m_display_col;
 };
@@ -313,6 +329,31 @@ test_line_span ()
 
 #endif /* #if CHECKING_P */
 
+/* A bundle of information containing how to print unicode
+   characters and bytes when quoting source code.
+
+   Provides a unified place to support escaping some subset
+   of characters to some format.
+
+   Extends char_column_policy; printing is split out to avoid
+   libcpp having to know about pretty_printer.  */
+
+struct char_display_policy : public cpp_char_column_policy
+{
+ public:
+  char_display_policy (int tabstop,
+		       int (*width_cb) (cppchar_t c),
+		       void (*print_cb) (pretty_printer *pp,
+					 const cpp_decoded_char &cp))
+  : cpp_char_column_policy (tabstop, width_cb),
+    m_print_cb (print_cb)
+  {
+  }
+
+  void (*m_print_cb) (pretty_printer *pp,
+		      const cpp_decoded_char &cp);
+};
+
 /* A class to control the overall layout when printing a diagnostic.
 
    The layout is determined within the constructor.
@@ -345,6 +386,8 @@ class layout
 
   void print_line (linenum_type row);
 
+  void on_bad_codepoint (const char *ptr, cppchar_t ch, size_t ch_sz);
+
  private:
   bool will_show_line_p (linenum_type row) const;
   void print_leading_fixits (linenum_type row);
@@ -386,6 +429,7 @@ class layout
  private:
   diagnostic_context *m_context;
   pretty_printer *m_pp;
+  char_display_policy m_policy;
   location_t m_primary_loc;
   exploc_with_display_col m_exploc;
   colorizer m_colorizer;
@@ -398,6 +442,7 @@ class layout
   auto_vec <line_span> m_line_spans;
   int m_linenum_width;
   int m_x_offset_display;
+  bool m_escape_on_output;
 };
 
 /* Implementation of "class colorizer".  */
@@ -646,6 +691,11 @@ layout_range::intersects_line_p (linenum_type row) const
 /* Default for when we don't care what the tab expansion is set to.  */
 static const int def_tabstop = 8;
 
+static cpp_char_column_policy def_policy ()
+{
+  return cpp_char_column_policy (def_tabstop, cpp_wcwidth);
+}
+
 /* Create some expanded locations for testing layout_range.  The filename
    member of the explocs is set to the empty string.  This member will only be
    inspected by the calls to location_compute_display_column() made from the
@@ -662,10 +712,13 @@ make_range (int start_line, int start_col, int end_line, int end_col)
     = {"", start_line, start_col, NULL, false};
   const expanded_location finish_exploc
     = {"", end_line, end_col, NULL, false};
-  return layout_range (exploc_with_display_col (start_exploc, def_tabstop),
-		       exploc_with_display_col (finish_exploc, def_tabstop),
+  return layout_range (exploc_with_display_col (start_exploc, def_policy (),
+						LOCATION_ASPECT_START),
+		       exploc_with_display_col (finish_exploc, def_policy (),
+						LOCATION_ASPECT_FINISH),
 		       SHOW_RANGE_WITHOUT_CARET,
-		       exploc_with_display_col (start_exploc, def_tabstop),
+		       exploc_with_display_col (start_exploc, def_policy (),
+						LOCATION_ASPECT_CARET),
 		       0, NULL);
 }
 
@@ -959,6 +1012,164 @@ fixit_cmp (const void *p_a, const void *p_b)
   return hint_a->get_start_loc () - hint_b->get_start_loc ();
 }
 
+/* Callbacks for use when not escaping the source.  */
+
+/* The default callback for char_column_policy::m_width_cb is cpp_wcwidth.  */
+
+/* Callback for char_display_policy::m_print_cb for printing source chars
+   when not escaping the source.  */
+
+static void
+default_print_decoded_ch (pretty_printer *pp,
+			  const cpp_decoded_char &decoded_ch)
+{
+  for (const char *ptr = decoded_ch.m_start_byte;
+       ptr != decoded_ch.m_next_byte; ptr++)
+    {
+      if (*ptr == '\0' || *ptr == '\r')
+	{
+	  pp_space (pp);
+	  continue;
+	}
+
+      pp_character (pp, *ptr);
+    }
+}
+
+/* Callbacks for use with DIAGNOSTICS_ESCAPE_FORMAT_BYTES.  */
+
+static const int width_per_escaped_byte = 4;
+
+/* Callback for char_column_policy::m_width_cb for determining the
+   display width when escaping with DIAGNOSTICS_ESCAPE_FORMAT_BYTES.  */
+
+static int
+escape_as_bytes_width (cppchar_t ch)
+{
+  if (ch < 0x80 && ISPRINT (ch))
+    return cpp_wcwidth (ch);
+  else
+    {
+      if (ch <=   0x7F) return 1 * width_per_escaped_byte;
+      if (ch <=  0x7FF) return 2 * width_per_escaped_byte;
+      if (ch <= 0xFFFF) return 3 * width_per_escaped_byte;
+      return 4 * width_per_escaped_byte;
+    }
+}
+
+/* Callback for char_display_policy::m_print_cb for printing source chars
+   when escaping with DIAGNOSTICS_ESCAPE_FORMAT_BYTES.  */
+
+static void
+escape_as_bytes_print (pretty_printer *pp,
+		       const cpp_decoded_char &decoded_ch)
+{
+  if (!decoded_ch.m_valid_ch)
+    {
+      for (const char *iter = decoded_ch.m_start_byte;
+	   iter != decoded_ch.m_next_byte; ++iter)
+	{
+	  char buf[16];
+	  sprintf (buf, "<%02x>", (unsigned char)*iter);
+	  pp_string (pp, buf);
+	}
+      return;
+    }
+
+  cppchar_t ch = decoded_ch.m_ch;
+  if (ch < 0x80 && ISPRINT (ch))
+    pp_character (pp, ch);
+  else
+    {
+      for (const char *iter = decoded_ch.m_start_byte;
+	   iter < decoded_ch.m_next_byte; ++iter)
+	{
+	  char buf[16];
+	  sprintf (buf, "<%02x>", (unsigned char)*iter);
+	  pp_string (pp, buf);
+	}
+    }
+}
+
+/* Callbacks for use with DIAGNOSTICS_ESCAPE_FORMAT_UNICODE.  */
+
+/* Callback for char_column_policy::m_width_cb for determining the
+   display width when escaping with DIAGNOSTICS_ESCAPE_FORMAT_UNICODE.  */
+
+static int
+escape_as_unicode_width (cppchar_t ch)
+{
+  if (ch < 0x80 && ISPRINT (ch))
+    return cpp_wcwidth (ch);
+  else
+    {
+      // Width of "<U+%04x>"
+      if (ch > 0xfffff)
+	return 10;
+      else if (ch > 0xffff)
+	return 9;
+      else
+	return 8;
+    }
+}
+
+/* Callback for char_display_policy::m_print_cb for printing source chars
+   when escaping with DIAGNOSTICS_ESCAPE_FORMAT_UNICODE.  */
+
+static void
+escape_as_unicode_print (pretty_printer *pp,
+			 const cpp_decoded_char &decoded_ch)
+{
+  if (!decoded_ch.m_valid_ch)
+    {
+      escape_as_bytes_print (pp, decoded_ch);
+      return;
+    }
+
+  cppchar_t ch = decoded_ch.m_ch;
+  if (ch < 0x80 && ISPRINT (ch))
+    pp_character (pp, ch);
+  else
+    {
+      char buf[16];
+      sprintf (buf, "<U+%04X>", ch);
+      pp_string (pp, buf);
+    }
+}
+
+/* Populate a char_display_policy based on DC and RICHLOC.  */
+
+static char_display_policy
+make_policy (const diagnostic_context &dc,
+	     const rich_location &richloc)
+{
+  /* The default is to not escape non-ASCII bytes.  */
+  char_display_policy result
+    (dc.tabstop, cpp_wcwidth, default_print_decoded_ch);
+
+  /* If the diagnostic suggests escaping non-ASCII bytes, then
+     use policy from user-supplied options.  */
+  if (richloc.escape_on_output_p ())
+    {
+      result.m_undecoded_byte_width = width_per_escaped_byte;
+      switch (dc.escape_format)
+	{
+	default:
+	  gcc_unreachable ();
+	case DIAGNOSTICS_ESCAPE_FORMAT_UNICODE:
+	  result.m_width_cb = escape_as_unicode_width;
+	  result.m_print_cb = escape_as_unicode_print;
+	  break;
+	case DIAGNOSTICS_ESCAPE_FORMAT_BYTES:
+	  result.m_width_cb = escape_as_bytes_width;
+	  result.m_print_cb = escape_as_bytes_print;
+	  break;
+	}
+    }
+
+  return result;
+}
+
 /* Implementation of class layout.  */
 
 /* Constructor for class layout.
@@ -975,8 +1186,10 @@ layout::layout (diagnostic_context * context,
 		diagnostic_t diagnostic_kind)
 : m_context (context),
   m_pp (context->printer),
+  m_policy (make_policy (*context, *richloc)),
   m_primary_loc (richloc->get_range (0)->m_loc),
-  m_exploc (richloc->get_expanded_location (0), context->tabstop),
+  m_exploc (richloc->get_expanded_location (0), m_policy,
+	    LOCATION_ASPECT_CARET),
   m_colorizer (context, diagnostic_kind),
   m_colorize_source_p (context->colorize_source_p),
   m_show_labels_p (context->show_labels_p),
@@ -986,7 +1199,8 @@ layout::layout (diagnostic_context * context,
   m_fixit_hints (richloc->get_num_fixit_hints ()),
   m_line_spans (1 + richloc->get_num_locations ()),
   m_linenum_width (0),
-  m_x_offset_display (0)
+  m_x_offset_display (0),
+  m_escape_on_output (richloc->escape_on_output_p ())
 {
   for (unsigned int idx = 0; idx < richloc->get_num_locations (); idx++)
     {
@@ -1072,10 +1286,13 @@ layout::maybe_add_location_range (const location_range *loc_range,
 
   /* Everything is now known to be in the correct source file,
      but it may require further sanitization.  */
-  layout_range ri (exploc_with_display_col (start, m_context->tabstop),
-		   exploc_with_display_col (finish, m_context->tabstop),
+  layout_range ri (exploc_with_display_col (start, m_policy,
+					    LOCATION_ASPECT_START),
+		   exploc_with_display_col (finish, m_policy,
+					    LOCATION_ASPECT_FINISH),
 		   loc_range->m_range_display_kind,
-		   exploc_with_display_col (caret, m_context->tabstop),
+		   exploc_with_display_col (caret, m_policy,
+					    LOCATION_ASPECT_CARET),
 		   original_idx, loc_range->m_label);
 
   /* If we have a range that finishes before it starts (perhaps
@@ -1409,7 +1626,7 @@ layout::calculate_x_offset_display ()
     = get_line_bytes_without_trailing_whitespace (line.get_buffer (),
 						  line.length ());
   int eol_display_column
-    = cpp_display_width (line.get_buffer (), line_bytes, m_context->tabstop);
+    = cpp_display_width (line.get_buffer (), line_bytes, m_policy);
   if (caret_display_column > eol_display_column
       || !caret_display_column)
     {
@@ -1488,7 +1705,7 @@ layout::print_source_line (linenum_type row, const char *line, int line_bytes)
   /* This object helps to keep track of which display column we are at, which is
      necessary for computing the line bounds in display units, for doing
      tab expansion, and for implementing m_x_offset_display.  */
-  cpp_display_width_computation dw (line, line_bytes, m_context->tabstop);
+  cpp_display_width_computation dw (line, line_bytes, m_policy);
 
   /* Skip the first m_x_offset_display display columns.  In case the leading
      portion that will be skipped ends with a character with wcwidth > 1, then
@@ -1536,7 +1753,8 @@ layout::print_source_line (linenum_type row, const char *line, int line_bytes)
 	 tabs and replacing some control bytes with spaces as necessary.  */
       const char *c = dw.next_byte ();
       const int start_disp_col = dw.display_cols_processed () + 1;
-      const int this_display_width = dw.process_next_codepoint ();
+      cpp_decoded_char cp;
+      const int this_display_width = dw.process_next_codepoint (&cp);
       if (*c == '\t')
 	{
 	  /* The returned display width is the number of spaces into which the
@@ -1545,15 +1763,6 @@ layout::print_source_line (linenum_type row, const char *line, int line_bytes)
 	    pp_space (m_pp);
 	  continue;
 	}
-      if (*c == '\0' || *c == '\r')
-	{
-	  /* cpp_wcwidth() promises to return 1 for all control bytes, and we
-	     want to output these as a single space too, so this case is
-	     actually the same as the '\t' case.  */
-	  gcc_assert (this_display_width == 1);
-	  pp_space (m_pp);
-	  continue;
-	}
 
       /* We have a (possibly multibyte) character to output; update the line
 	 bounds if it is not whitespace.  */
@@ -1565,7 +1774,8 @@ layout::print_source_line (linenum_type row, const char *line, int line_bytes)
 	}
 
       /* Output the character.  */
-      while (c != dw.next_byte ()) pp_character (m_pp, *c++);
+      m_policy.m_print_cb (m_pp, cp);
+      c = dw.next_byte ();
     }
   print_newline ();
   return lbounds;
@@ -1664,14 +1874,14 @@ layout::print_annotation_line (linenum_type row, const line_bounds lbounds)
 class line_label
 {
 public:
-  line_label (diagnostic_context *context, int state_idx, int column,
+  line_label (const cpp_char_column_policy &policy,
+	      int state_idx, int column,
 	      label_text text)
   : m_state_idx (state_idx), m_column (column),
     m_text (text), m_label_line (0), m_has_vbar (true)
   {
     const int bytes = strlen (text.m_buffer);
-    m_display_width
-      = cpp_display_width (text.m_buffer, bytes, context->tabstop);
+    m_display_width = cpp_display_width (text.m_buffer, bytes, policy);
   }
 
   /* Sorting is primarily by column, then by state index.  */
@@ -1731,7 +1941,7 @@ layout::print_any_labels (linenum_type row)
 	if (text.m_buffer == NULL)
 	  continue;
 
-	labels.safe_push (line_label (m_context, i, disp_col, text));
+	labels.safe_push (line_label (m_policy, i, disp_col, text));
       }
   }
 
@@ -2011,7 +2221,7 @@ public:
 
 /* Get the range of bytes or display columns that HINT would affect.  */
 static column_range
-get_affected_range (diagnostic_context *context,
+get_affected_range (const cpp_char_column_policy &policy,
 		    const fixit_hint *hint, enum column_unit col_unit)
 {
   expanded_location exploc_start = expand_location (hint->get_start_loc ());
@@ -2022,13 +2232,11 @@ get_affected_range (diagnostic_context *context,
   int finish_column;
   if (col_unit == CU_DISPLAY_COLS)
     {
-      start_column
-	= location_compute_display_column (exploc_start, context->tabstop);
+      start_column = location_compute_display_column (exploc_start, policy);
       if (hint->insertion_p ())
 	finish_column = start_column - 1;
       else
-	finish_column
-	  = location_compute_display_column (exploc_finish, context->tabstop);
+	finish_column = location_compute_display_column (exploc_finish, policy);
     }
   else
     {
@@ -2041,12 +2249,13 @@ get_affected_range (diagnostic_context *context,
 /* Get the range of display columns that would be printed for HINT.  */
 
 static column_range
-get_printed_columns (diagnostic_context *context, const fixit_hint *hint)
+get_printed_columns (const cpp_char_column_policy &policy,
+		     const fixit_hint *hint)
 {
   expanded_location exploc = expand_location (hint->get_start_loc ());
-  int start_column = location_compute_display_column (exploc, context->tabstop);
+  int start_column = location_compute_display_column (exploc, policy);
   int hint_width = cpp_display_width (hint->get_string (), hint->get_length (),
-				      context->tabstop);
+				      policy);
   int final_hint_column = start_column + hint_width - 1;
   if (hint->insertion_p ())
     {
@@ -2056,8 +2265,7 @@ get_printed_columns (diagnostic_context *context, const fixit_hint *hint)
     {
       exploc = expand_location (hint->get_next_loc ());
       --exploc.column;
-      int finish_column
-	= location_compute_display_column (exploc, context->tabstop);
+      int finish_column = location_compute_display_column (exploc, policy);
       return column_range (start_column,
 			   MAX (finish_column, final_hint_column));
     }
@@ -2075,13 +2283,13 @@ public:
 	      column_range affected_columns,
 	      column_range printed_columns,
 	      const char *new_text, size_t new_text_len,
-	      int tabstop)
+	      const cpp_char_column_policy &policy)
   : m_affected_bytes (affected_bytes),
     m_affected_columns (affected_columns),
     m_printed_columns (printed_columns),
     m_text (xstrdup (new_text)),
     m_byte_length (new_text_len),
-    m_tabstop (tabstop),
+    m_policy (policy),
     m_alloc_sz (new_text_len + 1)
   {
     compute_display_cols ();
@@ -2099,7 +2307,7 @@ public:
 
   void compute_display_cols ()
   {
-    m_display_cols = cpp_display_width (m_text, m_byte_length, m_tabstop);
+    m_display_cols = cpp_display_width (m_text, m_byte_length, m_policy);
   }
 
   void overwrite (int dst_offset, const char_span &src_span)
@@ -2127,7 +2335,7 @@ public:
   char *m_text;
   size_t m_byte_length; /* Not including null-terminator.  */
   int m_display_cols;
-  int m_tabstop;
+  const cpp_char_column_policy &m_policy;
   size_t m_alloc_sz;
 };
 
@@ -2163,15 +2371,16 @@ correction::ensure_terminated ()
 class line_corrections
 {
 public:
-  line_corrections (diagnostic_context *context, const char *filename,
+  line_corrections (const char_display_policy &policy,
+		    const char *filename,
 		    linenum_type row)
-    : m_context (context), m_filename (filename), m_row (row)
+  : m_policy (policy), m_filename (filename), m_row (row)
   {}
   ~line_corrections ();
 
   void add_hint (const fixit_hint *hint);
 
-  diagnostic_context *m_context;
+  const char_display_policy &m_policy;
   const char *m_filename;
   linenum_type m_row;
   auto_vec <correction *> m_corrections;
@@ -2217,10 +2426,10 @@ source_line::source_line (const char *filename, int line)
 void
 line_corrections::add_hint (const fixit_hint *hint)
 {
-  column_range affected_bytes = get_affected_range (m_context, hint, CU_BYTES);
-  column_range affected_columns = get_affected_range (m_context, hint,
+  column_range affected_bytes = get_affected_range (m_policy, hint, CU_BYTES);
+  column_range affected_columns = get_affected_range (m_policy, hint,
 						      CU_DISPLAY_COLS);
-  column_range printed_columns = get_printed_columns (m_context, hint);
+  column_range printed_columns = get_printed_columns (m_policy, hint);
 
   /* Potentially consolidate.  */
   if (!m_corrections.is_empty ())
@@ -2289,7 +2498,7 @@ line_corrections::add_hint (const fixit_hint *hint)
 					   printed_columns,
 					   hint->get_string (),
 					   hint->get_length (),
-					   m_context->tabstop));
+					   m_policy));
 }
 
 /* If there are any fixit hints on source line ROW, print them.
@@ -2303,7 +2512,7 @@ layout::print_trailing_fixits (linenum_type row)
 {
   /* Build a list of correction instances for the line,
      potentially consolidating hints (for the sake of readability).  */
-  line_corrections corrections (m_context, m_exploc.file, row);
+  line_corrections corrections (m_policy, m_exploc.file, row);
   for (unsigned int i = 0; i < m_fixit_hints.length (); i++)
     {
       const fixit_hint *hint = m_fixit_hints[i];
@@ -2646,6 +2855,59 @@ namespace selftest {
 
 /* Selftests for diagnostic_show_locus.  */
 
+/* Verify that cpp_display_width correctly handles escaping.  */
+
+static void
+test_display_widths ()
+{
+  gcc_rich_location richloc (UNKNOWN_LOCATION);
+
+  /* U+03C0 "GREEK SMALL LETTER PI".  */
+  const char *pi = "\xCF\x80";
+  /* U+1F642 "SLIGHTLY SMILING FACE".  */
+  const char *emoji = "\xF0\x9F\x99\x82";
+  /* Stray trailing byte of a UTF-8 character.  */
+  const char *stray = "\xBF";
+  /* U+10FFFF.  */
+  const char *max_codepoint = "\xF4\x8F\xBF\xBF";
+
+  /* No escaping.  */
+  {
+    test_diagnostic_context dc;
+    char_display_policy policy (make_policy (dc, richloc));
+    ASSERT_EQ (cpp_display_width (pi, strlen (pi), policy), 1);
+    ASSERT_EQ (cpp_display_width (emoji, strlen (emoji), policy), 2);
+    ASSERT_EQ (cpp_display_width (stray, strlen (stray), policy), 1);
+    /* Don't check width of U+10FFFF; it's in a private use plane.  */
+  }
+
+  richloc.set_escape_on_output (true);
+
+  {
+    test_diagnostic_context dc;
+    dc.escape_format = DIAGNOSTICS_ESCAPE_FORMAT_UNICODE;
+    char_display_policy policy (make_policy (dc, richloc));
+    ASSERT_EQ (cpp_display_width (pi, strlen (pi), policy), 8);
+    ASSERT_EQ (cpp_display_width (emoji, strlen (emoji), policy), 9);
+    ASSERT_EQ (cpp_display_width (stray, strlen (stray), policy), 4);
+    ASSERT_EQ (cpp_display_width (max_codepoint, strlen (max_codepoint),
+				  policy),
+	       strlen ("<U+10FFFF>"));
+  }
+
+  {
+    test_diagnostic_context dc;
+    dc.escape_format = DIAGNOSTICS_ESCAPE_FORMAT_BYTES;
+    char_display_policy policy (make_policy (dc, richloc));
+    ASSERT_EQ (cpp_display_width (pi, strlen (pi), policy), 8);
+    ASSERT_EQ (cpp_display_width (emoji, strlen (emoji), policy), 16);
+    ASSERT_EQ (cpp_display_width (stray, strlen (stray), policy), 4);
+    ASSERT_EQ (cpp_display_width (max_codepoint, strlen (max_codepoint),
+				  policy),
+	       16);
+  }
+}
+
 /* For precise tests of the layout, make clear where the source line will
    start.  test_left_margin sets the total byte count from the left side of the
    screen to the start of source lines, after the line number and the separator,
@@ -2715,10 +2977,10 @@ test_layout_x_offset_display_utf8 (const line_table_case &case_)
   char_span lspan = location_get_source_line (tmp.get_filename (), 1);
   ASSERT_EQ (line_display_cols,
 	     cpp_display_width (lspan.get_buffer (), lspan.length (),
-				def_tabstop));
+				def_policy ()));
   ASSERT_EQ (line_display_cols,
 	     location_compute_display_column (expand_location (line_end),
-					      def_tabstop));
+					      def_policy ()));
   ASSERT_EQ (0, memcmp (lspan.get_buffer () + (emoji_col - 1),
 			"\xf0\x9f\x98\x82\xf0\x9f\x98\x82", 8));
 
@@ -2866,12 +3128,13 @@ test_layout_x_offset_display_tab (const line_table_case &case_)
   ASSERT_EQ ('\t', *(lspan.get_buffer () + (tab_col - 1)));
   for (int tabstop = 1; tabstop != num_tabstops; ++tabstop)
     {
+      cpp_char_column_policy policy (tabstop, cpp_wcwidth);
       ASSERT_EQ (line_bytes + extra_width[tabstop],
 		 cpp_display_width (lspan.get_buffer (), lspan.length (),
-				    tabstop));
+				    policy));
       ASSERT_EQ (line_bytes + extra_width[tabstop],
 		 location_compute_display_column (expand_location (line_end),
-						  tabstop));
+						  policy));
     }
 
   /* Check that the tab is expanded to the expected number of spaces.  */
@@ -4003,6 +4266,43 @@ test_one_liner_labels_utf8 ()
 			   " bb\xf0\x9f\x98\x82\xf0\x9f\x98\x82\n",
 		  pp_formatted_text (dc.printer));
   }
+
+  /* Example of escaping the source lines.  */
+  {
+    text_range_label label0 ("label 0\xf0\x9f\x98\x82");
+    text_range_label label1 ("label 1\xcf\x80");
+    text_range_label label2 ("label 2\xcf\x80");
+    gcc_rich_location richloc (foo, &label0);
+    richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
+    richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
+    richloc.set_escape_on_output (true);
+
+    {
+      test_diagnostic_context dc;
+      dc.escape_format = DIAGNOSTICS_ESCAPE_FORMAT_UNICODE;
+      diagnostic_show_locus (&dc, &richloc, DK_ERROR);
+      ASSERT_STREQ (" <U+1F602>_foo = <U+03C0>_bar.<U+1F602>_field<U+03C0>;\n"
+		    " ^~~~~~~~~~~~~   ~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~\n"
+		    " |               |            |\n"
+		    " |               |            label 2\xcf\x80\n"
+		    " |               label 1\xcf\x80\n"
+		    " label 0\xf0\x9f\x98\x82\n",
+		    pp_formatted_text (dc.printer));
+    }
+    {
+      test_diagnostic_context dc;
+      dc.escape_format = DIAGNOSTICS_ESCAPE_FORMAT_BYTES;
+      diagnostic_show_locus (&dc, &richloc, DK_ERROR);
+      ASSERT_STREQ
+	(" <f0><9f><98><82>_foo = <cf><80>_bar.<f0><9f><98><82>_field<cf><80>;\n"
+	 " ^~~~~~~~~~~~~~~~~~~~   ~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
+	 " |                      |            |\n"
+	 " |                      |            label 2\xcf\x80\n"
+	 " |                      label 1\xcf\x80\n"
+	 " label 0\xf0\x9f\x98\x82\n",
+	 pp_formatted_text (dc.printer));
+    }
+  }
 }
 
 /* Make sure that colorization codes don't interrupt a multibyte
@@ -4057,9 +4357,9 @@ test_diagnostic_show_locus_one_liner_utf8 (const line_table_case &case_)
 
   char_span lspan = location_get_source_line (tmp.get_filename (), 1);
   ASSERT_EQ (25, cpp_display_width (lspan.get_buffer (), lspan.length (),
-				    def_tabstop));
+				    def_policy ()));
   ASSERT_EQ (25, location_compute_display_column (expand_location (line_end),
-						  def_tabstop));
+						  def_policy ()));
 
   test_one_liner_simple_caret_utf8 ();
   test_one_liner_caret_and_range_utf8 ();
@@ -4445,30 +4745,31 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 		  pp_formatted_text (dc.printer));
 
     /* Unit-test the line_corrections machinery.  */
+    char_display_policy policy (make_policy (dc, richloc));
     ASSERT_EQ (3, richloc.get_num_fixit_hints ());
     const fixit_hint *hint_0 = richloc.get_fixit_hint (0);
     ASSERT_EQ (column_range (12, 12),
-	       get_affected_range (&dc, hint_0, CU_BYTES));
+	       get_affected_range (policy, hint_0, CU_BYTES));
     ASSERT_EQ (column_range (12, 12),
-	       get_affected_range (&dc, hint_0, CU_DISPLAY_COLS));
-    ASSERT_EQ (column_range (12, 22), get_printed_columns (&dc, hint_0));
+	       get_affected_range (policy, hint_0, CU_DISPLAY_COLS));
+    ASSERT_EQ (column_range (12, 22), get_printed_columns (policy, hint_0));
     const fixit_hint *hint_1 = richloc.get_fixit_hint (1);
     ASSERT_EQ (column_range (18, 18),
-	       get_affected_range (&dc, hint_1, CU_BYTES));
+	       get_affected_range (policy, hint_1, CU_BYTES));
     ASSERT_EQ (column_range (18, 18),
-	       get_affected_range (&dc, hint_1, CU_DISPLAY_COLS));
-    ASSERT_EQ (column_range (18, 20), get_printed_columns (&dc, hint_1));
+	       get_affected_range (policy, hint_1, CU_DISPLAY_COLS));
+    ASSERT_EQ (column_range (18, 20), get_printed_columns (policy, hint_1));
     const fixit_hint *hint_2 = richloc.get_fixit_hint (2);
     ASSERT_EQ (column_range (29, 28),
-	       get_affected_range (&dc, hint_2, CU_BYTES));
+	       get_affected_range (policy, hint_2, CU_BYTES));
     ASSERT_EQ (column_range (29, 28),
-	       get_affected_range (&dc, hint_2, CU_DISPLAY_COLS));
-    ASSERT_EQ (column_range (29, 29), get_printed_columns (&dc, hint_2));
+	       get_affected_range (policy, hint_2, CU_DISPLAY_COLS));
+    ASSERT_EQ (column_range (29, 29), get_printed_columns (policy, hint_2));
 
     /* Add each hint in turn to a line_corrections instance,
        and verify that they are consolidated into one correction instance
        as expected.  */
-    line_corrections lc (&dc, tmp.get_filename (), 1);
+    line_corrections lc (policy, tmp.get_filename (), 1);
 
     /* The first replace hint by itself.  */
     lc.add_hint (hint_0);
@@ -4660,30 +4961,31 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 		  pp_formatted_text (dc.printer));
 
     /* Unit-test the line_corrections machinery.  */
+    char_display_policy policy (make_policy (dc, richloc));
     ASSERT_EQ (3, richloc.get_num_fixit_hints ());
     const fixit_hint *hint_0 = richloc.get_fixit_hint (0);
     ASSERT_EQ (column_range (14, 14),
-	       get_affected_range (&dc, hint_0, CU_BYTES));
+	       get_affected_range (policy, hint_0, CU_BYTES));
     ASSERT_EQ (column_range (12, 12),
-	       get_affected_range (&dc, hint_0, CU_DISPLAY_COLS));
-    ASSERT_EQ (column_range (12, 22), get_printed_columns (&dc, hint_0));
+	       get_affected_range (policy, hint_0, CU_DISPLAY_COLS));
+    ASSERT_EQ (column_range (12, 22), get_printed_columns (policy, hint_0));
     const fixit_hint *hint_1 = richloc.get_fixit_hint (1);
     ASSERT_EQ (column_range (22, 22),
-	       get_affected_range (&dc, hint_1, CU_BYTES));
+	       get_affected_range (policy, hint_1, CU_BYTES));
     ASSERT_EQ (column_range (18, 18),
-	       get_affected_range (&dc, hint_1, CU_DISPLAY_COLS));
-    ASSERT_EQ (column_range (18, 20), get_printed_columns (&dc, hint_1));
+	       get_affected_range (policy, hint_1, CU_DISPLAY_COLS));
+    ASSERT_EQ (column_range (18, 20), get_printed_columns (policy, hint_1));
     const fixit_hint *hint_2 = richloc.get_fixit_hint (2);
     ASSERT_EQ (column_range (35, 34),
-	       get_affected_range (&dc, hint_2, CU_BYTES));
+	       get_affected_range (policy, hint_2, CU_BYTES));
     ASSERT_EQ (column_range (30, 29),
-	       get_affected_range (&dc, hint_2, CU_DISPLAY_COLS));
-    ASSERT_EQ (column_range (30, 30), get_printed_columns (&dc, hint_2));
+	       get_affected_range (policy, hint_2, CU_DISPLAY_COLS));
+    ASSERT_EQ (column_range (30, 30), get_printed_columns (policy, hint_2));
 
     /* Add each hint in turn to a line_corrections instance,
        and verify that they are consolidated into one correction instance
        as expected.  */
-    line_corrections lc (&dc, tmp.get_filename (), 1);
+    line_corrections lc (policy, tmp.get_filename (), 1);
 
     /* The first replace hint by itself.  */
     lc.add_hint (hint_0);
@@ -4877,15 +5179,16 @@ test_overlapped_fixit_printing_2 (const line_table_case &case_)
     richloc.add_fixit_insert_before (col_21, "}");
 
     /* These fixits should be accepted; they can't be consolidated.  */
+    char_display_policy policy (make_policy (dc, richloc));
     ASSERT_EQ (2, richloc.get_num_fixit_hints ());
     const fixit_hint *hint_0 = richloc.get_fixit_hint (0);
     ASSERT_EQ (column_range (23, 22),
-	       get_affected_range (&dc, hint_0, CU_BYTES));
-    ASSERT_EQ (column_range (23, 23), get_printed_columns (&dc, hint_0));
+	       get_affected_range (policy, hint_0, CU_BYTES));
+    ASSERT_EQ (column_range (23, 23), get_printed_columns (policy, hint_0));
     const fixit_hint *hint_1 = richloc.get_fixit_hint (1);
     ASSERT_EQ (column_range (21, 20),
-	       get_affected_range (&dc, hint_1, CU_BYTES));
-    ASSERT_EQ (column_range (21, 21), get_printed_columns (&dc, hint_1));
+	       get_affected_range (policy, hint_1, CU_BYTES));
+    ASSERT_EQ (column_range (21, 21), get_printed_columns (policy, hint_1));
 
     /* Verify that they're printed correctly.  */
     diagnostic_show_locus (&dc, &richloc, DK_ERROR);
@@ -5152,10 +5455,11 @@ test_tab_expansion (const line_table_case &case_)
      ....................123 45678901234 56789012345  columns  */
 
   const int tabstop = 8;
+  cpp_char_column_policy policy (tabstop, cpp_wcwidth);
   const int first_non_ws_byte_col = 7;
   const int right_quote_byte_col = 15;
   const int last_byte_col = 25;
-  ASSERT_EQ (35, cpp_display_width (content, last_byte_col, tabstop));
+  ASSERT_EQ (35, cpp_display_width (content, last_byte_col, policy));
 
   temp_source_file tmp (SELFTEST_LOCATION, ".c", content);
   line_table_test ltt (case_);
@@ -5198,6 +5502,114 @@ test_tab_expansion (const line_table_case &case_)
   }
 }
 
+/* Verify that the escaping machinery can cope with a variety of different
+   invalid bytes.  */
+
+static void
+test_escaping_bytes_1 (const line_table_case &case_)
+{
+  const char content[] = "before\0\1\2\3\r\x80\xff""after\n";
+  const size_t sz = sizeof (content);
+  temp_source_file tmp (SELFTEST_LOCATION, ".c", content, sz);
+  line_table_test ltt (case_);
+  const line_map_ordinary *ord_map = linemap_check_ordinary
+    (linemap_add (line_table, LC_ENTER, false, tmp.get_filename (), 0));
+  linemap_line_start (line_table, 1, 100);
+
+  location_t finish
+    = linemap_position_for_line_and_column (line_table, ord_map, 1,
+					    strlen (content));
+
+  if (finish > LINE_MAP_MAX_LOCATION_WITH_COLS)
+    return;
+
+  /* Locations of the NUL and \r bytes.  */
+  location_t nul_loc
+    = linemap_position_for_line_and_column (line_table, ord_map, 1, 7);
+  location_t r_loc
+    = linemap_position_for_line_and_column (line_table, ord_map, 1, 11);
+  gcc_rich_location richloc (nul_loc);
+  richloc.add_range (r_loc);
+
+  {
+    test_diagnostic_context dc;
+    diagnostic_show_locus (&dc, &richloc, DK_ERROR);
+    ASSERT_STREQ (" before \1\2\3 \x80\xff""after\n"
+		  "       ^   ~\n",
+		  pp_formatted_text (dc.printer));
+  }
+  richloc.set_escape_on_output (true);
+  {
+    test_diagnostic_context dc;
+    dc.escape_format = DIAGNOSTICS_ESCAPE_FORMAT_UNICODE;
+    diagnostic_show_locus (&dc, &richloc, DK_ERROR);
+    ASSERT_STREQ
+      (" before<U+0000><U+0001><U+0002><U+0003><U+000D><80><ff>after\n"
+       "       ^~~~~~~~                        ~~~~~~~~\n",
+       pp_formatted_text (dc.printer));
+  }
+  {
+    test_diagnostic_context dc;
+    dc.escape_format = DIAGNOSTICS_ESCAPE_FORMAT_BYTES;
+    diagnostic_show_locus (&dc, &richloc, DK_ERROR);
+    ASSERT_STREQ (" before<00><01><02><03><0d><80><ff>after\n"
+		  "       ^~~~            ~~~~\n",
+		  pp_formatted_text (dc.printer));
+  }
+}
+
+/* As above, but verify that we handle the initial byte of a line
+   correctly.  */
+
+static void
+test_escaping_bytes_2 (const line_table_case &case_)
+{
+  const char content[]  = "\0after\n";
+  const size_t sz = sizeof (content);
+  temp_source_file tmp (SELFTEST_LOCATION, ".c", content, sz);
+  line_table_test ltt (case_);
+  const line_map_ordinary *ord_map = linemap_check_ordinary
+    (linemap_add (line_table, LC_ENTER, false, tmp.get_filename (), 0));
+  linemap_line_start (line_table, 1, 100);
+
+  location_t finish
+    = linemap_position_for_line_and_column (line_table, ord_map, 1,
+					    strlen (content));
+
+  if (finish > LINE_MAP_MAX_LOCATION_WITH_COLS)
+    return;
+
+  /* Location of the NUL byte.  */
+  location_t nul_loc
+    = linemap_position_for_line_and_column (line_table, ord_map, 1, 1);
+  gcc_rich_location richloc (nul_loc);
+
+  {
+    test_diagnostic_context dc;
+    diagnostic_show_locus (&dc, &richloc, DK_ERROR);
+    ASSERT_STREQ ("  after\n"
+		  " ^\n",
+		  pp_formatted_text (dc.printer));
+  }
+  richloc.set_escape_on_output (true);
+  {
+    test_diagnostic_context dc;
+    dc.escape_format = DIAGNOSTICS_ESCAPE_FORMAT_UNICODE;
+    diagnostic_show_locus (&dc, &richloc, DK_ERROR);
+    ASSERT_STREQ (" <U+0000>after\n"
+		  " ^~~~~~~~\n",
+		  pp_formatted_text (dc.printer));
+  }
+  {
+    test_diagnostic_context dc;
+    dc.escape_format = DIAGNOSTICS_ESCAPE_FORMAT_BYTES;
+    diagnostic_show_locus (&dc, &richloc, DK_ERROR);
+    ASSERT_STREQ (" <00>after\n"
+		  " ^~~~\n",
+		  pp_formatted_text (dc.printer));
+  }
+}
+
 /* Verify that line numbers are correctly printed for the case of
    a multiline range in which the width of the line numbers changes
    (e.g. from "9" to "10").  */
@@ -5254,6 +5666,8 @@ diagnostic_show_locus_c_tests ()
   test_layout_range_for_single_line ();
   test_layout_range_for_multiple_lines ();
 
+  test_display_widths ();
+
   for_each_line_table_case (test_layout_x_offset_display_utf8);
   for_each_line_table_case (test_layout_x_offset_display_tab);
 
@@ -5274,6 +5688,8 @@ diagnostic_show_locus_c_tests ()
   for_each_line_table_case (test_fixit_replace_containing_newline);
   for_each_line_table_case (test_fixit_deletion_affecting_newline);
   for_each_line_table_case (test_tab_expansion);
+  for_each_line_table_case (test_escaping_bytes_1);
+  for_each_line_table_case (test_escaping_bytes_2);
 
   test_line_numbers_multiline_range ();
 }
diff --git a/gcc/diagnostic-spec.c b/gcc/diagnostic-spec.c
index 85ffb72..d1e563d 100644
--- a/gcc/diagnostic-spec.c
+++ b/gcc/diagnostic-spec.c
@@ -107,7 +107,7 @@ nowarn_spec_t::nowarn_spec_t (opt_code opt)
 
 /* A mapping from a 'location_t' to the warning spec set for it.  */
 
-GTY(()) xint_hash_map_t *nowarn_map;
+GTY(()) nowarn_map_t *nowarn_map;
 
 /* Return the no-warning disposition for location LOC and option OPT
    or for all/any otions by default.  */
@@ -163,7 +163,7 @@ suppress_warning_at (location_t loc, opt_code opt /* = all_warnings */,
     return false;
 
   if (!nowarn_map)
-    nowarn_map = xint_hash_map_t::create_ggc (32);
+    nowarn_map = nowarn_map_t::create_ggc (32);
 
   nowarn_map->put (loc, optspec);
   return true;
diff --git a/gcc/diagnostic-spec.h b/gcc/diagnostic-spec.h
index 9b3aaaa..368b75f 100644
--- a/gcc/diagnostic-spec.h
+++ b/gcc/diagnostic-spec.h
@@ -130,10 +130,9 @@ operator!= (const nowarn_spec_t &lhs, const nowarn_spec_t &rhs)
   return !(lhs == rhs);
 }
 
-typedef int_hash <location_t, 0, UINT_MAX> xint_hash_t;
-typedef hash_map<xint_hash_t, nowarn_spec_t> xint_hash_map_t;
+typedef hash_map<location_hash, nowarn_spec_t> nowarn_map_t;
 
 /* A mapping from a 'location_t' to the warning spec set for it.  */
-extern GTY(()) xint_hash_map_t *nowarn_map;
+extern GTY(()) nowarn_map_t *nowarn_map;
 
 #endif // DIAGNOSTIC_SPEC_H_INCLUDED
diff --git a/gcc/diagnostic.c b/gcc/diagnostic.c
index b3afbea..4ded176 100644
--- a/gcc/diagnostic.c
+++ b/gcc/diagnostic.c
@@ -230,6 +230,7 @@ diagnostic_initialize (diagnostic_context *context, int n_opts)
   context->column_unit = DIAGNOSTICS_COLUMN_UNIT_DISPLAY;
   context->column_origin = 1;
   context->tabstop = 8;
+  context->escape_format = DIAGNOSTICS_ESCAPE_FORMAT_UNICODE;
   context->edit_context_ptr = NULL;
   context->diagnostic_group_nesting_depth = 0;
   context->diagnostic_group_emission_count = 0;
@@ -393,7 +394,10 @@ convert_column_unit (enum diagnostics_column_unit column_unit,
       gcc_unreachable ();
 
     case DIAGNOSTICS_COLUMN_UNIT_DISPLAY:
-      return location_compute_display_column (s, tabstop);
+      {
+	cpp_char_column_policy policy (tabstop, cpp_wcwidth);
+	return location_compute_display_column (s, policy);
+      }
 
     case DIAGNOSTICS_COLUMN_UNIT_BYTE:
       return s.column;
@@ -2373,8 +2377,8 @@ test_diagnostic_get_location_text ()
     const char *const content = "smile \xf0\x9f\x98\x82\n";
     const int line_bytes = strlen (content) - 1;
     const int def_tabstop = 8;
-    const int display_width = cpp_display_width (content, line_bytes,
-						 def_tabstop);
+    const cpp_char_column_policy policy (def_tabstop, cpp_wcwidth);
+    const int display_width = cpp_display_width (content, line_bytes, policy);
     ASSERT_EQ (line_bytes - 2, display_width);
     temp_source_file tmp (SELFTEST_LOCATION, ".c", content);
     const char *const fname = tmp.get_filename ();
diff --git a/gcc/diagnostic.h b/gcc/diagnostic.h
index f90d20a..c2aea50 100644
--- a/gcc/diagnostic.h
+++ b/gcc/diagnostic.h
@@ -38,6 +38,20 @@ enum diagnostics_column_unit
   DIAGNOSTICS_COLUMN_UNIT_BYTE
 };
 
+/* An enum for controlling how to print non-ASCII characters/bytes when
+   a diagnostic suggests escaping the source code on output.  */
+
+enum diagnostics_escape_format
+{
+  /* Escape non-ASCII Unicode characters in the form <U+XXXX> and
+     non-UTF-8 bytes in the form <XX>.  */
+  DIAGNOSTICS_ESCAPE_FORMAT_UNICODE,
+
+  /* Escape non-ASCII bytes in the form <XX> (thus showing the underlying
+     encoding of non-ASCII Unicode characters).  */
+  DIAGNOSTICS_ESCAPE_FORMAT_BYTES
+};
+
 /* Enum for overriding the standard output format.  */
 
 enum diagnostics_output_format
@@ -340,6 +354,10 @@ struct diagnostic_context
   /* The size of the tabstop for tab expansion.  */
   int tabstop;
 
+  /* How should non-ASCII/non-printable bytes be escaped when
+     a diagnostic suggests escaping the source code on output.  */
+  enum diagnostics_escape_format escape_format;
+
   /* If non-NULL, an edit_context to which fix-it hints should be
      applied, for generating patches.  */
   edit_context *edit_context_ptr;
diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
index 171b7a8..8434533 100644
--- a/gcc/doc/extend.texi
+++ b/gcc/doc/extend.texi
@@ -23762,6 +23762,25 @@ restored.
   foo(d);                       /* depends on command-line options */
 @end smallexample
 
+@item #pragma GCC diagnostic ignored_attributes
+
+Similarly to @option{-Wno-attributes=}, this pragma allows users to suppress
+warnings about unknown scoped attributes (in C++11 and C2X).  For example,
+@code{#pragma GCC diagnostic ignored_attributes "vendor::attr"} disables
+warning about the following declaration:
+
+@smallexample
+[[vendor::attr]] void f();
+@end smallexample
+
+whereas @code{#pragma GCC diagnostic ignored_attributes "vendor::"} prevents
+warning about both of these declarations:
+
+@smallexample
+[[vendor::safe]] void f();
+[[vendor::unsafe]] void f2();
+@end smallexample
+
 @end table
 
 GCC also offers a simple mechanism for printing messages during
diff --git a/gcc/doc/generic.texi b/gcc/doc/generic.texi
index d317f65..69f6f37 100644
--- a/gcc/doc/generic.texi
+++ b/gcc/doc/generic.texi
@@ -1223,25 +1223,12 @@ meanings.  The type of these expressions must be an array whose component
 type is the same as that of the first operand.  The range of that array
 type determines the amount of data these expressions access.
 
-@item TARGET_MEM_REF
-These nodes represent memory accesses whose address directly map to
-an addressing mode of the target architecture.  The first argument
-is @code{TMR_SYMBOL} and must be a @code{VAR_DECL} of an object with
-a fixed address.  The second argument is @code{TMR_BASE} and the
-third one is @code{TMR_INDEX}.  The fourth argument is
-@code{TMR_STEP} and must be an @code{INTEGER_CST}.  The fifth
-argument is @code{TMR_OFFSET} and must be an @code{INTEGER_CST}.
-Any of the arguments may be NULL if the appropriate component
-does not appear in the address.  Address of the @code{TARGET_MEM_REF}
-is determined in the following way.
-
-@smallexample
-&TMR_SYMBOL + TMR_BASE + TMR_INDEX * TMR_STEP + TMR_OFFSET
-@end smallexample
-
-The sixth argument is the reference to the original memory access, which
-is preserved for the purposes of the RTL alias analysis.  The seventh
-argument is a tag representing the results of tree level alias analysis.
+@item COMPONENT_REF
+These nodes represent non-static data member accesses.  The first
+operand is the object (rather than a pointer to it); the second operand
+is the @code{FIELD_DECL} for the data member.  The third operand represents
+the byte offset of the field, but should not be used directly; call
+@code{component_ref_field_offset} instead.
 
 @item ADDR_EXPR
 These nodes are used to represent the address of an object.  (These
@@ -1264,16 +1251,33 @@ pointer or reference type.
 These nodes are used to represent the object pointed to by a pointer
 offset by a constant.
 The first operand is the pointer being dereferenced; it will always have
-pointer or reference type.  The second operand is a pointer constant.
-Its type is specifying the type to be used for type-based alias analysis.
+pointer or reference type.  The second operand is a pointer constant
+serving as constant offset applied to the pointer being dereferenced
+with its type specifying the type to be used for type-based alias analysis.
+The type of the node specifies the alignment of the access.
 
-@item COMPONENT_REF
-These nodes represent non-static data member accesses.  The first
-operand is the object (rather than a pointer to it); the second operand
-is the @code{FIELD_DECL} for the data member.  The third operand represents
-the byte offset of the field, but should not be used directly; call
-@code{component_ref_field_offset} instead.
+@item TARGET_MEM_REF
+These nodes represent memory accesses whose address directly map to
+an addressing mode of the target architecture.  The first argument
+is @code{TMR_BASE} and is a pointer to the object being accessed.
+The second argument is @code{TMR_OFFSET} which is a pointer constant
+with dual purpose serving both as constant offset and holder of
+the type used for type-based alias analysis.  The first two operands
+have exactly the same semantics as @code{MEM_REF}.  The third
+and fourth operand are @code{TMR_INDEX} and @code{TMR_STEP} where
+the former is an integer and the latter an integer constant.  The
+fifth and last operand is @code{TMR_INDEX2} which is an alternate
+non-constant offset.  Any of the third to last operands may be
+@code{NULL} if the corresponding component does not appear in
+the address, but @code{TMR_INDEX} and @code{TMR_STEP} shall be
+always supplied in pair.  The Address of the @code{TARGET_MEM_REF}
+is determined in the following way.
+
+@smallexample
+TMR_BASE + TMR_OFFSET + TMR_INDEX * TMR_STEP + TMR_INDEX2
+@end smallexample
 
+The type of the node specifies the alignment of the access.
 
 @end table
 
diff --git a/gcc/doc/gty.texi b/gcc/doc/gty.texi
index 2ad7793..b996ff2 100644
--- a/gcc/doc/gty.texi
+++ b/gcc/doc/gty.texi
@@ -64,6 +64,14 @@ The parser understands simple typedefs such as
 @code{typedef int @var{name};}.
 These don't need to be marked.
 
+However, in combination with GTY, avoid using typedefs such as
+@code{typedef int_hash<@dots{}> @var{name};}
+for these generate infinite-recursion code.
+See @uref{https://gcc.gnu.org/PR103157,PR103157}.
+Instead, you may use
+@code{struct @var{name} : int_hash<@dots{}> @{@};},
+for example.
+
 Since @code{gengtype}'s understanding of C++ is limited, there are
 several constructs and declarations that are not supported inside
 classes/structures marked for automatic GC code generation.  The
diff --git a/gcc/doc/install.texi b/gcc/doc/install.texi
index 4f60e48..54d1d21 100644
--- a/gcc/doc/install.texi
+++ b/gcc/doc/install.texi
@@ -449,7 +449,7 @@ Necessary when modifying @command{gperf} input files, e.g.@:
 @file{gcc/cp/cfns.gperf} to regenerate its associated header file, e.g.@:
 @file{gcc/cp/cfns.h}.
 
-@item DejaGnu 1.4.4
+@item DejaGnu version 1.5.3 (or later)
 @itemx Expect
 @itemx Tcl
 @c Once Tcl 8.5 or higher is required, remove any obsolete
@@ -3435,7 +3435,7 @@ Microsoft Windows:
 The @uref{https://sourceware.org/cygwin/,,Cygwin} project;
 @item
 The @uref{https://osdn.net/projects/mingw/,,MinGW} and
-@uref{http://www.mingw-w64.org/,,mingw-w64} projects.
+@uref{https://www.mingw-w64.org/,,mingw-w64} projects.
 @end itemize
 
 @item
@@ -4160,15 +4160,6 @@ If you are using the installed system libunwind library with
 @option{--with-system-libunwind}, then you must use libunwind 0.98 or
 later.
 
-None of the following versions of GCC has an ABI that is compatible
-with any of the other versions in this list, with the exception that
-Red Hat 2.96 and Trillian 000171 are compatible with each other:
-3.1, 3.0.2, 3.0.1, 3.0, Red Hat 2.96, and Trillian 000717.
-This primarily affects C++ programs and programs that create shared libraries.
-GCC 3.1 or later is recommended for compiling linux, the kernel.
-As of version 3.1 GCC is believed to be fully ABI compliant, and hence no
-more major ABI changes are expected.
-
 @html
 <hr />
 @end html
@@ -5042,11 +5033,9 @@ the Win32 subsystem that provides a subset of POSIX.
 
 @subheading Intel 64-bit versions
 GCC contains support for x86-64 using the mingw-w64
-runtime library, available from @uref{http://mingw-w64.org/doku.php}.
+runtime library, available from @uref{https://mingw-w64.org/doku.php}.
 This library should be used with the target triple x86_64-pc-mingw32.
 
-Presently Windows for Itanium is not supported.
-
 @subheading Windows CE
 Windows CE is supported as a target only on Hitachi
 SuperH (sh-wince-pe), and MIPS (mips-wince-pe).
diff --git a/gcc/doc/install.texi2html b/gcc/doc/install.texi2html
index 09bbbc4..001a869 100755
--- a/gcc/doc/install.texi2html
+++ b/gcc/doc/install.texi2html
@@ -46,9 +46,9 @@ fi
    echo "@set srcdir $SOURCEDIR/.."
 ) > $DESTDIR/gcc-vers.texi
 
-for x in index.html specific.html prerequisites.html download.html configure.html \
-         build.html test.html finalinstall.html binaries.html old.html \
-         gfdl.html
+for x in index.html specific.html prerequisites.html download.html \
+         configure.html build.html test.html finalinstall.html \
+         binaries.html gfdl.html
 do
     define=`echo $x | sed -e 's/\.//g'`
     echo "define = $define"
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index acfa766..3ae66ac 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -275,7 +275,8 @@ Objective-C and Objective-C++ Dialects}.
 -fdiagnostics-show-path-depths @gol
 -fno-show-column @gol
 -fdiagnostics-column-unit=@r{[}display@r{|}byte@r{]} @gol
--fdiagnostics-column-origin=@var{origin}}
+-fdiagnostics-column-origin=@var{origin} @gol
+-fdiagnostics-escape-format=@r{[}unicode@r{|}bytes@r{]}}
 
 @item Warning Options
 @xref{Warning Options,,Options to Request or Suppress Warnings}.
@@ -411,7 +412,11 @@ Objective-C and Objective-C++ Dialects}.
 -Wno-analyzer-shift-count-negative @gol
 -Wno-analyzer-shift-count-overflow @gol
 -Wno-analyzer-stale-setjmp-buffer @gol
+-Wno-analyzer-tainted-allocation-size @gol
 -Wno-analyzer-tainted-array-index @gol
+-Wno-analyzer-tainted-divisor @gol
+-Wno-analyzer-tainted-offset @gol
+-Wno-analyzer-tainted-size @gol
 -Wanalyzer-too-complex @gol
 -Wno-analyzer-unsafe-call-within-signal-handler @gol
 -Wno-analyzer-use-after-free @gol
@@ -559,6 +564,7 @@ Objective-C and Objective-C++ Dialects}.
 -fasan-shadow-offset=@var{number}  -fsanitize-sections=@var{s1},@var{s2},... @gol
 -fsanitize-undefined-trap-on-error  -fbounds-check @gol
 -fcf-protection=@r{[}full@r{|}branch@r{|}return@r{|}none@r{|}check@r{]} @gol
+-fharden-compares -fharden-conditional-branches @gol
 -fstack-protector  -fstack-protector-all  -fstack-protector-strong @gol
 -fstack-protector-explicit  -fstack-check @gol
 -fstack-limit-register=@var{reg}  -fstack-limit-symbol=@var{sym} @gol
@@ -1321,7 +1327,7 @@ See RS/6000 and PowerPC Options.
 -mbig-switch}
 
 @emph{VAX Options}
-@gccoptlist{-mg  -mgnu  -munix}
+@gccoptlist{-mg  -mgnu  -munix  -mlra}
 
 @emph{Visium Options}
 @gccoptlist{-mdebug  -msim  -mfpu  -mno-fpu  -mhard-float  -msoft-float @gol
@@ -1380,7 +1386,7 @@ See RS/6000 and PowerPC Options.
 -mstack-protector-guard-reg=@var{reg} @gol
 -mstack-protector-guard-offset=@var{offset} @gol
 -mstack-protector-guard-symbol=@var{symbol} @gol
--mgeneral-regs-only  -mcall-ms2sysv-xlogues @gol
+-mgeneral-regs-only  -mcall-ms2sysv-xlogues -mrelax-cmpxchg-loop @gol
 -mindirect-branch=@var{choice}  -mfunction-return=@var{choice} @gol
 -mindirect-branch-register -mneeded}
 
@@ -5088,6 +5094,38 @@ first column.  The default value of 1 corresponds to traditional GCC
 behavior and to the GNU style guide.  Some utilities may perform better with an
 origin of 0; any non-negative value may be specified.
 
+@item -fdiagnostics-escape-format=@var{FORMAT}
+@opindex fdiagnostics-escape-format
+When GCC prints pertinent source lines for a diagnostic it normally attempts
+to print the source bytes directly.  However, some diagnostics relate to encoding
+issues in the source file, such as malformed UTF-8, or issues with Unicode
+normalization.  These diagnostics are flagged so that GCC will escape bytes
+that are not printable ASCII when printing their pertinent source lines.
+
+This option controls how such bytes should be escaped.
+
+The default @var{FORMAT}, @samp{unicode} displays Unicode characters that
+are not printable ASCII in the form @samp{<U+XXXX>}, and bytes that do not
+correspond to a Unicode character validly-encoded in UTF-8-encoded will be
+displayed as hexadecimal in the form @samp{<XX>}.
+
+For example, a source line containing the string @samp{before} followed by the
+Unicode character U+03C0 (``GREEK SMALL LETTER PI'', with UTF-8 encoding
+0xCF 0x80) followed by the byte 0xBF (a stray UTF-8 trailing byte), followed by
+the string @samp{after} will be printed for such a diagnostic as:
+
+@smallexample
+ before<U+03C0><BF>after
+@end smallexample
+
+Setting @var{FORMAT} to @samp{bytes} will display all non-printable-ASCII bytes
+in the form @samp{<XX>}, thus showing the underlying encoding of non-ASCII
+Unicode characters.  For the example above, the following will be printed:
+
+@smallexample
+ before<CF><80><BF>after
+@end smallexample
+
 @item -fdiagnostics-format=@var{FORMAT}
 @opindex fdiagnostics-format
 Select a different format for printing diagnostics.
@@ -5155,9 +5193,11 @@ might be printed in JSON form (after formatting) like this:
                         @}
                     @}
                 ],
+                "escape-source": false,
                 "message": "...this statement, but the latter is @dots{}"
             @}
         ]
+	"escape-source": false,
 	"column-origin": 1,
     @}
 ]
@@ -5243,6 +5283,7 @@ of the expression, which have labels.  It might be printed in JSON form as:
                 "label": "T @{aka struct t@}"
             @}
         ],
+        "escape-source": false,
         "message": "invalid operands to binary + @dots{}"
     @}
 @end smallexample
@@ -5296,6 +5337,7 @@ might be printed in JSON form as:
                 @}
             @}
         ],
+        "escape-source": false,
         "message": "\u2018struct s\u2019 has no member named @dots{}"
     @}
 @end smallexample
@@ -5353,6 +5395,10 @@ For example, the intraprocedural example shown for
     ]
 @end smallexample
 
+Diagnostics have a boolean attribute @code{escape-source}, hinting whether
+non-ASCII bytes should be escaped when printing the pertinent lines of
+source code (@code{true} for diagnostics involving source encoding issues).
+
 @end table
 
 @node Warning Options
@@ -8628,6 +8674,26 @@ unrecognized attributes, function attributes applied to variables,
 etc.  This does not stop errors for incorrect use of supported
 attributes.
 
+Additionally, using @option{-Wno-attributes=}, it is possible to suppress
+warnings about unknown scoped attributes (in C++11 and C2X).  For example,
+@option{-Wno-attributes=vendor::attr} disables warning about the following
+declaration:
+
+@smallexample
+[[vendor::attr]] void f();
+@end smallexample
+
+It is also possible to disable warning about all attributes in a namespace
+using @option{-Wno-attributes=vendor::} which prevents warning about both
+of these declarations:
+
+@smallexample
+[[vendor::safe]] void f();
+[[vendor::unsafe]] void f2();
+@end smallexample
+
+Note that @option{-Wno-attributes=} does not imply @option{-Wno-attributes}.
+
 @item -Wno-builtin-declaration-mismatch
 @opindex Wno-builtin-declaration-mismatch
 @opindex Wbuiltin-declaration-mismatch
@@ -9301,7 +9367,11 @@ Enabling this option effectively enables the following warnings:
 -Wanalyzer-shift-count-negative @gol
 -Wanalyzer-shift-count-overflow @gol
 -Wanalyzer-stale-setjmp-buffer @gol
+-Wanalyzer-tainted-allocation-size @gol
 -Wanalyzer-tainted-array-index @gol
+-Wanalyzer-tainted-divisor @gol
+-Wanalyzer-tainted-offset @gol
+-Wanalyzer-tainted-size @gol
 -Wanalyzer-unsafe-call-within-signal-handler @gol
 -Wanalyzer-use-after-free @gol
 -Wanalyzer-use-of-uninitialized-value @gol
@@ -9484,6 +9554,21 @@ when the function containing the @code{setjmp} call returns.  Attempting
 to rewind to it via @code{longjmp} would reference a stack frame that
 no longer exists, and likely lead to a crash (or worse).
 
+@item -Wno-analyzer-tainted-allocation-size
+@opindex Wanalyzer-tainted-allocation-size
+@opindex Wno-analyzer-tainted-allocation-size
+This warning requires both @option{-fanalyzer} and
+@option{-fanalyzer-checker=taint} to enable it;
+use @option{-Wno-analyzer-tainted-allocation-size} to disable it.
+
+This diagnostic warns for paths through the code in which a value
+that could be under an attacker's control is used as the size
+of an allocation without being sanitized, so that an attacker could
+inject an excessively large allocation and potentially cause a denial
+of service attack.
+
+See @url{https://cwe.mitre.org/data/definitions/789.html, CWE-789: Memory Allocation with Excessive Size Value}.
+
 @item -Wno-analyzer-tainted-array-index
 @opindex Wanalyzer-tainted-array-index
 @opindex Wno-analyzer-tainted-array-index
@@ -9493,7 +9578,48 @@ use @option{-Wno-analyzer-tainted-array-index} to disable it.
 
 This diagnostic warns for paths through the code in which a value
 that could be under an attacker's control is used as the index
-of an array access without being sanitized.
+of an array access without being sanitized, so that an attacker
+could inject an out-of-bounds access.
+
+See @url{https://cwe.mitre.org/data/definitions/129.html, CWE-129: Improper Validation of Array Index}.
+
+@item -Wno-analyzer-tainted-divisor
+@opindex Wanalyzer-tainted-divisor
+@opindex Wno-analyzer-tainted-divisor
+This warning requires both @option{-fanalyzer} and
+@option{-fanalyzer-checker=taint} to enable it;
+use @option{-Wno-analyzer-tainted-divisor} to disable it.
+
+This diagnostic warns for paths through the code in which a value
+that could be under an attacker's control is used as the divisor
+in a division or modulus operation without being sanitized, so that
+an attacker could inject a division-by-zero.
+
+@item -Wno-analyzer-tainted-offset
+@opindex Wanalyzer-tainted-offset
+@opindex Wno-analyzer-tainted-offset
+This warning requires both @option{-fanalyzer} and
+@option{-fanalyzer-checker=taint} to enable it;
+use @option{-Wno-analyzer-tainted-offset} to disable it.
+
+This diagnostic warns for paths through the code in which a value
+that could be under an attacker's control is used as a pointer offset
+without being sanitized, so that an attacker could inject an out-of-bounds
+access.
+
+See @url{https://cwe.mitre.org/data/definitions/823.html, CWE-823: Use of Out-of-range Pointer Offset}.
+
+@item -Wno-analyzer-tainted-size
+@opindex Wanalyzer-tainted-size
+@opindex Wno-analyzer-tainted-size
+This warning requires both @option{-fanalyzer} and
+@option{-fanalyzer-checker=taint} to enable it;
+use @option{-Wno-analyzer-tainted-size} to disable it.
+
+This diagnostic warns for paths through the code in which a value
+that could be under an attacker's control is used as the size of
+an operation such as @code{memset} without being sanitized, so that an
+attacker could inject an out-of-bounds access.
 
 @item -Wno-analyzer-unsafe-call-within-signal-handler
 @opindex Wanalyzer-unsafe-call-within-signal-handler
@@ -11879,13 +12005,13 @@ and @option{-ftree-slp-vectorize} if not explicitly specified.
 @item -ftree-loop-vectorize
 @opindex ftree-loop-vectorize
 Perform loop vectorization on trees. This flag is enabled by default at
-@option{-O3} and by @option{-ftree-vectorize}, @option{-fprofile-use},
+@option{-O2} and by @option{-ftree-vectorize}, @option{-fprofile-use},
 and @option{-fauto-profile}.
 
 @item -ftree-slp-vectorize
 @opindex ftree-slp-vectorize
 Perform basic block vectorization on trees. This flag is enabled by default at
-@option{-O3} and by @option{-ftree-vectorize}, @option{-fprofile-use},
+@option{-O2} and by @option{-ftree-vectorize}, @option{-fprofile-use},
 and @option{-fauto-profile}.
 
 @item -ftrivial-auto-var-init=@var{choice}
@@ -14209,6 +14335,11 @@ Maximum depth of recursive cloning for self-recursive function.
 Recursive cloning only when the probability of call being executed exceeds
 the parameter.
 
+@item ipa-cp-profile-count-base
+When using @option{-fprofile-use} option, IPA-CP will consider the measured
+execution count of a call graph edge at this percentage position in their
+histogram as the basis for its heuristics calculation.
+
 @item ipa-cp-recursive-freq-factor
 The number of times interprocedural copy propagation expects recursive
 functions to call themselves.
@@ -14434,6 +14565,9 @@ finite state automaton jump thread path.
 @item max-fsm-thread-length
 Maximum number of basic blocks on a jump thread path.
 
+@item threader-debug
+threader-debug=[none|all] Enables verbose dumping of the threader solver.
+
 @item parloops-chunk-size
 Chunk size of omp schedule for loops parallelized by parloops.
 
@@ -14467,6 +14601,15 @@ Maximum number of basic blocks before EVRP uses a sparse cache.
 @item evrp-mode
 Specifies the mode Early VRP should operate in.
 
+@item vrp1-mode
+Specifies the mode VRP pass 1 should operate in.
+
+@item vrp2-mode
+Specifies the mode VRP pass 2 should operate in.
+
+@item ranger-debug
+Specifies the type of debug output to be issued for ranges.
+
 @item evrp-switch-limit
 Specifies the maximum number of switch cases before EVRP ignores a switch.
 
@@ -15496,7 +15639,26 @@ which functions and calls should be skipped from instrumentation
 (@pxref{Function Attributes}).
 
 Currently the x86 GNU/Linux target provides an implementation based
-on Intel Control-flow Enforcement Technology (CET).
+on Intel Control-flow Enforcement Technology (CET) which works for
+i686 processor or newer.
+
+@item -fharden-compares
+@opindex fharden-compares
+For every logical test that survives gimple optimizations and is
+@emph{not} the condition in a conditional branch (for example,
+conditions tested for conditional moves, or to store in boolean
+variables), emit extra code to compute and verify the reversed
+condition, and to call @code{__builtin_trap} if the results do not
+match.  Use with @samp{-fharden-conditional-branches} to cover all
+conditionals.
+
+@item -fharden-conditional-branches
+@opindex fharden-conditional-branches
+For every non-vectorized conditional branch that survives gimple
+optimizations, emit extra code to compute and verify the reversed
+condition, and to call @code{__builtin_trap} if the result is
+unexpected.  Use with @samp{-fharden-compares} to cover all
+conditionals.
 
 @item -fstack-protector
 @opindex fstack-protector
@@ -18625,7 +18787,7 @@ performance of the code.  Permissible values for this option are:
 @samp{cortex-a73.cortex-a35}, @samp{cortex-a73.cortex-a53},
 @samp{cortex-a75.cortex-a55}, @samp{cortex-a76.cortex-a55},
 @samp{cortex-r82}, @samp{cortex-x1}, @samp{cortex-x2},
-@samp{cortex-a510}, @samp{cortex-a710}, @samp{native}.
+@samp{cortex-a510}, @samp{cortex-a710}, @samp{ampere1}, @samp{native}.
 
 The values @samp{cortex-a57.cortex-a53}, @samp{cortex-a72.cortex-a53},
 @samp{cortex-a73.cortex-a35}, @samp{cortex-a73.cortex-a53},
@@ -20003,6 +20165,7 @@ Permissible names are:
 @samp{armv7-m}, @samp{armv7e-m},
 @samp{armv8-m.base}, @samp{armv8-m.main},
 @samp{armv8.1-m.main},
+@samp{armv9-a},
 @samp{iwmmxt} and @samp{iwmmxt2}.
 
 Additionally, the following architectures, which lack support for the
@@ -20485,7 +20648,7 @@ Permissible names are: @samp{arm7tdmi}, @samp{arm7tdmi-s}, @samp{arm710t},
 @samp{cortex-a32}, @samp{cortex-a35}, @samp{cortex-a53}, @samp{cortex-a55},
 @samp{cortex-a57}, @samp{cortex-a72}, @samp{cortex-a73}, @samp{cortex-a75},
 @samp{cortex-a76}, @samp{cortex-a76ae}, @samp{cortex-a77},
-@samp{cortex-a78}, @samp{cortex-a78ae}, @samp{cortex-a78c},
+@samp{cortex-a78}, @samp{cortex-a78ae}, @samp{cortex-a78c}, @samp{cortex-a710},
 @samp{ares}, @samp{cortex-r4}, @samp{cortex-r4f}, @samp{cortex-r5},
 @samp{cortex-r7}, @samp{cortex-r8}, @samp{cortex-r52}, @samp{cortex-r52plus},
 @samp{cortex-m0}, @samp{cortex-m0plus}, @samp{cortex-m1}, @samp{cortex-m3},
@@ -27477,7 +27640,7 @@ Supported values for @var{cpu_type} are @samp{401}, @samp{403},
 @samp{e6500}, @samp{ec603e}, @samp{G3}, @samp{G4}, @samp{G5},
 @samp{titan}, @samp{power3}, @samp{power4}, @samp{power5}, @samp{power5+},
 @samp{power6}, @samp{power6x}, @samp{power7}, @samp{power8},
-@samp{power9}, @samp{future}, @samp{powerpc}, @samp{powerpc64},
+@samp{power9}, @samp{power10}, @samp{powerpc}, @samp{powerpc64},
 @samp{powerpc64le}, @samp{rs64}, and @samp{native}.
 
 @option{-mcpu=powerpc}, @option{-mcpu=powerpc64}, and
@@ -27659,10 +27822,9 @@ Enable/disable the @var{__float128} keyword for IEEE 128-bit floating point
 and use either software emulation for IEEE 128-bit floating point or
 hardware instructions.
 
-The VSX instruction set (@option{-mvsx}, @option{-mcpu=power7},
-@option{-mcpu=power8}), or @option{-mcpu=power9} must be enabled to
-use the IEEE 128-bit floating point support.  The IEEE 128-bit
-floating point support only works on PowerPC Linux systems.
+The VSX instruction set (@option{-mvsx}) must be enabled to use the IEEE
+128-bit floating point support.  The IEEE 128-bit floating point is only
+supported on Linux.
 
 The default for @option{-mfloat128} is enabled on PowerPC Linux
 systems using the VSX instruction set, and disabled on other systems.
@@ -28492,24 +28654,25 @@ the offset with a symbol reference to a canary in the TLS block.
 @itemx -mno-pcrel
 @opindex mpcrel
 @opindex mno-pcrel
-Generate (do not generate) pc-relative addressing when the option
-@option{-mcpu=future} is used.  The @option{-mpcrel} option requires
-that the medium code model (@option{-mcmodel=medium}) and prefixed
-addressing (@option{-mprefixed}) options are enabled.
+Generate (do not generate) pc-relative addressing.  The @option{-mpcrel}
+option requires that the medium code model (@option{-mcmodel=medium})
+and prefixed addressing (@option{-mprefixed}) options are enabled.
 
 @item -mprefixed
 @itemx -mno-prefixed
 @opindex mprefixed
 @opindex mno-prefixed
 Generate (do not generate) addressing modes using prefixed load and
-store instructions when the option @option{-mcpu=future} is used.
+store instructions.  The @option{-mprefixed} option requires that
+the option @option{-mcpu=power10} (or later) is enabled.
 
 @item -mmma
 @itemx -mno-mma
 @opindex mmma
 @opindex mno-mma
-Generate (do not generate) the MMA instructions when the option
-@option{-mcpu=future} is used.
+Generate (do not generate) the MMA instructions.  The @option{-mma}
+option requires that the option @option{-mcpu=power10} (or later)
+is enabled.
 
 @item -mrop-protect
 @itemx -mno-rop-protect
@@ -30319,6 +30482,13 @@ GNU assembler is being used.
 @item -mg
 @opindex mg
 Output code for G-format floating-point numbers instead of D-format.
+
+@item -mlra
+@itemx -mno-lra
+@opindex mlra
+@opindex mno-lra
+Enable Local Register Allocation.  This is still experimental for the VAX,
+so by default the compiler uses standard reload.
 @end table
 
 @node Visium Options
@@ -32119,6 +32289,13 @@ Generate code that uses only the general-purpose registers.  This
 prevents the compiler from using floating-point, vector, mask and bound
 registers.
 
+@item -mrelax-cmpxchg-loop
+@opindex mrelax-cmpxchg-loop
+Relax cmpxchg loop by emitting an early load and compare before cmpxchg,
+execute pause if load value is not expected. This reduces excessive
+cachline bouncing when and works for all atomic logic fetch builtins
+that generates compare and swap loop.
+
 @item -mindirect-branch=@var{choice}
 @opindex mindirect-branch
 Convert indirect call and jump with @var{choice}.  The default is
@@ -32779,8 +32956,7 @@ Substitute the contents of spec string @var{name} at this point.
 Accumulate an option for @samp{%X}.
 
 @item %X
-Output the accumulated linker options specified by @option{-Wl} or a @samp{%x}
-spec string.
+Output the accumulated linker options specified by a @samp{%x} spec string.
 
 @item %Y
 Output the accumulated assembler options specified by @option{-Wa}.
diff --git a/gcc/doc/loop.texi b/gcc/doc/loop.texi
index 94eed67..29a5800 100644
--- a/gcc/doc/loop.texi
+++ b/gcc/doc/loop.texi
@@ -249,8 +249,8 @@ are only reliable for the innermost loops:
 @item @code{create_iv}: Creates a new induction variable.  Only works on
 GIMPLE@.  @code{standard_iv_increment_position} can be used to find a
 suitable place for the iv increment.
-@item @code{duplicate_loop_to_header_edge},
-@code{tree_duplicate_loop_to_header_edge}: These functions (on RTL and
+@item @code{duplicate_loop_body_to_header_edge},
+@code{tree_duplicate_loop_body_to_header_edge}: These functions (on RTL and
 on GIMPLE) duplicate the body of the loop prescribed number of times on
 one of the edges entering loop header, thus performing either loop
 unrolling or loop peeling.  @code{can_duplicate_loop_p}
diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
index 6a16576..40b1e0d 100644
--- a/gcc/doc/sourcebuild.texi
+++ b/gcc/doc/sourcebuild.texi
@@ -1846,37 +1846,59 @@ Target supports loop vectorization with partial vectors and
 Target supports loop vectorization with partial vectors and
 @code{vect-partial-vector-usage} is nonzero.
 
-@item vect_slp_v2qi_store
+@item vect_slp_v2qi_store_align
 Target supports vectorization of 2-byte char stores with 2-byte aligned
 address at plain @option{-O2}.
 
-@item vect_slp_v4qi_store
+@item vect_slp_v4qi_store_align
 Target supports vectorization of 4-byte char stores with 4-byte aligned
 address at plain @option{-O2}.
 
-@item vect_slp_v8qi_store
-Target supports vectorization of 8-byte char stores with 8-byte aligned
-address at plain @option{-O2}.
+@item vect_slp_v4qi_store_unalign
+Target supports vectorization of 4-byte char stores with unaligned address
+at plain @option{-O2}.
 
-@item vect_slp_v16qi_store
-Target supports vectorization of 16-byte char stores with 16-byte aligned
-address at plain @option{-O2}.
+@item struct_4char_block_move
+Target supports block move for 8-byte aligned 4-byte size struct initialization.
+
+@item vect_slp_v4qi_store_unalign_1
+Target supports vectorization of 4-byte char stores with unaligned address
+or store them with constant pool at plain @option{-O2}.
+
+@item struct_8char_block_move
+Target supports block move for 8-byte aligned 8-byte size struct initialization.
+
+@item vect_slp_v8qi_store_unalign_1
+Target supports vectorization of 8-byte char stores with unaligned address
+or store them with constant pool at plain @option{-O2}.
+
+@item struct_16char_block_move
+Target supports block move for 8-byte aligned 16-byte size struct
+initialization.
 
-@item vect_slp_v2hi_store
+@item vect_slp_v16qi_store_unalign_1
+Target supports vectorization of 16-byte char stores with unaligned address
+or store them with constant pool at plain @option{-O2}.
+
+@item vect_slp_v2hi_store_align
 Target supports vectorization of 4-byte short stores with 4-byte aligned
-address at plain @option{-O2}.
+addressat plain @option{-O2}.
 
-@item vect_slp_v4hi_store
-Target supports vectorization of 8-byte short stores with 8-byte aligned
-address at plain @option{-O2}.
+@item vect_slp_v2hi_store_unalign
+Target supports vectorization of 4-byte short stores with unaligned address
+at plain @option{-O2}.
 
-@item vect_slp_v2si_store
-Target supports vectorization of 8-byte int stores with 8-byte aligned
-address at plain @option{-O2}.
+@item vect_slp_v4hi_store_unalign
+Target supports vectorization of 8-byte short stores with unaligned address
+at plain @option{-O2}.
 
-@item vect_slp_v4si_store
-Target supports vectorization of 16-byte int stores with 16-byte aligned
-address at plain @option{-O2}.
+@item vect_slp_v2si_store_align
+Target supports vectorization of 8-byte int stores with 8-byte aligned address
+at plain @option{-O2}.
+
+@item vect_slp_v4si_store_unalign
+Target supports vectorization of 16-byte int stores with unaligned address
+at plain @option{-O2}.
 @end table
 
 @subsubsection Thread Local Storage attributes
diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi
index fa66a09..8b7f710 100644
--- a/gcc/doc/tm.texi
+++ b/gcc/doc/tm.texi
@@ -6580,8 +6580,8 @@ all zeros.  GCC can then try to branch around the instruction instead.
 @end deftypefn
 @c hook-end
 
-@deftypefn {Target Hook} {void *} TARGET_VECTORIZE_INIT_COST (class loop *@var{loop_info}, bool @var{costing_for_scalar})
-@c hook-start:TARGET_VECTORIZE_INIT_COST
+@deftypefn {Target Hook} {class vector_costs *} TARGET_VECTORIZE_CREATE_COSTS (vec_info *@var{vinfo}, bool @var{costing_for_scalar})
+@c hook-start:TARGET_VECTORIZE_CREATE_COSTS
 This hook should initialize target-specific data structures in preparation
 for modeling the costs of vectorizing a loop or basic block.  The default
 allocates three unsigned integers for accumulating costs for the prologue,
@@ -6593,35 +6593,6 @@ it is for the vector version.
 @end deftypefn
 @c hook-end
 
-@deftypefn {Target Hook} unsigned TARGET_VECTORIZE_ADD_STMT_COST (class vec_info *@var{}, void *@var{data}, int @var{count}, enum vect_cost_for_stmt @var{kind}, class _stmt_vec_info *@var{stmt_info}, tree @var{vectype}, int @var{misalign}, enum vect_cost_model_location @var{where})
-@c hook-start:TARGET_VECTORIZE_ADD_STMT_COST
-This hook should update the target-specific @var{data} in response to
-adding @var{count} copies of the given @var{kind} of statement to a
-loop or basic block.  The default adds the builtin vectorizer cost for
-the copies of the statement to the accumulator specified by @var{where},
-(the prologue, body, or epilogue) and returns the amount added.  The
-return value should be viewed as a tentative cost that may later be
-revised.
-@end deftypefn
-@c hook-end
-
-@deftypefn {Target Hook} void TARGET_VECTORIZE_FINISH_COST (void *@var{data}, unsigned *@var{prologue_cost}, unsigned *@var{body_cost}, unsigned *@var{epilogue_cost})
-@c hook-start:TARGET_VECTORIZE_FINISH_COST
-This hook should complete calculations of the cost of vectorizing a loop
-or basic block based on @var{data}, and return the prologue, body, and
-epilogue costs as unsigned integers.  The default returns the value of
-the three accumulators.
-@end deftypefn
-@c hook-end
-
-@deftypefn {Target Hook} void TARGET_VECTORIZE_DESTROY_COST_DATA (void *@var{data})
-@c hook-start:TARGET_VECTORIZE_DESTROY_COST_DATA
-This hook should release @var{data} and any related data structures
-allocated by TARGET_VECTORIZE_INIT_COST.  The default releases the
-accumulator.
-@end deftypefn
-@c hook-end
-
 @deftypefn {Target Hook} tree TARGET_VECTORIZE_BUILTIN_GATHER (const_tree @var{mem_vectype}, const_tree @var{index_type}, int @var{scale})
 @c hook-start:TARGET_VECTORIZE_BUILTIN_GATHER
 Target builtin that implements vector gather operation.  @var{mem_vectype}
@@ -9857,6 +9828,14 @@ It is false if we must use @command{collect2}.
 @end deftypevr
 @c hook-end
 
+@deftypevr {Target Hook} bool TARGET_DTORS_FROM_CXA_ATEXIT
+This value is true if the target wants destructors to be queued to be
+run from __cxa_atexit.  If this is the case then, for each priority level,
+a new constructor will be entered that registers the destructors for that
+level with __cxa_atexit (and there will be no destructors emitted).
+It is false the method implied by @code{have_ctors_dtors} is used.
+@end deftypevr
+
 @deftypefn {Target Hook} void TARGET_ASM_CONSTRUCTOR (rtx @var{symbol}, int @var{priority})
 @c hook-start:TARGET_ASM_CONSTRUCTOR
 If defined, a function that outputs assembler code to arrange to call
diff --git a/gcc/doc/tm.texi.in b/gcc/doc/tm.texi.in
index 193c9bd..2b9960b 100644
--- a/gcc/doc/tm.texi.in
+++ b/gcc/doc/tm.texi.in
@@ -4182,13 +4182,7 @@ address;  but often a machine-dependent strategy can generate better code.
 
 @hook TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE
 
-@hook TARGET_VECTORIZE_INIT_COST
-
-@hook TARGET_VECTORIZE_ADD_STMT_COST
-
-@hook TARGET_VECTORIZE_FINISH_COST
-
-@hook TARGET_VECTORIZE_DESTROY_COST_DATA
+@hook TARGET_VECTORIZE_CREATE_COSTS
 
 @hook TARGET_VECTORIZE_BUILTIN_GATHER
 
@@ -6021,6 +6015,8 @@ encountering an @code{init_priority} attribute.
 
 @hook TARGET_HAVE_CTORS_DTORS
 
+@hook TARGET_DTORS_FROM_CXA_ATEXIT
+
 @hook TARGET_ASM_CONSTRUCTOR
 
 @hook TARGET_ASM_DESTRUCTOR
diff --git a/gcc/dumpfile.c b/gcc/dumpfile.c
index e6ead5d..8169daf 100644
--- a/gcc/dumpfile.c
+++ b/gcc/dumpfile.c
@@ -145,7 +145,6 @@ static const kv_pair<dump_flags_t> dump_options[] =
   {"missed", MSG_MISSED_OPTIMIZATION},
   {"note", MSG_NOTE},
   {"optall", MSG_ALL_KINDS},
-  {"threading", TDF_THREADING},
   {"all", dump_flags_t (TDF_ALL_VALUES
 			& ~(TDF_RAW | TDF_SLIM | TDF_LINENO | TDF_GRAPH
 			    | TDF_STMTADDR | TDF_RHS_ONLY | TDF_NOUID
diff --git a/gcc/dumpfile.h b/gcc/dumpfile.h
index 7622675..2487ae2 100644
--- a/gcc/dumpfile.h
+++ b/gcc/dumpfile.h
@@ -146,20 +146,17 @@ enum dump_flag : uint32_t
   /* Dump folding details.  */
   TDF_FOLDING = (1 << 21),
 
-  /* Dumping for range path solver.  */
-  TDF_THREADING = (1 << 22),
-
   /* MSG_* flags for expressing the kinds of message to
      be emitted by -fopt-info.  */
 
   /* -fopt-info optimized sources.  */
-  MSG_OPTIMIZED_LOCATIONS = (1 << 23),
+  MSG_OPTIMIZED_LOCATIONS = (1 << 22),
 
   /* Missed opportunities.  */
-  MSG_MISSED_OPTIMIZATION = (1 << 24),
+  MSG_MISSED_OPTIMIZATION = (1 << 23),
 
   /* General optimization info.  */
-  MSG_NOTE = (1 << 25),
+  MSG_NOTE = (1 << 24),
 
   /* Mask for selecting MSG_-kind flags.  */
   MSG_ALL_KINDS = (MSG_OPTIMIZED_LOCATIONS
@@ -178,16 +175,16 @@ enum dump_flag : uint32_t
      sub-option of -fopt-info to show the internal messages.  */
 
   /* Implicitly supplied for messages at the top-level dump scope.  */
-  MSG_PRIORITY_USER_FACING = (1 << 26),
+  MSG_PRIORITY_USER_FACING = (1 << 25),
 
   /* Implicitly supplied for messages within nested dump scopes.  */
-  MSG_PRIORITY_INTERNALS = (1 << 27),
+  MSG_PRIORITY_INTERNALS = (1 << 26),
 
   /* Supplied when an opt_problem generated in a nested scope is re-emitted
      at the top-level.   We want to default to showing these in -fopt-info
      output, but to *not* show them in dump files, as the message would be
      shown twice, messing up "scan-tree-dump-times" in DejaGnu tests.  */
-  MSG_PRIORITY_REEMITTED = (1 << 28),
+  MSG_PRIORITY_REEMITTED = (1 << 27),
 
   /* Mask for selecting MSG_PRIORITY_* flags.  */
   MSG_ALL_PRIORITIES = (MSG_PRIORITY_USER_FACING
@@ -195,16 +192,16 @@ enum dump_flag : uint32_t
 			| MSG_PRIORITY_REEMITTED),
 
   /* All -fdump- flags.  */
-  TDF_ALL_VALUES = (1 << 29) - 1,
+  TDF_ALL_VALUES = (1 << 28) - 1,
 
   /* Dumping for -fcompare-debug.  */
-  TDF_COMPARE_DEBUG = (1 << 29),
+  TDF_COMPARE_DEBUG = (1 << 28),
 
   /* Dump a GIMPLE value which means wrapping certain things with _Literal.  */
-  TDF_GIMPLE_VAL = (1 << 30),
+  TDF_GIMPLE_VAL = (1 << 29),
 
   /* For error.  */
-  TDF_ERROR = ((uint32_t)1 << 31),
+  TDF_ERROR = ((uint32_t)1 << 30),
 };
 
 /* Dump flags type.  */
diff --git a/gcc/dwarf2out.c b/gcc/dwarf2out.c
index 20f2c5d..e1d6a79 100644
--- a/gcc/dwarf2out.c
+++ b/gcc/dwarf2out.c
@@ -3854,7 +3854,7 @@ static void add_AT_location_description	(dw_die_ref, enum dwarf_attribute,
 					 dw_loc_list_ref);
 static void add_data_member_location_attribute (dw_die_ref, tree,
 						struct vlr_context *);
-static bool add_const_value_attribute (dw_die_ref, rtx);
+static bool add_const_value_attribute (dw_die_ref, machine_mode, rtx);
 static void insert_int (HOST_WIDE_INT, unsigned, unsigned char *);
 static void insert_wide_int (const wide_int &, unsigned char *, int);
 static unsigned insert_float (const_rtx, unsigned char *);
@@ -19646,6 +19646,7 @@ field_byte_offset (const_tree decl, struct vlr_context *ctx,
      properly dynamic byte offsets only when PCC bitfield type doesn't
      matter.  */
   if (PCC_BITFIELD_TYPE_MATTERS
+      && DECL_BIT_FIELD_TYPE (decl)
       && TREE_CODE (DECL_FIELD_OFFSET (decl)) == INTEGER_CST)
     {
       offset_int object_offset_in_bits;
@@ -20081,8 +20082,10 @@ insert_float (const_rtx rtl, unsigned char *array)
    constants do not necessarily get memory "homes".  */
 
 static bool
-add_const_value_attribute (dw_die_ref die, rtx rtl)
+add_const_value_attribute (dw_die_ref die, machine_mode mode, rtx rtl)
 {
+  scalar_mode int_mode;
+
   switch (GET_CODE (rtl))
     {
     case CONST_INT:
@@ -20097,15 +20100,15 @@ add_const_value_attribute (dw_die_ref die, rtx rtl)
       return true;
 
     case CONST_WIDE_INT:
-      {
-	wide_int w1 = rtx_mode_t (rtl, MAX_MODE_INT);
-	unsigned int prec = MIN (wi::min_precision (w1, UNSIGNED),
-				 (unsigned int) CONST_WIDE_INT_NUNITS (rtl)
-				 * HOST_BITS_PER_WIDE_INT);
-	wide_int w = wide_int::from (w1, prec, UNSIGNED);
-	add_AT_wide (die, DW_AT_const_value, w);
-      }
-      return true;
+      if (is_int_mode (mode, &int_mode)
+	  && (GET_MODE_PRECISION (int_mode)
+	      & (HOST_BITS_PER_WIDE_INT - 1)) == 0)
+	{
+	  wide_int w = rtx_mode_t (rtl, int_mode);
+	  add_AT_wide (die, DW_AT_const_value, w);
+	  return true;
+	}
+      return false;
 
     case CONST_DOUBLE:
       /* Note that a CONST_DOUBLE rtx could represent either an integer or a
@@ -20190,7 +20193,7 @@ add_const_value_attribute (dw_die_ref die, rtx rtl)
 
     case CONST:
       if (CONSTANT_P (XEXP (rtl, 0)))
-	return add_const_value_attribute (die, XEXP (rtl, 0));
+	return add_const_value_attribute (die, mode, XEXP (rtl, 0));
       /* FALLTHROUGH */
     case SYMBOL_REF:
       if (!const_ok_for_output (rtl))
@@ -20703,7 +20706,7 @@ add_location_or_const_value_attribute (dw_die_ref die, tree decl, bool cache_p)
 
   rtl = rtl_for_decl_location (decl);
   if (rtl && (CONSTANT_P (rtl) || GET_CODE (rtl) == CONST_STRING)
-      && add_const_value_attribute (die, rtl))
+      && add_const_value_attribute (die, DECL_MODE (decl), rtl))
     return true;
 
   /* See if we have single element location list that is equivalent to
@@ -20724,7 +20727,7 @@ add_location_or_const_value_attribute (dw_die_ref die, tree decl, bool cache_p)
       if (GET_CODE (rtl) == EXPR_LIST)
 	rtl = XEXP (rtl, 0);
       if ((CONSTANT_P (rtl) || GET_CODE (rtl) == CONST_STRING)
-	  && add_const_value_attribute (die, rtl))
+	  && add_const_value_attribute (die, DECL_MODE (decl), rtl))
 	 return true;
     }
   /* If this decl is from BLOCK_NONLOCALIZED_VARS, we might need its
@@ -20799,7 +20802,7 @@ tree_add_const_value_attribute (dw_die_ref die, tree t)
      symbols.  */
   rtl = rtl_for_decl_init (init, type);
   if (rtl && !early_dwarf)
-    return add_const_value_attribute (die, rtl);
+    return add_const_value_attribute (die, TYPE_MODE (type), rtl);
   /* If the host and target are sane, try harder.  */
   if (CHAR_BIT == 8 && BITS_PER_UNIT == 8
       && initializer_constant_valid_p (init, type))
diff --git a/gcc/expmed.c b/gcc/expmed.c
index bbdd0e7..4abce11 100644
--- a/gcc/expmed.c
+++ b/gcc/expmed.c
@@ -1735,7 +1735,8 @@ extract_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
       FOR_EACH_MODE_FROM (new_mode, new_mode)
 	if (known_eq (GET_MODE_SIZE (new_mode), GET_MODE_SIZE (GET_MODE (op0)))
 	    && known_eq (GET_MODE_UNIT_SIZE (new_mode), GET_MODE_SIZE (tmode))
-	    && targetm.vector_mode_supported_p (new_mode))
+	    && targetm.vector_mode_supported_p (new_mode)
+	    && targetm.modes_tieable_p (GET_MODE (op0), new_mode))
 	  break;
       if (new_mode != VOIDmode)
 	op0 = gen_lowpart (new_mode, op0);
diff --git a/gcc/expmed.h b/gcc/expmed.h
index 93cd631..6b13ea9 100644
--- a/gcc/expmed.h
+++ b/gcc/expmed.h
@@ -133,15 +133,6 @@ struct alg_hash_entry {
 #define NUM_ALG_HASH_ENTRIES 307
 #endif
 
-#define NUM_MODE_INT \
-  (MAX_MODE_INT - MIN_MODE_INT + 1)
-#define NUM_MODE_PARTIAL_INT \
-  (MIN_MODE_PARTIAL_INT == E_VOIDmode ? 0 \
-   : MAX_MODE_PARTIAL_INT - MIN_MODE_PARTIAL_INT + 1)
-#define NUM_MODE_VECTOR_INT \
-  (MIN_MODE_VECTOR_INT == E_VOIDmode ? 0 \
-   : MAX_MODE_VECTOR_INT - MIN_MODE_VECTOR_INT + 1)
-
 #define NUM_MODE_IP_INT (NUM_MODE_INT + NUM_MODE_PARTIAL_INT)
 #define NUM_MODE_IPV_INT (NUM_MODE_IP_INT + NUM_MODE_VECTOR_INT)
 
diff --git a/gcc/expr.c b/gcc/expr.c
index eb33643..5673902 100644
--- a/gcc/expr.c
+++ b/gcc/expr.c
@@ -2508,19 +2508,6 @@ emit_group_load_1 (rtx *tmps, rtx dst, rtx orig_src, tree type,
 					   NULL);
 	    }
 	}
-      /* FIXME: A SIMD parallel will eventually lead to a subreg of a
-	 SIMD register, which is currently broken.  While we get GCC
-	 to emit proper RTL for these cases, let's dump to memory.  */
-      else if (VECTOR_MODE_P (GET_MODE (dst))
-	       && REG_P (src))
-	{
-	  poly_uint64 slen = GET_MODE_SIZE (GET_MODE (src));
-	  rtx mem;
-
-	  mem = assign_stack_temp (GET_MODE (src), slen);
-	  emit_move_insn (mem, src);
-	  tmps[i] = adjust_address (mem, mode, bytepos);
-	}
       else if (CONSTANT_P (src) && GET_MODE (dst) != BLKmode
                && XVECLEN (dst, 0) > 1)
         tmps[i] = simplify_gen_subreg (mode, src, GET_MODE (dst), bytepos);
diff --git a/gcc/flag-types.h b/gcc/flag-types.h
index ae0b216..cfd2a5f 100644
--- a/gcc/flag-types.h
+++ b/gcc/flag-types.h
@@ -449,18 +449,40 @@ enum parloops_schedule_type
   PARLOOPS_SCHEDULE_RUNTIME
 };
 
+/* Ranger debug mode.  */
+enum ranger_debug
+{
+  RANGER_DEBUG_NONE = 0,
+  RANGER_DEBUG_TRACE = 1,
+  RANGER_DEBUG_CACHE = 2,
+  RANGER_DEBUG_GORI = 4,
+  RANGER_DEBUG_TRACE_GORI = (RANGER_DEBUG_TRACE | RANGER_DEBUG_GORI),
+  RANGER_DEBUG_TRACE_CACHE = (RANGER_DEBUG_TRACE | RANGER_DEBUG_CACHE),
+  RANGER_DEBUG_ALL = (RANGER_DEBUG_GORI | RANGER_DEBUG_CACHE
+		      | RANGER_DEBUG_TRACE)
+};
+
+/* Jump threader verbose dumps.  */
+enum threader_debug
+{
+  THREADER_DEBUG_NONE = 0,
+  THREADER_DEBUG_ALL = 1
+};
+
 /* EVRP mode.  */
 enum evrp_mode
 {
-  EVRP_MODE_RVRP_ONLY = 0,
-  EVRP_MODE_EVRP_ONLY = 1,
-  EVRP_MODE_EVRP_FIRST = 2,
-  EVRP_MODE_RVRP_FIRST = 3,
-  EVRP_MODE_TRACE = 4,
-  EVRP_MODE_CACHE = (8 | EVRP_MODE_TRACE),
-  EVRP_MODE_GORI = 16,
-  EVRP_MODE_TRACE_GORI = (EVRP_MODE_TRACE | EVRP_MODE_GORI),
-  EVRP_MODE_DEBUG = (EVRP_MODE_GORI | EVRP_MODE_CACHE)
+  EVRP_MODE_RVRP_ONLY,
+  EVRP_MODE_EVRP_ONLY,
+  EVRP_MODE_EVRP_FIRST,
+  EVRP_MODE_RVRP_FIRST
+};
+
+/* VRP modes.  */
+enum vrp_mode
+{
+  VRP_MODE_VRP,
+  VRP_MODE_RANGER
 };
 
 /* Modes of OpenACC 'kernels' constructs handling.  */
diff --git a/gcc/fold-const.c b/gcc/fold-const.c
index ff23f12..90d8225 100644
--- a/gcc/fold-const.c
+++ b/gcc/fold-const.c
@@ -2139,6 +2139,12 @@ fold_convert_const_real_from_real (tree type, const_tree arg1)
       && REAL_VALUE_ISSIGNALING_NAN (TREE_REAL_CST (arg1)))
     return NULL_TREE; 
 
+  /* With flag_rounding_math we should respect the current rounding mode
+     unless the conversion is exact.  */
+  if (HONOR_SIGN_DEPENDENT_ROUNDING (arg1)
+      && !exact_real_truncate (TYPE_MODE (type), &TREE_REAL_CST (arg1)))
+    return NULL_TREE;
+
   real_convert (&value, TYPE_MODE (type), &TREE_REAL_CST (arg1));
   t = build_real (type, value);
 
@@ -2284,7 +2290,20 @@ fold_convert_const (enum tree_code code, tree type, tree arg1)
   else if (TREE_CODE (type) == REAL_TYPE)
     {
       if (TREE_CODE (arg1) == INTEGER_CST)
-	return build_real_from_int_cst (type, arg1);
+	{
+	  tree res = build_real_from_int_cst (type, arg1);
+	  /* Avoid the folding if flag_rounding_math is on and the
+	     conversion is not exact.  */
+	  if (HONOR_SIGN_DEPENDENT_ROUNDING (type))
+	    {
+	      bool fail = false;
+	      wide_int w = real_to_integer (&TREE_REAL_CST (res), &fail,
+					    TYPE_PRECISION (TREE_TYPE (arg1)));
+	      if (fail || wi::ne_p (w, wi::to_wide (arg1)))
+		return NULL_TREE;
+	    }
+	  return res;
+	}
       else if (TREE_CODE (arg1) == REAL_CST)
 	return fold_convert_const_real_from_real (type, arg1);
       else if (TREE_CODE (arg1) == FIXED_CST)
@@ -8772,6 +8791,7 @@ native_interpret_expr (tree type, const unsigned char *ptr, int len)
     case BOOLEAN_TYPE:
     case POINTER_TYPE:
     case REFERENCE_TYPE:
+    case OFFSET_TYPE:
       return native_interpret_int (type, ptr, len);
 
     case REAL_TYPE:
@@ -8808,6 +8828,7 @@ can_native_interpret_type_p (tree type)
     case REAL_TYPE:
     case COMPLEX_TYPE:
     case VECTOR_TYPE:
+    case OFFSET_TYPE:
       return true;
     default:
       return false;
@@ -13920,6 +13941,18 @@ fold_build_call_array_loc (location_t loc, tree type, tree fn,
   folding_initializer = saved_folding_initializer;
 
 tree
+fold_init (tree expr)
+{
+  tree result;
+  START_FOLD_INIT;
+
+  result = fold (expr);
+
+  END_FOLD_INIT;
+  return result;
+}
+
+tree
 fold_build1_initializer_loc (location_t loc, enum tree_code code,
 			     tree type, tree op)
 {
@@ -16473,6 +16506,132 @@ tree_nonzero_bits (const_tree t)
   return wi::shwi (-1, TYPE_PRECISION (TREE_TYPE (t)));
 }
 
+/* Helper function for address compare simplifications in match.pd.
+   OP0 and OP1 are ADDR_EXPR operands being compared by CODE.
+   BASE0, BASE1, OFF0 and OFF1 are set by the function.
+   GENERIC is true if GENERIC folding and false for GIMPLE folding.
+   Returns 0 if OP0 is known to be unequal to OP1 regardless of OFF{0,1},
+   1 if bases are known to be equal and OP0 cmp OP1 depends on OFF0 cmp OFF1,
+   and 2 if unknown.  */
+
+int
+address_compare (tree_code code, tree type, tree op0, tree op1,
+		 tree &base0, tree &base1, poly_int64 &off0, poly_int64 &off1,
+		 bool generic)
+{
+  gcc_checking_assert (TREE_CODE (op0) == ADDR_EXPR);
+  gcc_checking_assert (TREE_CODE (op1) == ADDR_EXPR);
+  base0 = get_addr_base_and_unit_offset (TREE_OPERAND (op0, 0), &off0);
+  base1 = get_addr_base_and_unit_offset (TREE_OPERAND (op1, 0), &off1);
+  if (base0 && TREE_CODE (base0) == MEM_REF)
+    {
+      off0 += mem_ref_offset (base0).force_shwi ();
+      base0 = TREE_OPERAND (base0, 0);
+    }
+  if (base1 && TREE_CODE (base1) == MEM_REF)
+    {
+      off1 += mem_ref_offset (base1).force_shwi ();
+      base1 = TREE_OPERAND (base1, 0);
+    }
+  if (base0 == NULL_TREE || base1 == NULL_TREE)
+    return 2;
+
+  int equal = 2;
+  /* Punt in GENERIC on variables with value expressions;
+     the value expressions might point to fields/elements
+     of other vars etc.  */
+  if (generic
+      && ((VAR_P (base0) && DECL_HAS_VALUE_EXPR_P (base0))
+	  || (VAR_P (base1) && DECL_HAS_VALUE_EXPR_P (base1))))
+    return 2;
+  else if (decl_in_symtab_p (base0) && decl_in_symtab_p (base1))
+    {
+      symtab_node *node0 = symtab_node::get_create (base0);
+      symtab_node *node1 = symtab_node::get_create (base1);
+      equal = node0->equal_address_to (node1);
+    }
+  else if ((DECL_P (base0)
+	    || TREE_CODE (base0) == SSA_NAME
+	    || TREE_CODE (base0) == STRING_CST)
+	   && (DECL_P (base1)
+	       || TREE_CODE (base1) == SSA_NAME
+	       || TREE_CODE (base1) == STRING_CST))
+    equal = (base0 == base1);
+  if (equal == 1)
+    {
+      if (code == EQ_EXPR
+	  || code == NE_EXPR
+	  /* If the offsets are equal we can ignore overflow.  */
+	  || known_eq (off0, off1)
+	  || TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (op0))
+	  /* Or if we compare using pointers to decls or strings.  */
+	  || (POINTER_TYPE_P (type)
+	      && (DECL_P (base0) || TREE_CODE (base0) == STRING_CST)))
+	return 1;
+      return 2;
+    }
+  if (equal != 0)
+    return equal;
+  if (code != EQ_EXPR && code != NE_EXPR)
+    return 2;
+
+  HOST_WIDE_INT ioff0 = -1, ioff1 = -1;
+  off0.is_constant (&ioff0);
+  off1.is_constant (&ioff1);
+  if ((DECL_P (base0) && TREE_CODE (base1) == STRING_CST)
+       || (TREE_CODE (base0) == STRING_CST && DECL_P (base1))
+       || (TREE_CODE (base0) == STRING_CST
+	   && TREE_CODE (base1) == STRING_CST
+	   && ioff0 >= 0 && ioff1 >= 0
+	   && ioff0 < TREE_STRING_LENGTH (base0)
+	   && ioff1 < TREE_STRING_LENGTH (base1)
+	  /* This is a too conservative test that the STRING_CSTs
+	     will not end up being string-merged.  */
+	   && strncmp (TREE_STRING_POINTER (base0) + ioff0,
+		       TREE_STRING_POINTER (base1) + ioff1,
+		       MIN (TREE_STRING_LENGTH (base0) - ioff0,
+			    TREE_STRING_LENGTH (base1) - ioff1)) != 0))
+    ;
+  else if (!DECL_P (base0) || !DECL_P (base1))
+    return 2;
+  /* If this is a pointer comparison, ignore for now even
+     valid equalities where one pointer is the offset zero
+     of one object and the other to one past end of another one.  */
+  else if (!INTEGRAL_TYPE_P (type))
+    ;
+  /* Assume that automatic variables can't be adjacent to global
+     variables.  */
+  else if (is_global_var (base0) != is_global_var (base1))
+    ;
+  else
+    {
+      tree sz0 = DECL_SIZE_UNIT (base0);
+      tree sz1 = DECL_SIZE_UNIT (base1);
+      /* If sizes are unknown, e.g. VLA or not representable, punt.  */
+      if (!tree_fits_poly_int64_p (sz0) || !tree_fits_poly_int64_p (sz1))
+	return 2;
+
+      poly_int64 size0 = tree_to_poly_int64 (sz0);
+      poly_int64 size1 = tree_to_poly_int64 (sz1);
+      /* If one offset is pointing (or could be) to the beginning of one
+	 object and the other is pointing to one past the last byte of the
+	 other object, punt.  */
+      if (maybe_eq (off0, 0) && maybe_eq (off1, size1))
+	equal = 2;
+      else if (maybe_eq (off1, 0) && maybe_eq (off0, size0))
+	equal = 2;
+      /* If both offsets are the same, there are some cases we know that are
+	 ok.  Either if we know they aren't zero, or if we know both sizes
+	 are no zero.  */
+      if (equal == 2
+	  && known_eq (off0, off1)
+	  && (known_ne (off0, 0)
+	      || (known_ne (size0, 0) && known_ne (size1, 0))))
+	equal = 0;
+     }
+  return equal;
+}
+
 #if CHECKING_P
 
 namespace selftest {
diff --git a/gcc/fold-const.h b/gcc/fold-const.h
index 7bac84b..56e9d39 100644
--- a/gcc/fold-const.h
+++ b/gcc/fold-const.h
@@ -44,6 +44,7 @@ extern void shift_bytes_in_array_right (unsigned char *, unsigned int,
    subexpressions are not changed.  */
 
 extern tree fold (tree);
+extern tree fold_init (tree);
 #define fold_unary(CODE,T1,T2)\
    fold_unary_loc (UNKNOWN_LOCATION, CODE, T1, T2)
 extern tree fold_unary_loc (location_t, enum tree_code, tree, tree);
@@ -213,6 +214,8 @@ extern bool negate_mathfn_p (combined_fn);
 extern const char *getbyterep (tree, unsigned HOST_WIDE_INT *);
 extern const char *c_getstr (tree);
 extern wide_int tree_nonzero_bits (const_tree);
+extern int address_compare (tree_code, tree, tree, tree, tree &, tree &,
+			    poly_int64 &, poly_int64 &, bool);
 
 /* Return OFF converted to a pointer offset type suitable as offset for
    POINTER_PLUS_EXPR.  Use location LOC for this conversion.  */
diff --git a/gcc/fortran/ChangeLog b/gcc/fortran/ChangeLog
index f75b880..10dfca4 100644
--- a/gcc/fortran/ChangeLog
+++ b/gcc/fortran/ChangeLog
@@ -1,3 +1,395 @@
+2021-11-15  Tobias Burnus  <tobias@codesourcery.com>
+
+	* openmp.c (OMP_TARGET_CLAUSES): Add thread_limit.
+	* trans-openmp.c (gfc_split_omp_clauses): Add thread_limit also to
+	teams.
+
+2021-11-12  Tobias Burnus  <tobias@codesourcery.com>
+
+	* parse.c (decode_omp_directive): Fix permitting 'nowait' for some
+	combined directives, add missing 'omp end ... loop'.
+	(gfc_ascii_statement): Fix ST_OMP_END_TEAMS_LOOP result.
+	* openmp.c (resolve_omp_clauses): Add missing combined loop constructs
+	case values to the 'if(directive-name: ...)' check.
+	* trans-openmp.c (gfc_split_omp_clauses): Put nowait on target if
+	first leaf construct accepting it.
+
+2021-11-12  Martin Jambor  <mjambor@suse.cz>
+
+	* trans-types.c (gfc_get_array_descr_info): Use build_debug_expr_decl
+	instead of building DEBUG_EXPR_DECL manually.
+
+2021-11-12  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/97896
+	* intrinsic.c (add_sym_4ind): Remove.
+	(add_functions): Use add_sym4 instead of add_sym4ind.
+	Don’t special case the index intrinsic.
+	* iresolve.c (gfc_resolve_index_func): Use the individual arguments
+	directly instead of the full argument list.
+	* intrinsic.h (gfc_resolve_index_func): Update the declaration
+	accordingly.
+	* trans-decl.c (gfc_get_extern_function_decl): Don’t modify the
+	list of arguments in the case of the index intrinsic.
+	* trans-array.h (gfc_get_intrinsic_for_expr,
+	gfc_get_proc_ifc_for_expr): New.
+	* trans-array.c (gfc_get_intrinsic_for_expr,
+	arg_evaluated_for_scalarization): New.
+	(gfc_walk_elemental_function_args): Add intrinsic procedure
+	as argument.  Count arguments.  Check arg_evaluated_for_scalarization.
+	* trans-intrinsic.c (gfc_walk_intrinsic_function): Update call.
+	* trans-stmt.c (get_intrinsic_for_code): New.
+	(gfc_trans_call): Update call.
+
+2021-11-12  Jakub Jelinek  <jakub@redhat.com>
+
+	* types.def (BT_FN_VOID_UINT_UINT): Remove.
+	(BT_FN_BOOL_UINT_UINT_UINT_BOOL): New.
+
+2021-11-11  Tobias Burnus  <tobias@codesourcery.com>
+
+	* gfortran.h (struct gfc_omp_clauses): Rename num_teams to
+	num_teams_upper, add num_teams_upper.
+	* dump-parse-tree.c (show_omp_clauses): Update to handle
+	lower-bound num_teams clause.
+	* frontend-passes.c (gfc_code_walker): Likewise
+	* openmp.c (gfc_free_omp_clauses, gfc_match_omp_clauses,
+	resolve_omp_clauses): Likewise.
+	* trans-openmp.c (gfc_trans_omp_clauses, gfc_split_omp_clauses,
+	gfc_trans_omp_target): Likewise.
+
+2021-11-11  Jakub Jelinek  <jakub@redhat.com>
+
+	* trans-openmp.c (gfc_trans_omp_clauses): Use
+	OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR instead of OMP_CLAUSE_NUM_TEAMS_EXPR.
+
+2021-11-10  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/103137
+	PR fortran/103138
+	* check.c (gfc_check_shape): Avoid NULL pointer dereference on
+	missing ref.
+	* simplify.c (gfc_simplify_cshift): Avoid NULL pointer dereference
+	when shape not set.
+	(gfc_simplify_transpose): Likewise.
+
+2021-11-09  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	PR fortran/68800
+	* class.c (generate_finalization_wrapper): Do not leak
+	finalization wrappers if they will not be used.
+	* expr.c (gfc_free_actual_arglist): Formatting fix.
+	* gfortran.h (gfc_free_symbol): Pass argument by reference.
+	(gfc_release_symbol): Likewise.
+	(gfc_free_namespace): Likewise.
+	* symbol.c (gfc_release_symbol): Adjust acordingly.
+	(free_components): Set procedure pointer components
+	of derived types to NULL after freeing.
+	(free_tb_tree): Likewise.
+	(gfc_free_symbol): Set sym to NULL after freeing.
+	(gfc_free_namespace): Set namespace to NULL after freeing.
+
+2021-11-09  Martin Liska  <mliska@suse.cz>
+
+	* symbol.c (gfc_get_ultimate_derived_super_type): Remove.
+
+2021-11-09  Aldy Hernandez  <aldyh@redhat.com>
+
+	* misc.c (gfc_dummy_typename): Make sure ts->kind is
+	non-negative.
+
+2021-11-07  Thomas Koenig  <tkoenig@gcc.gnu.org>
+
+	* intrinsic.c (add_subroutines): Change keyword "operator"
+	to the correct one, "operation".
+	* check.c (gfc_check_co_reduce): Change OPERATOR to
+	OPERATION in error messages.
+	* intrinsic.texi: Change OPERATOR to OPERATION in
+	documentation.
+
+2021-11-07  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* interface.c (gfc_compare_actual_formal): Continue checking
+	all arguments after encountering an error.
+	* intrinsic.c (do_ts29113_check): Likewise.
+	* resolve.c (resolve_operator): Continue resolving on op2 error.
+
+2021-11-06  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/102715
+	* decl.c (add_init_expr_to_sym): Reject rank mismatch between
+	array and its initializer.
+
+2021-11-05  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/102817
+	* expr.c (simplify_parameter_variable): Copy shape of referenced
+	subobject when simplifying.
+
+2021-11-05  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/69419
+	* match.c (gfc_match_common): Check array spec of a symbol in a
+	COMMON object list and reject it if it is a coarray.
+
+2021-11-05  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	PR fortran/100972
+	* decl.c (gfc_match_implicit_none): Fix typo in warning.
+	* resolve.c (resolve_unknown_f): Reject external procedures
+	without explicit EXTERNAL attribute whe IMPLICIT none (external)
+	is in effect.
+
+2021-11-05  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* decl.c (gfc_insert_kind_parameter_exprs): Make static.
+	* expr.c (gfc_build_init_expr): Make static
+	(gfc_build_default_init_expr): Move below its static helper.
+	* gfortran.h (gfc_insert_kind_parameter_exprs, gfc_add_saved_common,
+	gfc_add_common, gfc_use_derived_tree, gfc_free_charlen,
+	gfc_get_ultimate_derived_super_type,
+	gfc_resolve_oacc_parallel_loop_blocks, gfc_build_init_expr,
+	gfc_iso_c_sub_interface): Delete.
+	* symbol.c (gfc_new_charlen, gfc_get_derived_super_type): Make
+	static.
+
+2021-11-05  Sandra Loosemore  <sandra@codesourcery.com>
+
+	PR fortran/35276
+	* gfortran.texi (Mixed-Language Programming): Talk about C++,
+	and how to link.
+
+2021-11-04  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* gfortran.texi (Projects): Add bullet for helping with
+	incomplete standards compliance.
+	(Proposed Extensions): Delete section.
+
+2021-11-04  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* intrinsic.texi (Introduction to Intrinsics): Genericize
+	references to standard versions.
+	* invoke.texi (-fall-intrinsics): Likewise.
+	(-fmax-identifier-length=): Likewise.
+
+2021-11-04  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* gfortran.texi (Interoperability with C): Copy-editing.  Add
+	more index entries.
+	(Intrinsic Types): Likewise.
+	(Derived Types and struct): Likewise.
+	(Interoperable Global Variables): Likewise.
+	(Interoperable Subroutines and Functions): Likewise.
+	(Working with C Pointers): Likewise.
+	(Further Interoperability of Fortran with C): Likewise.  Rewrite
+	to reflect that this is now fully supported by gfortran.
+
+2021-11-04  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* gfortran.texi (About GNU Fortran): Consolidate material
+	formerly in other sections.  Copy-editing.
+	(Preprocessing and conditional compilation): Delete, moving
+	most material to invoke.texi.
+	(GNU Fortran and G77): Delete.
+	(Project Status): Delete.
+	(Standards): Update.
+	(Fortran 95 status): Mention conditional compilation here.
+	(Fortran 2003 status): Rewrite to mention the 1 missing feature
+	instead of all the ones implemented.
+	(Fortran 2008 status): Similarly for the 2 missing features.
+	(Fortran 2018 status): Rewrite to reflect completion of TS29113
+	feature support.
+	* invoke.texi (Preprocessing Options): Move material formerly
+	in introductory chapter here.
+
+2021-11-04  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* gfortran.texi (Standards): Move discussion of specific
+	standard versions here....
+	(Fortran standards status): ...from here, and delete this node.
+
+2021-10-31  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* symbol.c (gfc_get_typebound_proc): Revert memcpy.
+
+2021-10-31  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* resolve.c (resolve_fl_procedure): Initialize
+	allocatable_or_pointer.
+
+2021-10-30  Manfred Schwarb  <manfred99@gmx.ch>
+
+	* intrinsic.texi: Remove entries for SHORT and LONG intrinsics.
+
+2021-10-30  Manfred Schwarb  <manfred99@gmx.ch>
+
+	* check.c (gfc_check_intconv): Change error message.
+
+2021-10-30  Manfred Schwarb  <manfred99@gmx.ch>
+
+	* intrinsic.texi (REAL): Fix entries in Specific names table.
+
+2021-10-30  Manfred Schwarb  <manfred99@gmx.ch>
+
+	* intrinsic.texi: Adjust @columnfractions commands to improve
+	appearance for narrow 80 character terminals.
+
+2021-10-30  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* parse.c (clean_up_modules): Free gsym.
+
+2021-10-30  Harald Anlauf  <anlauf@gmx.de>
+
+	* gfortran.texi (bug reports): credit Gerhard Steinmetz for
+	  numerous bug reports.
+
+2021-10-30  Steve Kargl  <kargl@gcc.gnu.org>
+
+	PR fortran/99853
+	* resolve.c (resolve_select): Generate regular gfc_error on
+	invalid conversions instead of an gfc_internal_error.
+
+2021-10-29  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* symbol.c (free_tb_tree): Free type-bound procedure struct.
+	(gfc_get_typebound_proc): Use explicit memcpy for clarity.
+
+2021-10-27  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* intrinsic.h (gfc_check_sum, gfc_resolve_atan2d, gfc_resolve_kill,
+	gfc_resolve_kill_sub): Delete declaration.
+
+2021-10-27  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* trans-types.h (gfc_convert_function_code): Delete.
+
+2021-10-27  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* trans-stmt.h (gfc_trans_deallocate_array): Delete.
+
+2021-10-27  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* trans-array.c (gfc_trans_scalarized_loop_end): Make static.
+	* trans-array.h (gfc_trans_scalarized_loop_end,
+	gfc_conv_tmp_ref, gfc_conv_array_transpose): Delete declaration.
+
+2021-10-27  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* constructor.c (gfc_constructor_get_base): Make static.
+	(gfc_constructor_expr_foreach, gfc_constructor_swap): Delete.
+	* constructor.h (gfc_constructor_get_base): Remove declaration.
+	(gfc_constructor_expr_foreach, gfc_constructor_swap): Delete.
+
+2021-10-27  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* decl.c (gfc_match_old_kind_spec, set_com_block_bind_c,
+	set_verify_bind_c_sym, set_verify_bind_c_com_block,
+	get_bind_c_idents, gfc_match_suffix, gfc_get_type_attr_spec,
+	check_extended_derived_type): Make static.
+	(gfc_match_gcc_unroll): Add comment.
+	* match.c (gfc_match_small_int_expr): Delete definition.
+	* match.h (gfc_match_small_int_expr): Delete declaration.
+	(gfc_match_name_C, gfc_match_old_kind_spec, set_com_block_bind_c,
+	set_verify_bind_c_sym, set_verify_bind_c_com_block,
+	get_bind_c_idents, gfc_match_suffix,
+	gfc_get_type_attr_spec): Delete declaration.
+
+2021-10-27  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	* expr.c (is_non_empty_structure_constructor): Make static.
+	* gfortran.h (gfc_check_any_c_kind): Delete.
+	* match.c (gfc_match_label): Make static.
+	* match.h (gfc_match_label): Delete declaration.
+	* scanner.c (file_changes_cur, file_changes_count,
+	file_changes_allocated): Make static.
+	* trans-expr.c (gfc_get_character_len): Make static.
+	(gfc_class_len_or_zero_get): Make static.
+	(VTAB_GET_FIELD_GEN): Undefine.
+	(gfc_get_class_array_ref): Make static.
+	(gfc_finish_interface_mapping): Make static.
+	* trans-types.c (gfc_check_any_c_kind): Delete.
+	(pfunc_type_node, dtype_type_node, gfc_get_ppc_type): Make static.
+	* trans-types.h (gfc_get_ppc_type): Delete declaration.
+	* trans.c (gfc_msg_wrong_return): Delete.
+	* trans.h (gfc_class_len_or_zero_get, gfc_class_vtab_extends_get,
+	gfc_vptr_extends_get, gfc_get_class_array_ref, gfc_get_character_len,
+	gfc_finish_interface_mapping, gfc_msg_wrong_return,
+	gfc_get_function_decl): Delete declaration.
+
+2021-10-27  Tobias Burnus  <tobias@codesourcery.com>
+
+	* trans-stmt.c (gfc_trans_select_rank_cases): Fix condition
+	for allocatables/pointers.
+
+2021-10-26  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/102956
+	* symbol.c (gfc_check_conflict): Add conflict check for PDT KIND
+	and LEN type parameters.
+
+2021-10-26  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/102917
+	* decl.c (match_attr_spec): Remove invalid integer kind checks on
+	KIND and LEN attributes of PDTs.
+
+2021-10-26  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/102816
+	* resolve.c (resolve_structure_cons): Reject invalid array spec of
+	a DT component referenced in a structure constructor.
+
+2021-10-26  Tobias Burnus  <tobias@codesourcery.com>
+
+	PR fortran/102885
+	* trans-decl.c (gfc_conv_cfi_to_gfc): Properly handle nonconstant
+	character lenghts.
+
+2021-10-25  Andrew MacLeod  <amacleod@redhat.com>
+
+	* trans-decl.c (gfc_conv_cfi_to_gfc): Initialize rank to NULL_TREE.
+
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Make-lang.in: Allow dvi-formatted documentation
+	to be installed.
+
+2021-10-22  Tobias Burnus  <tobias@codesourcery.com>
+
+	PR fortran/92621
+	* trans-expr.c (gfc_trans_assignment_1): Add STRIP_NOPS.
+
+2021-10-21  Chung-Lin Tang  <cltang@codesourcery.com>
+
+	* decl.c (gfc_match_end): Add COMP_OMP_STRICTLY_STRUCTURED_BLOCK case
+	together with COMP_BLOCK.
+	* parse.c (parse_omp_structured_block): Change return type to
+	'gfc_statement', add handling for strictly-structured block case, adjust
+	recursive calls to parse_omp_structured_block.
+	(parse_executable): Adjust calls to parse_omp_structured_block.
+	* parse.h (enum gfc_compile_state): Add
+	COMP_OMP_STRICTLY_STRUCTURED_BLOCK.
+	* trans-openmp.c (gfc_trans_omp_workshare): Add EXEC_BLOCK case
+	handling.
+
+2021-10-21  Sandra Loosemore  <sandra@codesourcery.com>
+
+	PR fortran/94070
+	* expr.c (gfc_simplify_expr): Handle GFC_ISYM_SHAPE along with
+	GFC_ISYM_LBOUND and GFC_ISYM_UBOUND.
+	* trans-array.c (gfc_conv_ss_startstride): Likewise.
+	(set_loop_bounds): Likewise.
+	* trans-intrinsic.c (gfc_trans_intrinsic_bound): Extend to
+	handle SHAPE.  Correct logic for zero-size special cases and
+	detecting assumed-rank arrays associated with an assumed-size
+	argument.
+	(gfc_conv_intrinsic_shape): Deleted.
+	(gfc_conv_intrinsic_function): Handle GFC_ISYM_SHAPE like
+	GFC_ISYM_LBOUND and GFC_ISYM_UBOUND.
+	(gfc_add_intrinsic_ss_code): Likewise.
+	(gfc_walk_intrinsic_bound): Likewise.
+
 2021-10-20  Chung-Lin Tang  <cltang@codesourcery.com>
 
 	* openmp.c (gfc_match_omp_clause_reduction): Add 'openmp_target' default
diff --git a/gcc/fortran/Make-lang.in b/gcc/fortran/Make-lang.in
index 63195a9..58ce589 100644
--- a/gcc/fortran/Make-lang.in
+++ b/gcc/fortran/Make-lang.in
@@ -117,7 +117,20 @@ fortran.tags: force
 	etags --include TAGS.sub --include ../TAGS.sub
 
 fortran.info: doc/gfortran.info doc/gfc-internals.info
-fortran.dvi: doc/gfortran.dvi doc/gfc-internals.dvi
+
+F95_DVIFILES = doc/gfortran.dvi
+
+fortran.dvi: $(F95_DVIFILES) doc/gfc-internals.dvi
+
+fortran.install-dvi: $(F95_DVIFILES)
+	@$(NORMAL_INSTALL)
+	test -z "$(dvidir)/gcc" || $(mkinstalldirs) "$(DESTDIR)$(dvidir)/gcc"
+	@list='$(F95_DVIFILES)'; for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  f=$(dvi__strip_dir) \
+	  echo " $(INSTALL_DATA) '$$d$$p' '$(DESTDIR)$(dvidir)/gcc/$$f'"; \
+	  $(INSTALL_DATA) "$$d$$p" "$(DESTDIR)$(dvidir)/gcc/$$f"; \
+	done
 
 F95_HTMLFILES = $(build_htmldir)/gfortran
 
diff --git a/gcc/fortran/check.c b/gcc/fortran/check.c
index cfaf9d2..ffa07b5 100644
--- a/gcc/fortran/check.c
+++ b/gcc/fortran/check.c
@@ -2265,7 +2265,7 @@ gfc_check_co_reduce (gfc_expr *a, gfc_expr *op, gfc_expr *result_image,
   attr = gfc_expr_attr (op);
   if (!attr.pure || !attr.function)
     {
-      gfc_error ("OPERATOR argument at %L must be a PURE function",
+      gfc_error ("OPERATION argument at %L must be a PURE function",
 		 &op->where);
       return false;
     }
@@ -2292,7 +2292,7 @@ gfc_check_co_reduce (gfc_expr *a, gfc_expr *op, gfc_expr *result_image,
 
   if (!formal || !formal->next || formal->next->next)
     {
-      gfc_error ("The function passed as OPERATOR at %L shall have two "
+      gfc_error ("The function passed as OPERATION at %L shall have two "
 		 "arguments", &op->where);
       return false;
     }
@@ -2303,7 +2303,7 @@ gfc_check_co_reduce (gfc_expr *a, gfc_expr *op, gfc_expr *result_image,
   if (!gfc_compare_types (&a->ts, &sym->result->ts))
     {
       gfc_error ("The A argument at %L has type %s but the function passed as "
-		 "OPERATOR at %L returns %s",
+		 "OPERATION at %L returns %s",
 		 &a->where, gfc_typename (a), &op->where,
 		 gfc_typename (&sym->result->ts));
       return false;
@@ -2311,7 +2311,7 @@ gfc_check_co_reduce (gfc_expr *a, gfc_expr *op, gfc_expr *result_image,
   if (!gfc_compare_types (&a->ts, &formal->sym->ts)
       || !gfc_compare_types (&a->ts, &formal->next->sym->ts))
     {
-      gfc_error ("The function passed as OPERATOR at %L has arguments of type "
+      gfc_error ("The function passed as OPERATION at %L has arguments of type "
 		 "%s and %s but shall have type %s", &op->where,
 		 gfc_typename (&formal->sym->ts),
 		 gfc_typename (&formal->next->sym->ts), gfc_typename (a));
@@ -2322,7 +2322,7 @@ gfc_check_co_reduce (gfc_expr *a, gfc_expr *op, gfc_expr *result_image,
       || formal->next->sym->attr.allocatable || formal->sym->attr.pointer
       || formal->next->sym->attr.pointer)
     {
-      gfc_error ("The function passed as OPERATOR at %L shall have scalar "
+      gfc_error ("The function passed as OPERATION at %L shall have scalar "
 		 "nonallocatable nonpointer arguments and return a "
 		 "nonallocatable nonpointer scalar", &op->where);
       return false;
@@ -2330,21 +2330,21 @@ gfc_check_co_reduce (gfc_expr *a, gfc_expr *op, gfc_expr *result_image,
 
   if (formal->sym->attr.value != formal->next->sym->attr.value)
     {
-      gfc_error ("The function passed as OPERATOR at %L shall have the VALUE "
+      gfc_error ("The function passed as OPERATION at %L shall have the VALUE "
 		 "attribute either for none or both arguments", &op->where);
       return false;
     }
 
   if (formal->sym->attr.target != formal->next->sym->attr.target)
     {
-      gfc_error ("The function passed as OPERATOR at %L shall have the TARGET "
+      gfc_error ("The function passed as OPERATION at %L shall have the TARGET "
 		 "attribute either for none or both arguments", &op->where);
       return false;
     }
 
   if (formal->sym->attr.asynchronous != formal->next->sym->attr.asynchronous)
     {
-      gfc_error ("The function passed as OPERATOR at %L shall have the "
+      gfc_error ("The function passed as OPERATION at %L shall have the "
 		 "ASYNCHRONOUS attribute either for none or both arguments",
 		 &op->where);
       return false;
@@ -2352,7 +2352,7 @@ gfc_check_co_reduce (gfc_expr *a, gfc_expr *op, gfc_expr *result_image,
 
   if (formal->sym->attr.optional || formal->next->sym->attr.optional)
     {
-      gfc_error ("The function passed as OPERATOR at %L shall not have the "
+      gfc_error ("The function passed as OPERATION at %L shall not have the "
 		 "OPTIONAL attribute for either of the arguments", &op->where);
       return false;
     }
@@ -2383,14 +2383,14 @@ gfc_check_co_reduce (gfc_expr *a, gfc_expr *op, gfc_expr *result_image,
 	       || (formal_size2 && actual_size != formal_size2)))
 	{
 	  gfc_error ("The character length of the A argument at %L and of the "
-		     "arguments of the OPERATOR at %L shall be the same",
+		     "arguments of the OPERATION at %L shall be the same",
 		     &a->where, &op->where);
 	  return false;
 	}
       if (actual_size && result_size && actual_size != result_size)
 	{
 	  gfc_error ("The character length of the A argument at %L and of the "
-		     "function result of the OPERATOR at %L shall be the same",
+		     "function result of the OPERATION at %L shall be the same",
 		     &a->where, &op->where);
 	  return false;
 	}
@@ -3240,7 +3240,7 @@ gfc_check_intconv (gfc_expr *x)
   if (strcmp (gfc_current_intrinsic, "short") == 0
       || strcmp (gfc_current_intrinsic, "long") == 0)
     {
-      gfc_error ("%qs intrinsic subprogram at %L has been deprecated.  "
+      gfc_error ("%qs intrinsic subprogram at %L has been removed.  "
 		 "Use INT intrinsic subprogram.", gfc_current_intrinsic,
 		 &x->where);
       return false;
@@ -5096,6 +5096,9 @@ gfc_check_shape (gfc_expr *source, gfc_expr *kind)
   if (source->rank == 0 || source->expr_type != EXPR_VARIABLE)
     return true;
 
+  if (source->ref == NULL)
+    return false;
+
   ar = gfc_find_array_ref (source);
 
   if (ar->as && ar->as->type == AS_ASSUMED_SIZE && ar->type == AR_FULL)
diff --git a/gcc/fortran/class.c b/gcc/fortran/class.c
index 93118ad..6b01766 100644
--- a/gcc/fortran/class.c
+++ b/gcc/fortran/class.c
@@ -1602,7 +1602,6 @@ generate_finalization_wrapper (gfc_symbol *derived, gfc_namespace *ns,
   gfc_code *last_code, *block;
   char *name;
   bool finalizable_comp = false;
-  bool expr_null_wrapper = false;
   gfc_expr *ancestor_wrapper = NULL, *rank;
   gfc_iterator *iter;
 
@@ -1636,7 +1635,11 @@ generate_finalization_wrapper (gfc_symbol *derived, gfc_namespace *ns,
       && !derived->attr.alloc_comp
       && (!derived->f2k_derived || !derived->f2k_derived->finalizers)
       && !has_finalizer_component (derived))
-    expr_null_wrapper = true;
+    {
+      vtab_final->initializer = gfc_get_null_expr (NULL);
+      gcc_assert (vtab_final->ts.interface == NULL);
+      return;
+    }
   else
     /* Check whether there are new allocatable components.  */
     for (comp = derived->components; comp; comp = comp->next)
@@ -1650,7 +1653,7 @@ generate_finalization_wrapper (gfc_symbol *derived, gfc_namespace *ns,
 
   /* If there is no new finalizer and no new allocatable, return with
      an expr to the ancestor's one.  */
-  if (!expr_null_wrapper && !finalizable_comp
+  if (!finalizable_comp
       && (!derived->f2k_derived || !derived->f2k_derived->finalizers))
     {
       gcc_assert (ancestor_wrapper && ancestor_wrapper->ref == NULL
@@ -1674,8 +1677,7 @@ generate_finalization_wrapper (gfc_symbol *derived, gfc_namespace *ns,
   /* Set up the namespace.  */
   sub_ns = gfc_get_namespace (ns, 0);
   sub_ns->sibling = ns->contained;
-  if (!expr_null_wrapper)
-    ns->contained = sub_ns;
+  ns->contained = sub_ns;
   sub_ns->resolved = 1;
 
   /* Set up the procedure symbol.  */
@@ -1691,7 +1693,7 @@ generate_finalization_wrapper (gfc_symbol *derived, gfc_namespace *ns,
   final->ts.kind = 4;
   final->attr.artificial = 1;
   final->attr.always_explicit = 1;
-  final->attr.if_source = expr_null_wrapper ? IFSRC_IFBODY : IFSRC_DECL;
+  final->attr.if_source = IFSRC_DECL;
   if (ns->proc_name->attr.flavor == FL_MODULE)
     final->module = ns->proc_name->name;
   gfc_set_sym_referenced (final);
@@ -1741,15 +1743,6 @@ generate_finalization_wrapper (gfc_symbol *derived, gfc_namespace *ns,
   final->formal->next->next->sym = fini_coarray;
   gfc_commit_symbol (fini_coarray);
 
-  /* Return with a NULL() expression but with an interface which has
-     the formal arguments.  */
-  if (expr_null_wrapper)
-    {
-      vtab_final->initializer = gfc_get_null_expr (NULL);
-      vtab_final->ts.interface = final;
-      return;
-    }
-
   /* Local variables.  */
 
   gfc_get_symbol ("idx", sub_ns, &idx);
diff --git a/gcc/fortran/constructor.c b/gcc/fortran/constructor.c
index 3e4377a..4b5a748 100644
--- a/gcc/fortran/constructor.c
+++ b/gcc/fortran/constructor.c
@@ -85,7 +85,8 @@ gfc_constructor_get (void)
   return c;
 }
 
-gfc_constructor_base gfc_constructor_get_base (void)
+static gfc_constructor_base
+gfc_constructor_get_base (void)
 {
   return splay_tree_new (splay_tree_compare_ints, NULL, node_free);
 }
@@ -209,23 +210,6 @@ gfc_constructor_lookup_expr (gfc_constructor_base base, int offset)
 }
 
 
-int
-gfc_constructor_expr_foreach (gfc_constructor *ctor ATTRIBUTE_UNUSED,
-			      int(*f)(gfc_expr *) ATTRIBUTE_UNUSED)
-{
-  gcc_assert (0);
-  return 0;
-}
-
-void
-gfc_constructor_swap (gfc_constructor *ctor ATTRIBUTE_UNUSED,
-                      int n ATTRIBUTE_UNUSED, int m ATTRIBUTE_UNUSED)
-{
-  gcc_assert (0);
-}
-
-
-
 gfc_constructor *
 gfc_constructor_first (gfc_constructor_base base)
 {
diff --git a/gcc/fortran/constructor.h b/gcc/fortran/constructor.h
index 85a72dc..25cd6a8 100644
--- a/gcc/fortran/constructor.h
+++ b/gcc/fortran/constructor.h
@@ -23,8 +23,6 @@ along with GCC; see the file COPYING3.  If not see
 /* Get a new constructor structure.  */
 gfc_constructor *gfc_constructor_get (void);
 
-gfc_constructor_base gfc_constructor_get_base (void);
-
 /* Copy a constructor structure.  */
 gfc_constructor_base gfc_constructor_copy (gfc_constructor_base base);
 
@@ -64,14 +62,6 @@ gfc_constructor *gfc_constructor_lookup (gfc_constructor_base base, int n);
 */
 gfc_expr *gfc_constructor_lookup_expr (gfc_constructor_base base, int n);
 
-
-int gfc_constructor_expr_foreach (gfc_constructor *ctor, int(*)(gfc_expr *));
-
-
-void gfc_constructor_swap (gfc_constructor *ctor, int n, int m);
-
-
-
 /* Get the first constructor node in the constructure structure.
    Returns NULL if there is no such expression.  */
 gfc_constructor *gfc_constructor_first (gfc_constructor_base base);
diff --git a/gcc/fortran/decl.c b/gcc/fortran/decl.c
index 6043e10..c0fec90 100644
--- a/gcc/fortran/decl.c
+++ b/gcc/fortran/decl.c
@@ -2105,6 +2105,14 @@ add_init_expr_to_sym (const char *name, gfc_expr **initp, locus *var_locus)
 	    }
 	}
 
+      if (sym->attr.flavor == FL_PARAMETER && sym->attr.dimension && sym->as
+	  && sym->as->rank && init->rank && init->rank != sym->as->rank)
+	{
+	  gfc_error ("Rank mismatch of array at %L and its initializer "
+		     "(%d/%d)", &sym->declared_at, sym->as->rank, init->rank);
+	  return false;
+	}
+
       /* If sym is implied-shape, set its upper bounds from init.  */
       if (sym->attr.flavor == FL_PARAMETER && sym->attr.dimension
 	  && sym->as->type == AS_IMPLIED_SHAPE)
@@ -3128,7 +3136,7 @@ cleanup:
    This assumes that the byte size is equal to the kind number for
    non-COMPLEX types, and equal to twice the kind number for COMPLEX.  */
 
-match
+static match
 gfc_match_old_kind_spec (gfc_typespec *ts)
 {
   match m;
@@ -3713,7 +3721,7 @@ insert_parameter_exprs (gfc_expr* e, gfc_symbol* sym ATTRIBUTE_UNUSED,
 }
 
 
-bool
+static bool
 gfc_insert_kind_parameter_exprs (gfc_expr *e)
 {
   return gfc_traverse_expr (e, NULL, &insert_parameter_exprs, 0);
@@ -4715,7 +4723,7 @@ gfc_match_implicit_none (void)
   if (c == '(')
     {
       (void) gfc_next_ascii_char ();
-      if (!gfc_notify_std (GFC_STD_F2018, "IMPORT NONE with spec list at %C"))
+      if (!gfc_notify_std (GFC_STD_F2018, "IMPLICIT NONE with spec list at %C"))
 	return MATCH_ERROR;
 
       gfc_gobble_whitespace ();
@@ -5592,14 +5600,6 @@ match_attr_spec (void)
 		  m = MATCH_ERROR;
 		  goto cleanup;
 		}
-	      if (current_ts.kind != gfc_default_integer_kind)
-		{
-		  gfc_error ("Component with KIND attribute at %C must be "
-			     "default integer kind (%d)",
-			      gfc_default_integer_kind);
-		  m = MATCH_ERROR;
-		  goto cleanup;
-		}
 	    }
 	  else if (d == DECL_LEN)
 	    {
@@ -5619,14 +5619,6 @@ match_attr_spec (void)
 		  m = MATCH_ERROR;
 		  goto cleanup;
 		}
-	      if (current_ts.kind != gfc_default_integer_kind)
-		{
-		  gfc_error ("Component with LEN attribute at %C must be "
-			     "default integer kind (%d)",
-			      gfc_default_integer_kind);
-		  m = MATCH_ERROR;
-		  goto cleanup;
-		}
 	    }
 	  else
 	    {
@@ -5867,7 +5859,7 @@ set_binding_label (const char **dest_label, const char *sym_name,
 /* Set the status of the given common block as being BIND(C) or not,
    depending on the given parameter, is_bind_c.  */
 
-void
+static void
 set_com_block_bind_c (gfc_common_head *com_block, int is_bind_c)
 {
   com_block->is_bind_c = is_bind_c;
@@ -6055,7 +6047,7 @@ verify_bind_c_sym (gfc_symbol *tmp_sym, gfc_typespec *ts,
    the type is C interoperable.  Errors are reported by the functions
    used to set/test these fields.  */
 
-bool
+static bool
 set_verify_bind_c_sym (gfc_symbol *tmp_sym, int num_idents)
 {
   bool retval = true;
@@ -6075,7 +6067,7 @@ set_verify_bind_c_sym (gfc_symbol *tmp_sym, int num_idents)
 /* Set the fields marking the given common block as BIND(C), including
    a binding label, and report any errors encountered.  */
 
-bool
+static bool
 set_verify_bind_c_com_block (gfc_common_head *com_block, int num_idents)
 {
   bool retval = true;
@@ -6095,7 +6087,7 @@ set_verify_bind_c_com_block (gfc_common_head *com_block, int num_idents)
 /* Retrieve the list of one or more identifiers that the given bind(c)
    attribute applies to.  */
 
-bool
+static bool
 get_bind_c_idents (void)
 {
   char name[GFC_MAX_SYMBOL_LEN + 1];
@@ -6804,7 +6796,7 @@ match_result (gfc_symbol *function, gfc_symbol **result)
    clause and BIND(C), either one, or neither.  The draft does not
    require them to come in a specific order.  */
 
-match
+static match
 gfc_match_suffix (gfc_symbol *sym, gfc_symbol **result)
 {
   match is_bind_c;   /* Found bind(c).  */
@@ -10116,7 +10108,7 @@ check_extended_derived_type (char *name)
    not a handled attribute, and MATCH_YES otherwise.  TODO: More error
    checking on attribute conflicts needs to be done.  */
 
-match
+static match
 gfc_get_type_attr_spec (symbol_attribute *attr, char *name)
 {
   /* See if the derived type is marked as private.  */
@@ -11794,6 +11786,7 @@ gfc_match_gcc_unroll (void)
 {
   int value;
 
+  /* FIXME: use gfc_match_small_literal_int instead, delete small_int  */
   if (gfc_match_small_int (&value) == MATCH_YES)
     {
       if (value < 0 || value > USHRT_MAX)
diff --git a/gcc/fortran/dump-parse-tree.c b/gcc/fortran/dump-parse-tree.c
index 14a3078..04660d5 100644
--- a/gcc/fortran/dump-parse-tree.c
+++ b/gcc/fortran/dump-parse-tree.c
@@ -1741,10 +1741,15 @@ show_omp_clauses (gfc_omp_clauses *omp_clauses)
 	}
       fprintf (dumpfile, " BIND(%s)", type);
     }
-  if (omp_clauses->num_teams)
+  if (omp_clauses->num_teams_upper)
     {
       fputs (" NUM_TEAMS(", dumpfile);
-      show_expr (omp_clauses->num_teams);
+      if (omp_clauses->num_teams_lower)
+	{
+	  show_expr (omp_clauses->num_teams_lower);
+	  fputc (':', dumpfile);
+	}
+      show_expr (omp_clauses->num_teams_upper);
       fputc (')', dumpfile);
     }
   if (omp_clauses->device)
diff --git a/gcc/fortran/expr.c b/gcc/fortran/expr.c
index b19d3a2..4084d18 100644
--- a/gcc/fortran/expr.c
+++ b/gcc/fortran/expr.c
@@ -543,7 +543,7 @@ gfc_free_actual_arglist (gfc_actual_arglist *a1)
     {
       a2 = a1->next;
       if (a1->expr)
-      gfc_free_expr (a1->expr);
+	gfc_free_expr (a1->expr);
       free (a1);
       a1 = a2;
     }
@@ -2128,6 +2128,8 @@ simplify_parameter_variable (gfc_expr *p, int type)
       if (e == NULL)
 	return false;
 
+      gfc_free_shape (&e->shape, e->rank);
+      e->shape = gfc_copy_shape (p->shape, p->rank);
       e->rank = p->rank;
 
       if (e->ts.type == BT_CHARACTER && p->ts.u.cl)
@@ -4587,21 +4589,12 @@ gfc_check_assign_symbol (gfc_symbol *sym, gfc_component *comp, gfc_expr *rvalue)
   return true;
 }
 
-/* Invoke gfc_build_init_expr to create an initializer expression, but do not
- * require that an expression be built.  */
-
-gfc_expr *
-gfc_build_default_init_expr (gfc_typespec *ts, locus *where)
-{
-  return gfc_build_init_expr (ts, where, false);
-}
-
 /* Build an initializer for a local integer, real, complex, logical, or
    character variable, based on the command line flags finit-local-zero,
    finit-integer=, finit-real=, finit-logical=, and finit-character=.
    With force, an initializer is ALWAYS generated.  */
 
-gfc_expr *
+static gfc_expr *
 gfc_build_init_expr (gfc_typespec *ts, locus *where, bool force)
 {
   gfc_expr *init_expr;
@@ -4758,6 +4751,15 @@ gfc_build_init_expr (gfc_typespec *ts, locus *where, bool force)
   return init_expr;
 }
 
+/* Invoke gfc_build_init_expr to create an initializer expression, but do not
+ * require that an expression be built.  */
+
+gfc_expr *
+gfc_build_default_init_expr (gfc_typespec *ts, locus *where)
+{
+  return gfc_build_init_expr (ts, where, false);
+}
+
 /* Apply an initialization expression to a typespec. Can be used for symbols or
    components. Similar to add_init_expr_to_sym in decl.c; could probably be
    combined with some effort.  */
@@ -4817,7 +4819,7 @@ gfc_apply_init (gfc_typespec *ts, symbol_attribute *attr, gfc_expr *init)
 /* Check whether an expression is a structure constructor and whether it has
    other values than NULL.  */
 
-bool
+static bool
 is_non_empty_structure_constructor (gfc_expr * e)
 {
   if (e->expr_type != EXPR_STRUCTURE)
diff --git a/gcc/fortran/frontend-passes.c b/gcc/fortran/frontend-passes.c
index 145bff5..f5ba7ce 100644
--- a/gcc/fortran/frontend-passes.c
+++ b/gcc/fortran/frontend-passes.c
@@ -5634,7 +5634,8 @@ gfc_code_walker (gfc_code **c, walk_code_fn_t codefn, walk_expr_fn_t exprfn,
 		  WALK_SUBEXPR (co->ext.omp_clauses->chunk_size);
 		  WALK_SUBEXPR (co->ext.omp_clauses->safelen_expr);
 		  WALK_SUBEXPR (co->ext.omp_clauses->simdlen_expr);
-		  WALK_SUBEXPR (co->ext.omp_clauses->num_teams);
+		  WALK_SUBEXPR (co->ext.omp_clauses->num_teams_lower);
+		  WALK_SUBEXPR (co->ext.omp_clauses->num_teams_upper);
 		  WALK_SUBEXPR (co->ext.omp_clauses->device);
 		  WALK_SUBEXPR (co->ext.omp_clauses->thread_limit);
 		  WALK_SUBEXPR (co->ext.omp_clauses->dist_chunk_size);
diff --git a/gcc/fortran/gfortran.h b/gcc/fortran/gfortran.h
index 66192c0..1ad2f0d 100644
--- a/gcc/fortran/gfortran.h
+++ b/gcc/fortran/gfortran.h
@@ -1502,7 +1502,8 @@ typedef struct gfc_omp_clauses
   struct gfc_expr *chunk_size;
   struct gfc_expr *safelen_expr;
   struct gfc_expr *simdlen_expr;
-  struct gfc_expr *num_teams;
+  struct gfc_expr *num_teams_lower;
+  struct gfc_expr *num_teams_upper;
   struct gfc_expr *device;
   struct gfc_expr *thread_limit;
   struct gfc_expr *grainsize;
@@ -3116,7 +3117,6 @@ struct gfc_vect_builtin_tuple
 extern hash_map<nofree_string_hash, int> *gfc_vectorized_builtins;
 
 /* Handling Parameterized Derived Types  */
-bool gfc_insert_kind_parameter_exprs (gfc_expr *);
 bool gfc_insert_parameter_exprs (gfc_expr *, gfc_actual_arglist *);
 match gfc_get_pdt_instance (gfc_actual_arglist *, gfc_symbol **,
 			    gfc_actual_arglist **);
@@ -3284,7 +3284,6 @@ bool gfc_check_character_range (gfc_char_t, int);
 extern bool gfc_seen_div0;
 
 /* trans-types.c */
-bool gfc_check_any_c_kind (gfc_typespec *);
 int gfc_validate_kind (bt, int, bool);
 int gfc_get_int_kind_from_width_isofortranenv (int size);
 int gfc_get_real_kind_from_width_isofortranenv (int size);
@@ -3349,11 +3348,9 @@ bool gfc_add_threadprivate (symbol_attribute *, const char *, locus *);
 bool gfc_add_omp_declare_target (symbol_attribute *, const char *, locus *);
 bool gfc_add_omp_declare_target_link (symbol_attribute *, const char *,
 				      locus *);
-bool gfc_add_saved_common (symbol_attribute *, locus *);
 bool gfc_add_target (symbol_attribute *, locus *);
 bool gfc_add_dummy (symbol_attribute *, const char *, locus *);
 bool gfc_add_generic (symbol_attribute *, const char *, locus *);
-bool gfc_add_common (symbol_attribute *, locus *);
 bool gfc_add_in_common (symbol_attribute *, const char *, locus *);
 bool gfc_add_in_equivalence (symbol_attribute *, const char *, locus *);
 bool gfc_add_data (symbol_attribute *, const char *, locus *);
@@ -3388,7 +3385,6 @@ bool gfc_copy_attr (symbol_attribute *, symbol_attribute *, locus *);
 int gfc_copy_dummy_sym (gfc_symbol **, gfc_symbol *, int);
 bool gfc_add_component (gfc_symbol *, const char *, gfc_component **);
 gfc_symbol *gfc_use_derived (gfc_symbol *);
-gfc_symtree *gfc_use_derived_tree (gfc_symtree *);
 gfc_component *gfc_find_component (gfc_symbol *, const char *, bool, bool,
                                    gfc_ref **);
 
@@ -3404,8 +3400,8 @@ void gfc_delete_symtree (gfc_symtree **, const char *);
 gfc_symtree *gfc_get_unique_symtree (gfc_namespace *);
 gfc_user_op *gfc_get_uop (const char *);
 gfc_user_op *gfc_find_uop (const char *, gfc_namespace *);
-void gfc_free_symbol (gfc_symbol *);
-void gfc_release_symbol (gfc_symbol *);
+void gfc_free_symbol (gfc_symbol *&);
+void gfc_release_symbol (gfc_symbol *&);
 gfc_symbol *gfc_new_symbol (const char *, gfc_namespace *);
 gfc_symtree* gfc_find_symtree_in_proc (const char *, gfc_namespace *);
 int gfc_find_symbol (const char *, gfc_namespace *, int, gfc_symbol **);
@@ -3429,8 +3425,7 @@ void gfc_undo_symbols (void);
 void gfc_commit_symbols (void);
 void gfc_commit_symbol (gfc_symbol *);
 gfc_charlen *gfc_new_charlen (gfc_namespace *, gfc_charlen *);
-void gfc_free_charlen (gfc_charlen *, gfc_charlen *);
-void gfc_free_namespace (gfc_namespace *);
+void gfc_free_namespace (gfc_namespace *&);
 
 void gfc_symbol_init_2 (void);
 void gfc_symbol_done_2 (void);
@@ -3449,7 +3444,6 @@ void gfc_traverse_gsymbol (gfc_gsymbol *, void (*)(gfc_gsymbol *, void *), void
 
 gfc_typebound_proc* gfc_get_typebound_proc (gfc_typebound_proc*);
 gfc_symbol* gfc_get_derived_super_type (gfc_symbol*);
-gfc_symbol* gfc_get_ultimate_derived_super_type (gfc_symbol*);
 bool gfc_type_is_extension_of (gfc_symbol *, gfc_symbol *);
 bool gfc_type_compatible (gfc_typespec *, gfc_typespec *);
 
@@ -3543,7 +3537,6 @@ void gfc_omp_restore_state (struct gfc_omp_saved_state *);
 void gfc_free_expr_list (gfc_expr_list *);
 void gfc_resolve_oacc_directive (gfc_code *, gfc_namespace *);
 void gfc_resolve_oacc_declare (gfc_namespace *);
-void gfc_resolve_oacc_parallel_loop_blocks (gfc_code *, gfc_namespace *);
 void gfc_resolve_oacc_blocks (gfc_code *, gfc_namespace *);
 void gfc_resolve_oacc_routines (gfc_namespace *);
 
@@ -3600,7 +3593,6 @@ bool gfc_check_pointer_assign (gfc_expr *lvalue, gfc_expr *rvalue,
 bool gfc_check_assign_symbol (gfc_symbol *, gfc_component *, gfc_expr *);
 
 gfc_expr *gfc_build_default_init_expr (gfc_typespec *, locus *);
-gfc_expr *gfc_build_init_expr (gfc_typespec *, locus *, bool);
 void gfc_apply_init (gfc_typespec *, symbol_attribute *, gfc_expr *);
 bool gfc_has_default_initializer (gfc_symbol *);
 gfc_expr *gfc_default_initializer (gfc_typespec *);
@@ -3666,7 +3658,6 @@ bool gfc_resolve_dim_arg (gfc_expr *);
 bool gfc_is_formal_arg (void);
 bool gfc_resolve_substring (gfc_ref *, bool *);
 void gfc_resolve_substring_charlen (gfc_expr *);
-match gfc_iso_c_sub_interface(gfc_code *, gfc_symbol *);
 gfc_expr *gfc_expr_to_initialize (gfc_expr *);
 bool gfc_type_is_extensible (gfc_symbol *);
 bool gfc_resolve_intrinsic (gfc_symbol *, locus *);
diff --git a/gcc/fortran/gfortran.texi b/gcc/fortran/gfortran.texi
index a54153b..3264709 100644
--- a/gcc/fortran/gfortran.texi
+++ b/gcc/fortran/gfortran.texi
@@ -180,7 +180,6 @@ Part I: Invoking GNU Fortran
 * Runtime::              Influencing runtime behavior with environment variables.
 
 Part II: Language Reference
-* Fortran standards status::      Fortran 2003, 2008 and 2018 features supported by GNU Fortran.
 * Compiler Characteristics::      User-visible implementation details.
 * Extensions::                    Language extensions implemented by GNU Fortran.
 * Mixed-Language Programming::    Interoperability with C
@@ -220,17 +219,9 @@ compiler.
 @end ifset
 @end iftex
 
-The GNU Fortran compiler front end was
-designed initially as a free replacement for,
-or alternative to, the Unix @command{f95} command;
-@command{gfortran} is the command you will use to invoke the compiler.
-
 @menu
 * About GNU Fortran::    What you should know about the GNU Fortran compiler.
 * GNU Fortran and GCC::  You can compile Fortran, C, or other programs.
-* Preprocessing and conditional compilation:: The Fortran preprocessor
-* GNU Fortran and G77::  Why we chose to start from scratch.
-* Project Status::       Status of GNU Fortran, roadmap, proposed extensions.
 * Standards::            Standards supported by GNU Fortran.
 @end menu
 
@@ -242,46 +233,67 @@ or alternative to, the Unix @command{f95} command;
 @node About GNU Fortran
 @section About GNU Fortran
 
-The GNU Fortran compiler supports the Fortran 77, 90 and 95 standards
-completely, parts of the Fortran 2003, 2008 and 2018 standards, and
-several vendor extensions.  The development goal is to provide the
-following features:
+The GNU Fortran compiler is the successor to @command{g77}, the
+Fortran 77 front end included in GCC prior to version 4 (released in
+2005).  While it is backward-compatible with most @command{g77}
+extensions and command-line options, @command{gfortran} is a completely new
+implemention designed to support more modern dialects of Fortran.
+GNU Fortran implements the Fortran 77, 90 and 95 standards
+completely, most of the Fortran 2003 and 2008 standards, and some
+features from the 2018 standard.  It also implements several extensions
+including OpenMP and OpenACC support for parallel programming.
+
+The GNU Fortran compiler passes the
+@uref{http://www.fortran-2000.com/ArnaudRecipes/fcvs21_f95.html,
+NIST Fortran 77 Test Suite}, and produces acceptable results on the
+@uref{http://www.netlib.org/lapack/faq.html#1.21, LAPACK Test Suite}.
+It also provides respectable performance on
+the @uref{https://polyhedron.com/?page_id=175,
+Polyhedron Fortran compiler benchmarks} and the
+@uref{http://www.netlib.org/benchmark/livermore,
+Livermore Fortran Kernels test}.  It has been used to compile a number of
+large real-world programs, including
+@uref{http://hirlam.org/, the HARMONIE and HIRLAM weather forecasting code} and
+@uref{https://github.com/dylan-jayatilaka/tonto,
+the Tonto quantum chemistry package}; see
+@url{https://gcc.gnu.org/@/wiki/@/GfortranApps} for an extended list.
+
+GNU Fortran provides the following functionality:
 
 @itemize @bullet
 @item
-Read a user's program, stored in a file and containing instructions
-written in Fortran 77, Fortran 90, Fortran 95, Fortran 2003, Fortran
-2008 or Fortran 2018.  This file contains @dfn{source code}.
+Read a program, stored in a file and containing @dfn{source code}
+instructions written in Fortran 77.
 
 @item
-Translate the user's program into instructions a computer
+Translate the program into instructions a computer
 can carry out more quickly than it takes to translate the
-instructions in the first
-place.  The result after compilation of a program is
+original Fortran instructions.
+The result after compilation of a program is
 @dfn{machine code},
-code designed to be efficiently translated and processed
+which is efficiently translated and processed
 by a machine such as your computer.
 Humans usually are not as good writing machine code
 as they are at writing Fortran (or C++, Ada, or Java),
 because it is easy to make tiny mistakes writing machine code.
 
 @item
-Provide the user with information about the reasons why
-the compiler is unable to create a binary from the source code.
-Usually this will be the case if the source code is flawed.
-The Fortran 90 standard requires that the compiler can point out
-mistakes to the user.
+Provide information about the reasons why
+the compiler may be unable to create a binary from the source code,
+for example if the source code is flawed.
+The Fortran language standards require that the compiler can point out
+mistakes in your code.
 An incorrect usage of the language causes an @dfn{error message}.
 
-The compiler will also attempt to diagnose cases where the
-user's program contains a correct usage of the language,
+The compiler also attempts to diagnose cases where your
+program contains a correct usage of the language,
 but instructs the computer to do something questionable.
-This kind of diagnostics message is called a @dfn{warning message}.
+This kind of diagnostic message is called a @dfn{warning message}.
 
 @item
 Provide optional information about the translation passes
 from the source code to machine code.
-This can help a user of the compiler to find the cause of
+This can help you to find the cause of
 certain bugs which may not be obvious in the source code,
 but may be more easily found at a lower level compiler output.
 It also helps developers to find bugs in the compiler itself.
@@ -293,7 +305,7 @@ called a @dfn{debugger}, such as the GNU Debugger @command{gdb}).
 
 @item
 Locate and gather machine code already generated to
-perform actions requested by statements in the user's program.
+perform actions requested by statements in the program.
 This machine code is organized into @dfn{modules} and is located
 and @dfn{linked} to the user program.
 @end itemize
@@ -317,8 +329,9 @@ which also might be installed as the
 system's @command{f95} command.
 @command{gfortran} is just another driver program,
 but specifically for the Fortran compiler only.
-The difference with @command{gcc} is that @command{gfortran}
-will automatically link the correct libraries to your program.
+The primary difference between the @command{gcc} and @command{gfortran}
+commands is that the latter automatically links the correct libraries
+to your program.
 
 @item
 A collection of run-time libraries.
@@ -339,7 +352,7 @@ linked to and interfaced with the GCC backend library.
 assembler code.  You would typically not use this
 program directly;
 instead, the @command{gcc} or @command{gfortran} driver
-programs will call it for you.
+programs call it for you.
 @end itemize
 
 
@@ -365,10 +378,10 @@ which provides the command-line interface for the compiler.  It calls
 the relevant compiler front-end program (e.g., @command{f951} for
 Fortran) for each file in the source code, and then calls the assembler
 and linker as appropriate to produce the compiled output.  In a copy of
-GCC which has been compiled with Fortran language support enabled,
-@command{gcc} will recognize files with @file{.f}, @file{.for}, @file{.ftn},
+GCC that has been compiled with Fortran language support enabled,
+@command{gcc} recognizes files with @file{.f}, @file{.for}, @file{.ftn},
 @file{.f90}, @file{.f95}, @file{.f03} and @file{.f08} extensions as
-Fortran source code, and compile it accordingly.  A @command{gfortran}
+Fortran source code, and compiles it accordingly.  A @command{gfortran}
 driver program is also provided, which is identical to @command{gcc}
 except that it automatically links the Fortran runtime libraries into the
 compiled program.
@@ -383,138 +396,12 @@ extension are also run through preprocessing.
 
 This manual specifically documents the Fortran front end, which handles
 the programming language's syntax and semantics.  The aspects of GCC
-which relate to the optimization passes and the back-end code generation
+that relate to the optimization passes and the back-end code generation
 are documented in the GCC manual; see
 @ref{Top,,Introduction,gcc,Using the GNU Compiler Collection (GCC)}.
 The two manuals together provide a complete reference for the GNU
 Fortran compiler.
 
-
-@c ---------------------------------------------------------------------
-@c Preprocessing and conditional compilation
-@c ---------------------------------------------------------------------
-
-@node Preprocessing and conditional compilation
-@section Preprocessing and conditional compilation
-@cindex CPP
-@cindex FPP
-@cindex Conditional compilation
-@cindex Preprocessing
-@cindex preprocessor, include file handling
-
-Many Fortran compilers including GNU Fortran allow passing the source code
-through a C preprocessor (CPP; sometimes also called the Fortran preprocessor,
-FPP) to allow for conditional compilation.  In the case of GNU Fortran,
-this is the GNU C Preprocessor in the traditional mode.  On systems with
-case-preserving file names, the preprocessor is automatically invoked if the
-filename extension is @file{.F}, @file{.FOR}, @file{.FTN}, @file{.fpp},
-@file{.FPP}, @file{.F90}, @file{.F95}, @file{.F03} or @file{.F08}.  To manually
-invoke the preprocessor on any file, use @option{-cpp}, to disable
-preprocessing on files where the preprocessor is run automatically, use
-@option{-nocpp}.
-
-If a preprocessed file includes another file with the Fortran @code{INCLUDE}
-statement, the included file is not preprocessed.  To preprocess included
-files, use the equivalent preprocessor statement @code{#include}.
-
-If GNU Fortran invokes the preprocessor, @code{__GFORTRAN__}
-is defined.  The macros @code{__GNUC__}, @code{__GNUC_MINOR__} and
-@code{__GNUC_PATCHLEVEL__} can be used to determine the version of the
-compiler.  See @ref{Top,,Overview,cpp,The C Preprocessor} for details.
-
-GNU Fortran supports a number of @code{INTEGER} and @code{REAL} kind types
-in additional to the kind types required by the Fortran standard.
-The availability of any given kind type is architecture dependent.  The
-following pre-defined preprocessor macros can be used to conditionally
-include code for these additional kind types: @code{__GFC_INT_1__},
-@code{__GFC_INT_2__}, @code{__GFC_INT_8__}, @code{__GFC_INT_16__},
-@code{__GFC_REAL_10__}, and @code{__GFC_REAL_16__}.
-
-While CPP is the de-facto standard for preprocessing Fortran code,
-Part 3 of the Fortran 95 standard (ISO/IEC 1539-3:1998) defines
-Conditional Compilation, which is not widely used and not directly
-supported by the GNU Fortran compiler.  You can use the program coco
-to preprocess such files (@uref{http://www.daniellnagle.com/coco.html}).
-
-
-@c ---------------------------------------------------------------------
-@c GNU Fortran and G77
-@c ---------------------------------------------------------------------
-
-@node GNU Fortran and G77
-@section GNU Fortran and G77
-@cindex Fortran 77
-@cindex @command{g77}
-
-The GNU Fortran compiler is the successor to @command{g77}, the Fortran
-77 front end included in GCC prior to version 4.  It is an entirely new
-program that has been designed to provide Fortran 95 support and
-extensibility for future Fortran language standards, as well as providing
-backwards compatibility for Fortran 77 and nearly all of the GNU language
-extensions supported by @command{g77}.
-
-
-@c ---------------------------------------------------------------------
-@c Project Status
-@c ---------------------------------------------------------------------
-
-@node Project Status
-@section Project Status
-
-@quotation
-As soon as @command{gfortran} can parse all of the statements correctly,
-it will be in the ``larva'' state.
-When we generate code, the ``puppa'' state.
-When @command{gfortran} is done,
-we'll see if it will be a beautiful butterfly,
-or just a big bug....
-
---Andy Vaught, April 2000
-@end quotation
-
-The start of the GNU Fortran 95 project was announced on
-the GCC homepage in March 18, 2000
-(even though Andy had already been working on it for a while,
-of course).
-
-The GNU Fortran compiler is able to compile nearly all
-standard-compliant Fortran 95, Fortran 90, and Fortran 77 programs,
-including a number of standard and non-standard extensions, and can be
-used on real-world programs.  In particular, the supported extensions
-include OpenMP, Cray-style pointers, some old vendor extensions, and several
-Fortran 2003 and Fortran 2008 features, including TR 15581.  However, it is
-still under development and has a few remaining rough edges.
-There also is initial support for OpenACC.
-
-At present, the GNU Fortran compiler passes the
-@uref{http://www.fortran-2000.com/ArnaudRecipes/fcvs21_f95.html,
-NIST Fortran 77 Test Suite}, and produces acceptable results on the
-@uref{http://www.netlib.org/lapack/faq.html#1.21, LAPACK Test Suite}.
-It also provides respectable performance on
-the @uref{http://www.polyhedron.com/fortran-compiler-comparisons/polyhedron-benchmark-suite,
-Polyhedron Fortran
-compiler benchmarks} and the
-@uref{http://www.netlib.org/benchmark/livermore,
-Livermore Fortran Kernels test}.  It has been used to compile a number of
-large real-world programs, including
-@uref{http://hirlam.org/, the HARMONIE and HIRLAM weather forecasting code} and
-@uref{http://physical-chemistry.scb.uwa.edu.au/tonto/wiki/index.php/Main_Page,
-the Tonto quantum chemistry package}; see
-@url{https://gcc.gnu.org/@/wiki/@/GfortranApps} for an extended list.
-
-Among other things, the GNU Fortran compiler is intended as a replacement
-for G77.  At this point, nearly all programs that could be compiled with
-G77 can be compiled with GNU Fortran, although there are a few minor known
-regressions.
-
-The primary work remaining to be done on GNU Fortran falls into three
-categories: bug fixing (primarily regarding the treatment of invalid
-code and providing useful error messages), improving the compiler
-optimizations and the performance of compiled code, and extending the
-compiler to support future standards---in particular, Fortran 2003,
-Fortran 2008 and Fortran 2018.
-
-
 @c ---------------------------------------------------------------------
 @c Standards
 @c ---------------------------------------------------------------------
@@ -524,21 +411,33 @@ Fortran 2008 and Fortran 2018.
 @cindex Standards
 
 @menu
-* Varying Length Character Strings::
+* Fortran 95 status::
+* Fortran 2003 status::
+* Fortran 2008 status::
+* Fortran 2018 status::
 @end menu
 
-The GNU Fortran compiler implements
-ISO/IEC 1539:1997 (Fortran 95).  As such, it can also compile essentially all
-standard-compliant Fortran 90 and Fortran 77 programs.   It also supports
-the ISO/IEC TR-15581 enhancements to allocatable arrays.
-
-GNU Fortran also have a partial support for ISO/IEC 1539-1:2004
-(Fortran 2003), ISO/IEC 1539-1:2010 (Fortran 2008), the Technical
-Specification @code{Further Interoperability of Fortran with C}
-(ISO/IEC TS 29113:2012).  Full support of those standards and future
-Fortran standards is planned.  The current status of the support is
-can be found in the @ref{Fortran 2003 status}, @ref{Fortran 2008
-status} and @ref{Fortran 2018 status} sections of the documentation.
+Fortran is developed by the Working Group 5 of Sub-Committee 22 of the
+Joint Technical Committee 1 of the International Organization for
+Standardization and the International Electrotechnical Commission (IEC).
+This group is known as @uref{http://www.nag.co.uk/sc22wg5/, WG5}.
+Official Fortran standard documents are available for purchase
+from ISO; a collection of free documents (typically final drafts) are
+also available on the @uref{https://gcc.gnu.org/wiki/GFortranStandards, wiki}.
+
+The GNU Fortran compiler implements ISO/IEC 1539:1997 (Fortran 95).
+As such, it can also compile essentially all standard-compliant
+Fortran 90 and Fortran 77 programs.  It also supports the ISO/IEC
+TR-15581 enhancements to allocatable arrays.
+
+GNU Fortran also supports almost all of ISO/IEC 1539-1:2004
+(Fortran 2003) and ISO/IEC 1539-1:2010 (Fortran 2008).
+It has partial support for features introduced in ISO/IEC
+1539:2018 (Fortran 2018), the most recent version of the Fortran
+language standard, including full support for the Technical Specification
+@code{Further Interoperability of Fortran with C} (ISO/IEC TS 29113:2012).
+More details on support for these standards can be
+found in the following sections of the documentation.
 
 Additionally, the GNU Fortran compilers supports the OpenMP specification
 (version 4.5 and partial support of the features of the 5.0 version,
@@ -547,11 +446,11 @@ There also is support for the OpenACC specification (targeting
 version 2.6, @uref{http://www.openacc.org/}).  See
 @uref{https://gcc.gnu.org/wiki/OpenACC} for more information.
 
-@node Varying Length Character Strings
-@subsection Varying Length Character Strings
-@cindex Varying length character strings
+@node Fortran 95 status
+@subsection Fortran 95 status
 @cindex Varying length strings
 @cindex strings, varying length
+@cindex conditional compilation
 
 The Fortran 95 standard specifies in Part 2 (ISO/IEC 1539-2:2000)
 varying length character strings.  While GNU Fortran currently does not
@@ -565,6 +464,92 @@ the features of @code{ISO_VARYING_STRING} and should be considered as
 replacement. (Namely, allocatable or pointers of the type
 @code{character(len=:)}.)
 
+Part 3 of the Fortran 95 standard (ISO/IEC 1539-3:1998) defines
+Conditional Compilation, which is not widely used and not directly
+supported by the GNU Fortran compiler.  You can use the program coco
+to preprocess such files (@uref{http://www.daniellnagle.com/coco.html}).
+
+@node Fortran 2003 status
+@subsection Fortran 2003 status
+
+GNU Fortran implements the Fortran 2003 (ISO/IEC 1539-1:2004) standard
+except for finalization support, which is incomplete.
+See the
+@uref{https://gcc.gnu.org/wiki/Fortran2003, wiki page} for a full list
+of new features introduced by Fortran 2003 and their implementation status.
+
+@node Fortran 2008 status
+@subsection Fortran 2008 status
+
+The GNU Fortran compiler supports almost all features of Fortran 2008;
+the @uref{https://gcc.gnu.org/wiki/Fortran2008Status, wiki}
+has some information about the current implementation status.
+In particular, the following are not yet supported:
+
+@itemize @bullet
+@item
+@code{DO CONCURRENT} and @code{FORALL} do not recognize a
+type-spec in the loop header.
+
+@item
+The change to permit any constant expression in subscripts and
+nested implied-do limits in a @code{DATA} statement has not been implemented.
+@end itemize
+
+
+@node Fortran 2018 status
+@subsection Fortran 2018 status
+
+Fortran 2018 (ISO/IEC 1539:2018) is the most recent version
+of the Fortran language standard.  GNU Fortran implements some of the
+new features of this standard:
+
+@itemize @bullet
+@item
+All Fortran 2018 features derived from ISO/IEC TS 29113:2012,
+``Further Interoperability of Fortran with C'', are supported by GNU Fortran.
+This includes assumed-type and assumed-rank objects and
+the @code{SELECT RANK} construct as well as the parts relating to
+@code{BIND(C)} functions.
+See also @ref{Further Interoperability of Fortran with C}.
+
+@item
+GNU Fortran supports a subset of features derived from ISO/IEC TS 18508:2015,
+``Additional Parallel Features in Fortran'':
+
+@itemize @bullet
+@item
+The new atomic ADD, CAS, FETCH and ADD/OR/XOR, OR and XOR intrinsics.
+
+@item
+The @code{CO_MIN} and @code{CO_MAX} and @code{SUM} reduction intrinsics,
+and the @code{CO_BROADCAST} and @code{CO_REDUCE} intrinsic, except that those
+do not support polymorphic types or types with allocatable, pointer or
+polymorphic components.
+
+@item
+Events (@code{EVENT POST}, @code{EVENT WAIT}, @code{EVENT_QUERY}).
+
+@item
+Failed images (@code{FAIL IMAGE}, @code{IMAGE_STATUS},
+@code{FAILED_IMAGES}, @code{STOPPED_IMAGES}).
+
+@end itemize
+
+@item
+An @code{ERROR STOP} statement is permitted in a @code{PURE}
+procedure.
+
+@item
+GNU Fortran supports the @code{IMPLICIT NONE} statement with an
+@code{implicit-none-spec-list}.
+
+@item
+The behavior of the @code{INQUIRE} statement with the @code{RECL=}
+specifier now conforms to Fortran 2018.
+
+@end itemize
+
 
 @c =====================================================================
 @c PART I: INVOCATION REFERENCE
@@ -801,386 +786,6 @@ The default value is 131072.
 \part{II}{Language Reference}
 @end tex
 
-@c ---------------------------------------------------------------------
-@c Fortran standards status
-@c ---------------------------------------------------------------------
-
-@node Fortran standards status
-@chapter Fortran standards status
-
-@menu
-* Fortran 2003 status::
-* Fortran 2008 status::
-* Fortran 2018 status::
-@end menu
-
-@node Fortran 2003 status
-@section Fortran 2003 status
-
-GNU Fortran supports several Fortran 2003 features; an incomplete
-list can be found below.  See also the
-@uref{https://gcc.gnu.org/wiki/Fortran2003, wiki page} about Fortran 2003.
-
-@itemize
-@item Procedure pointers including procedure-pointer components with
-@code{PASS} attribute.
-
-@item Procedures which are bound to a derived type (type-bound procedures)
-including @code{PASS}, @code{PROCEDURE} and @code{GENERIC}, and
-operators bound to a type.
-
-@item Abstract interfaces and type extension with the possibility to
-override type-bound procedures or to have deferred binding.
-
-@item Polymorphic entities (``@code{CLASS}'') for derived types and unlimited
-polymorphism (``@code{CLASS(*)}'') -- including @code{SAME_TYPE_AS},
-@code{EXTENDS_TYPE_OF} and @code{SELECT TYPE} for scalars and arrays and
-finalization.
-
-@item Generic interface names, which have the same name as derived types,
-are now supported. This allows one to write constructor functions.  Note
-that Fortran does not support static constructor functions.  For static
-variables, only default initialization or structure-constructor
-initialization are available.
-
-@item The @code{ASSOCIATE} construct.
-
-@item Interoperability with C including enumerations,
-
-@item In structure constructors the components with default values may be
-omitted.
-
-@item Extensions to the @code{ALLOCATE} statement, allowing for a
-type-specification with type parameter and for allocation and initialization
-from a @code{SOURCE=} expression; @code{ALLOCATE} and @code{DEALLOCATE}
-optionally return an error message string via @code{ERRMSG=}.
-
-@item Reallocation on assignment: If an intrinsic assignment is
-used, an allocatable variable on the left-hand side is automatically allocated
-(if unallocated) or reallocated (if the shape is different). Currently, scalar
-deferred character length left-hand sides are correctly handled but arrays
-are not yet fully implemented.
-
-@item Deferred-length character variables and scalar deferred-length character
-components of derived types are supported. (Note that array-valued components
-are not yet implemented.)
-
-@item Transferring of allocations via @code{MOVE_ALLOC}.
-
-@item The @code{PRIVATE} and @code{PUBLIC} attributes may be given individually
-to derived-type components.
-
-@item In pointer assignments, the lower bound may be specified and
-the remapping of elements is supported.
-
-@item For pointers an @code{INTENT} may be specified which affect the
-association status not the value of the pointer target.
-
-@item Intrinsics @code{command_argument_count}, @code{get_command},
-@code{get_command_argument}, and @code{get_environment_variable}.
-
-@item Support for Unicode characters (ISO 10646) and UTF-8, including
-the @code{SELECTED_CHAR_KIND} and @code{NEW_LINE} intrinsic functions.
-
-@item Support for binary, octal and hexadecimal (BOZ) constants in the
-intrinsic functions @code{INT}, @code{REAL}, @code{CMPLX} and @code{DBLE}.
-
-@item Support for namelist variables with allocatable and pointer
-attribute and nonconstant length type parameter.
-
-@item
-@cindex array, constructors
-@cindex @code{[...]}
-Array constructors using square brackets.  That is, @code{[...]} rather
-than @code{(/.../)}.  Type-specification for array constructors like
-@code{(/ some-type :: ... /)}.
-
-@item Extensions to the specification and initialization expressions,
-including the support for intrinsics with real and complex arguments.
-
-@item Support for the asynchronous input/output.
-
-@item
-@cindex @code{FLUSH} statement
-@cindex statement, @code{FLUSH}
-@code{FLUSH} statement.
-
-@item
-@cindex @code{IOMSG=} specifier
-@code{IOMSG=} specifier for I/O statements.
-
-@item
-@cindex @code{ENUM} statement
-@cindex @code{ENUMERATOR} statement
-@cindex statement, @code{ENUM}
-@cindex statement, @code{ENUMERATOR}
-@opindex @code{fshort-enums}
-Support for the declaration of enumeration constants via the
-@code{ENUM} and @code{ENUMERATOR} statements.  Interoperability with
-@command{gcc} is guaranteed also for the case where the
-@command{-fshort-enums} command line option is given.
-
-@item
-@cindex TR 15581
-TR 15581:
-@itemize
-@item
-@cindex @code{ALLOCATABLE} dummy arguments
-@code{ALLOCATABLE} dummy arguments.
-@item
-@cindex @code{ALLOCATABLE} function results
-@code{ALLOCATABLE} function results
-@item
-@cindex @code{ALLOCATABLE} components of derived types
-@code{ALLOCATABLE} components of derived types
-@end itemize
-
-@item
-@cindex @code{STREAM} I/O
-@cindex @code{ACCESS='STREAM'} I/O
-The @code{OPEN} statement supports the @code{ACCESS='STREAM'} specifier,
-allowing I/O without any record structure.
-
-@item
-Namelist input/output for internal files.
-
-@item Minor I/O features: Rounding during formatted output, using of
-a decimal comma instead of a decimal point, setting whether a plus sign
-should appear for positive numbers. On systems where @code{strtod} honours
-the rounding mode, the rounding mode is also supported for input.
-
-@item
-@cindex @code{PROTECTED} statement
-@cindex statement, @code{PROTECTED}
-The @code{PROTECTED} statement and attribute.
-
-@item
-@cindex @code{VALUE} statement
-@cindex statement, @code{VALUE}
-The @code{VALUE} statement and attribute.
-
-@item
-@cindex @code{VOLATILE} statement
-@cindex statement, @code{VOLATILE}
-The @code{VOLATILE} statement and attribute.
-
-@item
-@cindex @code{IMPORT} statement
-@cindex statement, @code{IMPORT}
-The @code{IMPORT} statement, allowing to import
-host-associated derived types.
-
-@item The intrinsic modules @code{ISO_FORTRAN_ENVIRONMENT} is supported,
-which contains parameters of the I/O units, storage sizes. Additionally,
-procedures for C interoperability are available in the @code{ISO_C_BINDING}
-module.
-
-@item
-@cindex @code{USE, INTRINSIC} statement
-@cindex statement, @code{USE, INTRINSIC}
-@cindex @code{ISO_FORTRAN_ENV} statement
-@cindex statement, @code{ISO_FORTRAN_ENV}
-@code{USE} statement with @code{INTRINSIC} and @code{NON_INTRINSIC}
-attribute; supported intrinsic modules: @code{ISO_FORTRAN_ENV},
-@code{ISO_C_BINDING}, @code{OMP_LIB} and @code{OMP_LIB_KINDS},
-and @code{OPENACC}.
-
-@item
-Renaming of operators in the @code{USE} statement.
-
-@end itemize
-
-
-@node Fortran 2008 status
-@section Fortran 2008 status
-
-The latest version of the Fortran standard is ISO/IEC 1539-1:2010, informally
-known as Fortran 2008.  The official version is available from International
-Organization for Standardization (ISO) or its national member organizations.
-The the final draft (FDIS) can be downloaded free of charge from
-@url{http://www.nag.co.uk/@/sc22wg5/@/links.html}.  Fortran is developed by the
-Working Group 5 of Sub-Committee 22 of the Joint Technical Committee 1 of the
-International Organization for Standardization and the International
-Electrotechnical Commission (IEC).  This group is known as
-@uref{http://www.nag.co.uk/sc22wg5/, WG5}.
-
-The GNU Fortran compiler supports several of the new features of Fortran 2008;
-the @uref{https://gcc.gnu.org/wiki/Fortran2008Status, wiki} has some information
-about the current Fortran 2008 implementation status.  In particular, the
-following is implemented.
-
-@itemize
-@item The @option{-std=f2008} option and support for the file extensions
-@file{.f08} and @file{.F08}.
-
-@item The @code{OPEN} statement now supports the @code{NEWUNIT=} option,
-which returns a unique file unit, thus preventing inadvertent use of the
-same unit in different parts of the program.
-
-@item The @code{g0} format descriptor and unlimited format items.
-
-@item The mathematical intrinsics @code{ASINH}, @code{ACOSH}, @code{ATANH},
-@code{ERF}, @code{ERFC}, @code{GAMMA}, @code{LOG_GAMMA}, @code{BESSEL_J0},
-@code{BESSEL_J1}, @code{BESSEL_JN}, @code{BESSEL_Y0}, @code{BESSEL_Y1},
-@code{BESSEL_YN}, @code{HYPOT}, @code{NORM2}, and @code{ERFC_SCALED}.
-
-@item Using complex arguments with @code{TAN}, @code{SINH}, @code{COSH},
-@code{TANH}, @code{ASIN}, @code{ACOS}, and @code{ATAN} is now possible;
-@code{ATAN}(@var{Y},@var{X}) is now an alias for @code{ATAN2}(@var{Y},@var{X}).
-
-@item Support of the @code{PARITY} intrinsic functions.
-
-@item The following bit intrinsics: @code{LEADZ} and @code{TRAILZ} for
-counting the number of leading and trailing zero bits, @code{POPCNT} and
-@code{POPPAR} for counting the number of one bits and returning the parity;
-@code{BGE}, @code{BGT}, @code{BLE}, and @code{BLT} for bitwise comparisons;
-@code{DSHIFTL} and @code{DSHIFTR} for combined left and right shifts,
-@code{MASKL} and @code{MASKR} for simple left and right justified masks,
-@code{MERGE_BITS} for a bitwise merge using a mask, @code{SHIFTA},
-@code{SHIFTL} and @code{SHIFTR} for shift operations, and the
-transformational bit intrinsics @code{IALL}, @code{IANY} and @code{IPARITY}.
-
-@item Support of the @code{EXECUTE_COMMAND_LINE} intrinsic subroutine.
-
-@item Support for the @code{STORAGE_SIZE} intrinsic inquiry function.
-
-@item The @code{INT@{8,16,32@}} and @code{REAL@{32,64,128@}} kind type
-parameters and the array-valued named constants @code{INTEGER_KINDS},
-@code{LOGICAL_KINDS}, @code{REAL_KINDS} and @code{CHARACTER_KINDS} of
-the intrinsic module @code{ISO_FORTRAN_ENV}.
-
-@item The module procedures @code{C_SIZEOF} of the intrinsic module
-@code{ISO_C_BINDINGS} and @code{COMPILER_VERSION} and @code{COMPILER_OPTIONS}
-of @code{ISO_FORTRAN_ENV}.
-
-@item Coarray support for serial programs with @option{-fcoarray=single} flag
-and experimental support for multiple images with the @option{-fcoarray=lib}
-flag.
-
-@item Submodules are supported. It should noted that @code{MODULEs} do not
-produce the smod file needed by the descendent @code{SUBMODULEs} unless they
-contain at least one @code{MODULE PROCEDURE} interface. The reason for this is
-that @code{SUBMODULEs} are useless without @code{MODULE PROCEDUREs}. See
-http://j3-fortran.org/doc/meeting/207/15-209.txt for a discussion and a draft
-interpretation. Adopting this interpretation has the advantage that code that
-does not use submodules does not generate smod files.
-
-@item The @code{DO CONCURRENT} construct is supported.
-
-@item The @code{BLOCK} construct is supported.
-
-@item The @code{STOP} and the new @code{ERROR STOP} statements now
-support all constant expressions. Both show the signals which were signaling
-at termination.
-
-@item Support for the @code{CONTIGUOUS} attribute.
-
-@item Support for @code{ALLOCATE} with @code{MOLD}.
-
-@item Support for the @code{IMPURE} attribute for procedures, which
-allows for @code{ELEMENTAL} procedures without the restrictions of
-@code{PURE}.
-
-@item Null pointers (including @code{NULL()}) and not-allocated variables
-can be used as actual argument to optional non-pointer, non-allocatable
-dummy arguments, denoting an absent argument.
-
-@item Non-pointer variables with @code{TARGET} attribute can be used as
-actual argument to @code{POINTER} dummies with @code{INTENT(IN)}.
-
-@item Pointers including procedure pointers and those in a derived
-type (pointer components) can now be initialized by a target instead
-of only by @code{NULL}.
-
-@item The @code{EXIT} statement (with construct-name) can be now be
-used to leave not only the @code{DO} but also the @code{ASSOCIATE},
-@code{BLOCK}, @code{IF}, @code{SELECT CASE} and @code{SELECT TYPE}
-constructs.
-
-@item Internal procedures can now be used as actual argument.
-
-@item Minor features: obsolesce diagnostics for @code{ENTRY} with
-@option{-std=f2008}; a line may start with a semicolon; for internal
-and module procedures @code{END} can be used instead of
-@code{END SUBROUTINE} and @code{END FUNCTION}; @code{SELECTED_REAL_KIND}
-now also takes a @code{RADIX} argument; intrinsic types are supported
-for @code{TYPE}(@var{intrinsic-type-spec}); multiple type-bound procedures
-can be declared in a single @code{PROCEDURE} statement; implied-shape
-arrays are supported for named constants (@code{PARAMETER}).
-@end itemize
-
-
-
-@node Fortran 2018 status
-@section Status of Fortran 2018 support
-
-@itemize
-@item ERROR STOP in a PURE procedure
-An @code{ERROR STOP} statement is permitted in a @code{PURE}
-procedure.
-
-@item IMPLICIT NONE with a spec-list
-Support the @code{IMPLICIT NONE} statement with an
-@code{implicit-none-spec-list}.
-
-@item Behavior of INQUIRE with the RECL= specifier
-
-The behavior of the @code{INQUIRE} statement with the @code{RECL=}
-specifier now conforms to Fortran 2018.
-
-@end itemize
-
-
-@subsection TS 29113 Status (Further Interoperability with C)
-
-GNU Fortran supports some of the new features of the Technical
-Specification (TS) 29113 on Further Interoperability of Fortran with C.
-The @uref{https://gcc.gnu.org/wiki/TS29113Status, wiki} has some information
-about the current TS 29113 implementation status.  In particular, the
-following is implemented.
-
-See also @ref{Further Interoperability of Fortran with C}.
-
-@itemize
-@item The @code{OPTIONAL} attribute is allowed for dummy arguments
-of @code{BIND(C) procedures.}
-
-@item The @code{RANK} intrinsic is supported.
-
-@item GNU Fortran's implementation for variables with @code{ASYNCHRONOUS}
-attribute is compatible with TS 29113.
-
-@item Assumed types (@code{TYPE(*)}).
-
-@item Assumed-rank (@code{DIMENSION(..)}).
-
-@item ISO_Fortran_binding (now in Fortran 2018 18.4) is implemented such that
-conversion of the array descriptor for assumed type or assumed rank arrays is
-done in the library. The include file ISO_Fortran_binding.h is can be found in
-@code{~prefix/lib/gcc/$target/$version}.
-@end itemize
-
-
-
-@subsection TS 18508 Status (Additional Parallel Features)
-
-GNU Fortran supports the following new features of the Technical
-Specification 18508 on Additional Parallel Features in Fortran:
-
-@itemize
-@item The new atomic ADD, CAS, FETCH and ADD/OR/XOR, OR and XOR intrinsics.
-
-@item The @code{CO_MIN} and @code{CO_MAX} and @code{SUM} reduction intrinsics.
-And the @code{CO_BROADCAST} and @code{CO_REDUCE} intrinsic, except that those
-do not support polymorphic types or types with allocatable, pointer or
-polymorphic components.
-
-@item Events (@code{EVENT POST}, @code{EVENT WAIT}, @code{EVENT_QUERY})
-
-@item Failed images (@code{FAIL IMAGE}, @code{IMAGE_STATUS},
-@code{FAILED_IMAGES}, @code{STOPPED_IMAGES})
-
-@end itemize
 
 
 @c ---------------------------------------------------------------------
@@ -3113,28 +2718,42 @@ reading from the position marked previously.
 * Naming and argument-passing conventions::
 @end menu
 
-This chapter is about mixed-language interoperability, but also applies
-if one links Fortran code compiled by different compilers.  In most cases,
-use of the C Binding features of the Fortran 2003 standard is sufficient,
-and their use is highly recommended.
+This chapter is about mixed-language interoperability, but also
+applies if you link Fortran code compiled by different compilers.  In
+most cases, use of the C Binding features of the Fortran 2003 and
+later standards is sufficient.
+
+For example, it is possible to mix Fortran code with C++ code as well
+as C, if you declare the interface functions as @code{extern "C"} on
+the C++ side and @code{BIND(C)} on the Fortran side, and follow the
+rules for interoperability with C.  Note that you cannot manipulate
+C++ class objects in Fortran or vice versa except as opaque pointers.
 
+You can use the @command{gfortran} command to link both Fortran and
+non-Fortran code into the same program, or you can use @command{gcc}
+or @command{g++} if you also add an explicit @option{-lgfortran} option
+to link with the Fortran library.  If your main program is written in
+C or some other language instead of Fortran, see
+@ref{Non-Fortran Main Program}, below.
 
 @node Interoperability with C
 @section Interoperability with C
+@cindex interoperability with C
+@cindex C interoperability
 
 @menu
 * Intrinsic Types::
 * Derived Types and struct::
 * Interoperable Global Variables::
 * Interoperable Subroutines and Functions::
-* Working with Pointers::
+* Working with C Pointers::
 * Further Interoperability of Fortran with C::
 @end menu
 
 Since Fortran 2003 (ISO/IEC 1539-1:2004(E)) there is a
 standardized way to generate procedure and derived-type
-declarations and global variables which are interoperable with C
-(ISO/IEC 9899:1999).  The @code{bind(C)} attribute has been added
+declarations and global variables that are interoperable with C
+(ISO/IEC 9899:1999).  The @code{BIND(C)} attribute has been added
 to inform the compiler that a symbol shall be interoperable with C;
 also, some constraints are added.  Note, however, that not
 all C features have a Fortran equivalent or vice versa.  For instance,
@@ -3150,12 +2769,16 @@ assuming @math{i < n}) in memory is @code{A(i+1,j)} (C: @code{A[j-1][i]}).
 
 @node Intrinsic Types
 @subsection Intrinsic Types
+@cindex C intrinsic type interoperability
+@cindex intrinsic type interoperability with C
+@cindex interoperability, intrinsic type
 
 In order to ensure that exactly the same variable type and kind is used
-in C and Fortran, the named constants shall be used which are defined in the
-@code{ISO_C_BINDING} intrinsic module.  That module contains named constants
-for kind parameters and character named constants for the escape sequences
-in C.  For a list of the constants, see @ref{ISO_C_BINDING}.
+in C and Fortran, you should use the named constants for kind parameters
+that are defined in the @code{ISO_C_BINDING} intrinsic module.
+That module contains named constants of character type representing
+the escaped special characters in C, such as newline.
+For a list of the constants, see @ref{ISO_C_BINDING}.
 
 For logical types, please note that the Fortran standard only guarantees
 interoperability between C99's @code{_Bool} and Fortran's @code{C_Bool}-kind
@@ -3165,12 +2788,13 @@ the value 0.  Using any other integer value with GNU Fortran's @code{LOGICAL}
 values than 0 and 1 to GCC's @code{_Bool} is also undefined, unless the
 integer is explicitly or implicitly casted to @code{_Bool}.)
 
-
-
 @node Derived Types and struct
 @subsection Derived Types and struct
+@cindex C derived type and struct interoperability
+@cindex derived type interoperability with C
+@cindex interoperability, derived type and struct
 
-For compatibility of derived types with @code{struct}, one needs to use
+For compatibility of derived types with @code{struct}, use
 the @code{BIND(C)} attribute in the type declaration.  For instance, the
 following type declaration
 
@@ -3185,6 +2809,7 @@ following type declaration
  END TYPE
 @end smallexample
 
+@noindent
 matches the following @code{struct} declaration in C
 
 @smallexample
@@ -3209,6 +2834,9 @@ with bit field or variable-length array members are interoperable.
 
 @node Interoperable Global Variables
 @subsection Interoperable Global Variables
+@cindex C variable interoperability
+@cindex variable interoperability with C
+@cindex interoperability, variable
 
 Variables can be made accessible from C using the C binding attribute,
 optionally together with specifying a binding name.  Those variables
@@ -3236,17 +2864,18 @@ a macro.  Use the @code{IERRNO} intrinsic (GNU extension) instead.
 
 @node Interoperable Subroutines and Functions
 @subsection Interoperable Subroutines and Functions
+@cindex C procedure interoperability
+@cindex procedure interoperability with C
+@cindex function interoperability with C
+@cindex subroutine interoperability with C
+@cindex interoperability, subroutine and function
 
 Subroutines and functions have to have the @code{BIND(C)} attribute to
 be compatible with C.  The dummy argument declaration is relatively
 straightforward.  However, one needs to be careful because C uses
 call-by-value by default while Fortran behaves usually similar to
 call-by-reference.  Furthermore, strings and pointers are handled
-differently.  Note that in Fortran 2003 and 2008 only explicit size
-and assumed-size arrays are supported but not assumed-shape or
-deferred-shape (i.e. allocatable or pointer) arrays.  However, those
-are allowed since the Technical Specification 29113, see
-@ref{Further Interoperability of Fortran with C}
+differently.
 
 To pass a variable by value, use the @code{VALUE} attribute.
 Thus, the following C prototype
@@ -3255,6 +2884,7 @@ Thus, the following C prototype
 @code{int func(int i, int *j)}
 @end smallexample
 
+@noindent
 matches the Fortran declaration
 
 @smallexample
@@ -3265,12 +2895,12 @@ matches the Fortran declaration
 @end smallexample
 
 Note that pointer arguments also frequently need the @code{VALUE} attribute,
-see @ref{Working with Pointers}.
+see @ref{Working with C Pointers}.
 
 Strings are handled quite differently in C and Fortran.  In C a string
 is a @code{NUL}-terminated array of characters while in Fortran each string
 has a length associated with it and is thus not terminated (by e.g.
-@code{NUL}).  For example, if one wants to use the following C function,
+@code{NUL}).  For example, if you want to use the following C function,
 
 @smallexample
   #include <stdio.h>
@@ -3280,7 +2910,8 @@ has a length associated with it and is thus not terminated (by e.g.
   @}
 @end smallexample
 
-to print ``Hello World'' from Fortran, one can call it using
+@noindent
+to print ``Hello World'' from Fortran, you can call it using
 
 @smallexample
   use iso_c_binding, only: C_CHAR, C_NULL_CHAR
@@ -3293,7 +2924,7 @@ to print ``Hello World'' from Fortran, one can call it using
   call print_c(C_CHAR_"Hello World"//C_NULL_CHAR)
 @end smallexample
 
-As the example shows, one needs to ensure that the
+As the example shows, you need to ensure that the
 string is @code{NUL} terminated.  Additionally, the dummy argument
 @var{string} of @code{print_C} is a length-one assumed-size
 array; using @code{character(len=*)} is not allowed.  The example
@@ -3309,6 +2940,7 @@ function @code{strncpy}, whose prototype is
   char *strncpy(char *restrict s1, const char *restrict s2, size_t n);
 @end smallexample
 
+@noindent
 The function @code{strncpy} copies at most @var{n} characters from
 string @var{s2} to @var{s1} and returns @var{s1}.  In the following
 example, we ignore the return value:
@@ -3336,18 +2968,21 @@ example, we ignore the return value:
 
 The intrinsic procedures are described in @ref{Intrinsic Procedures}.
 
-@node Working with Pointers
-@subsection Working with Pointers
+@node Working with C Pointers
+@subsection Working with C Pointers
+@cindex C pointers
+@cindex pointers, C
 
-C pointers are represented in Fortran via the special opaque derived type
-@code{type(c_ptr)} (with private components).  Thus one needs to
+C pointers are represented in Fortran via the special opaque derived
+type @code{type(c_ptr)} (with private components).  C pointers are distinct
+from Fortran objects with the @code{POINTER} attribute.  Thus one needs to
 use intrinsic conversion procedures to convert from or to C pointers.
+For some applications, using an assumed type (@code{TYPE(*)}) can be
+an alternative to a C pointer, and you can also use library routines
+to access Fortran pointers from C.  See @ref{Further Interoperability
+of Fortran with C}.
 
-For some applications, using an assumed type (@code{TYPE(*)}) can be an
-alternative to a C pointer; see
-@ref{Further Interoperability of Fortran with C}.
-
-For example,
+Here is an example of using C pointers in Fortran:
 
 @smallexample
   use iso_c_binding
@@ -3365,7 +3000,7 @@ For example,
 When converting C to Fortran arrays, the one-dimensional @code{SHAPE} argument
 has to be passed.
 
-If a pointer is a dummy-argument of an interoperable procedure, it usually
+If a pointer is a dummy argument of an interoperable procedure, it usually
 has to be declared using the @code{VALUE} attribute.  @code{void*}
 matches @code{TYPE(C_PTR), VALUE}, while @code{TYPE(C_PTR)} alone
 matches @code{void**}.
@@ -3491,81 +3126,31 @@ END MODULE m
 
 @node Further Interoperability of Fortran with C
 @subsection Further Interoperability of Fortran with C
-
-The Technical Specification ISO/IEC TS 29113:2012 on further
-interoperability of Fortran with C extends the interoperability support
-of Fortran 2003 and Fortran 2008. Besides removing some restrictions
-and constraints, it adds assumed-type (@code{TYPE(*)}) and assumed-rank
-(@code{dimension}) variables and allows for interoperability of
-assumed-shape, assumed-rank and deferred-shape arrays, including
-allocatables and pointers.
+@cindex Further Interoperability of Fortran with C
+@cindex TS 29113
+@cindex array descriptor
+@cindex dope vector
+@cindex assumed-type
+@cindex assumed-rank
+
+GNU Fortran implements the Technical Specification ISO/IEC TS
+29113:2012, which extends the interoperability support of Fortran 2003
+and Fortran 2008 and is now part of the 2018 Fortran standard.
+Besides removing some restrictions and constraints, the Technical
+Specification adds assumed-type (@code{TYPE(*)}) and assumed-rank
+(@code{DIMENSION(..)}) variables and allows for interoperability of
+assumed-shape, assumed-rank, and deferred-shape arrays, as well as
+allocatables and pointers.  Objects of these types are passed to
+@code{BIND(C)} functions as descriptors with a standard interface,
+declared in the header file @code{<ISO_Fortran_binding.h>}.
 
 Note: Currently, GNU Fortran does not use internally the array descriptor
 (dope vector) as specified in the Technical Specification, but uses
-an array descriptor with different fields. Assumed type and assumed rank
-formal arguments are converted in the library to the specified form. The
-ISO_Fortran_binding API functions (also Fortran 2018 18.4) are implemented
-in libgfortran. Alternatively, the Chasm Language Interoperability Tools,
-@url{http://chasm-interop.sourceforge.net/}, provide an interface to GNU
-Fortran's array descriptor.
-
-The Technical Specification adds the following new features, which
-are supported by GNU Fortran:
-
-@itemize @bullet
-
-@item The @code{ASYNCHRONOUS} attribute has been clarified and
-extended to allow its use with asynchronous communication in
-user-provided libraries such as in implementations of the
-Message Passing Interface specification.
-
-@item Many constraints have been relaxed, in particular for
-the @code{C_LOC} and @code{C_F_POINTER} intrinsics.
-
-@item The @code{OPTIONAL} attribute is now allowed for dummy
-arguments; an absent argument matches a @code{NULL} pointer.
-
-@item Assumed types (@code{TYPE(*)}) have been added, which may
-only be used for dummy arguments.  They are unlimited polymorphic
-but contrary to @code{CLASS(*)} they do not contain any type
-information, similar to C's @code{void *} pointers.  Expressions
-of any type and kind can be passed; thus, it can be used as
-replacement for @code{TYPE(C_PTR)}, avoiding the use of
-@code{C_LOC} in the caller.
-
-Note, however, that @code{TYPE(*)} only accepts scalar arguments,
-unless the @code{DIMENSION} is explicitly specified.  As
-@code{DIMENSION(*)} only supports array (including array elements) but
-no scalars, it is not a full replacement for @code{C_LOC}.  On the
-other hand, assumed-type assumed-rank dummy arguments
-(@code{TYPE(*), DIMENSION(..)}) allow for both scalars and arrays, but
-require special code on the callee side to handle the array descriptor.
-
-@item Assumed-rank arrays (@code{DIMENSION(..)}) as dummy argument
-allow that scalars and arrays of any rank can be passed as actual
-argument. As the Technical Specification does not provide for direct
-means to operate with them, they have to be used either from the C side
-or be converted using @code{C_LOC} and @code{C_F_POINTER} to scalars
-or arrays of a specific rank. The rank can be determined using the
-@code{RANK} intrinisic.
-@end itemize
-
-
-Currently unimplemented:
-
-@itemize @bullet
-
-@item GNU Fortran always uses an array descriptor, which does not
-match the one of the Technical Specification. The
-@code{ISO_Fortran_binding.h} header file and the C functions it
-specifies are not available.
-
-@item Using assumed-shape, assumed-rank and deferred-shape arrays in
-@code{BIND(C)} procedures is not fully supported. In particular,
-C interoperable strings of other length than one are not supported
-as this requires the new array descriptor.
-@end itemize
-
+an array descriptor with different fields in functions without the
+@code{BIND(C)} attribute.  Arguments to functions marked @code{BIND(C)}
+are converted to the specified form.  If you need to access GNU Fortran's
+internal array descriptor, you can use the Chasm Language Interoperability
+Tools, @url{http://chasm-interop.sourceforge.net/}.
 
 @node GNU Fortran Compiler Directives
 @section GNU Fortran Compiler Directives
@@ -5824,7 +5409,6 @@ but they are also things doable by someone who is willing and able.
 @menu
 * Contributors::
 * Projects::
-* Proposed Extensions::
 @end menu
 
 
@@ -5888,6 +5472,7 @@ GNU Fortran project:
 @item Dominique d'Humi@`eres
 @item Kate Hedstrom
 @item Erik Schnetter
+@item Gerhard Steinmetz
 @item Joost VandeVondele
 @end itemize
 
@@ -5917,91 +5502,14 @@ isolating them.  Going through the bugzilla database at
 add more information (for example, for which version does the testcase
 work, for which versions does it fail?) is also very helpful.
 
-@end table
-
+@item Missing features
+For a larger project, consider working on the missing features required for
+Fortran language standards compliance (@pxref{Standards}), or contributing
+to the implementation of extensions such as OpenMP (@pxref{OpenMP}) or
+OpenACC (@pxref{OpenACC}) that are under active development.  Again,
+contributing test cases for these features is useful too!
 
-@node Proposed Extensions
-@section Proposed Extensions
-
-Here's a list of proposed extensions for the GNU Fortran compiler, in no particular
-order.  Most of these are necessary to be fully compatible with
-existing Fortran compilers, but they are not part of the official
-J3 Fortran 95 standard.
-
-@subsection Compiler extensions:
-@itemize @bullet
-@item
-User-specified alignment rules for structures.
-
-@item
-Automatically extend single precision constants to double.
-
-@item
-Compile code that conserves memory by dynamically allocating common and
-module storage either on stack or heap.
-
-@item
-Compile flag to generate code for array conformance checking (suggest -CC).
-
-@item
-User control of symbol names (underscores, etc).
-
-@item
-Compile setting for maximum size of stack frame size before spilling
-parts to static or heap.
-
-@item
-Flag to force local variables into static space.
-
-@item
-Flag to force local variables onto stack.
-@end itemize
-
-
-@subsection Environment Options
-@itemize @bullet
-@item
-Pluggable library modules for random numbers, linear algebra.
-LA should use BLAS calling conventions.
-
-@item
-Environment variables controlling actions on arithmetic exceptions like
-overflow, underflow, precision loss---Generate NaN, abort, default.
-action.
-
-@item
-Set precision for fp units that support it (i387).
-
-@item
-Variable for setting fp rounding mode.
-
-@item
-Variable to fill uninitialized variables with a user-defined bit
-pattern.
-
-@item
-Environment variable controlling filename that is opened for that unit
-number.
-
-@item
-Environment variable to clear/trash memory being freed.
-
-@item
-Environment variable to control tracing of allocations and frees.
-
-@item
-Environment variable to display allocated memory at normal program end.
-
-@item
-Environment variable for filename for * IO-unit.
-
-@item
-Environment variable for temporary file directory.
-
-@item
-Environment variable forcing standard output to be line buffered (Unix).
-
-@end itemize
+@end table
 
 
 @c ---------------------------------------------------------------------
diff --git a/gcc/fortran/interface.c b/gcc/fortran/interface.c
index 24698be..30c99ef 100644
--- a/gcc/fortran/interface.c
+++ b/gcc/fortran/interface.c
@@ -3064,6 +3064,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
   gfc_array_spec *fas, *aas;
   bool pointer_dummy, pointer_arg, allocatable_arg;
 
+  bool ok = true;
+
   actual = *ap;
 
   if (actual == NULL && formal == NULL)
@@ -3134,7 +3136,6 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	  if (where)
 	    gfc_error ("More actual than formal arguments in procedure "
 		       "call at %L", where);
-
 	  return false;
 	}
 
@@ -3192,13 +3193,16 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	  else if (where)
 	    gfc_error ("Fortran 2008: Null pointer at %L to non-pointer "
 		       "dummy %qs", where, f->sym->name);
-
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       if (!compare_parameter (f->sym, a->expr, ranks_must_agree,
 			      is_elemental, where))
-	return false;
+	{
+	  ok = false;
+	  goto match;
+	}
 
       /* TS 29113, 6.3p2; F2018 15.5.2.4.  */
       if (f->sym->ts.type == BT_ASSUMED
@@ -3217,7 +3221,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 			 "has type parameters or is of "
 			 "derived type with type-bound or FINAL procedures",
 			 &a->expr->where);
-	      return false;
+	      ok = false;
+	      goto match;
 	    }
 	}
 
@@ -3249,7 +3254,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 			 mpz_get_si (a->expr->ts.u.cl->length->value.integer),
 			 mpz_get_si (f->sym->ts.u.cl->length->value.integer),
 			 f->sym->name, &a->expr->where);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       if ((f->sym->attr.pointer || f->sym->attr.allocatable)
@@ -3261,7 +3267,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 		       "pointer dummy argument %qs must have a deferred "
 		       "length type parameter if and only if the dummy has one",
 		       &a->expr->where, f->sym->name);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       if (f->sym->ts.type == BT_CLASS)
@@ -3295,7 +3302,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 			       "at %L", f->sym->name, actual_size,
 			       formal_size, &a->expr->where);
 	    }
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
      skip_size_check:
@@ -3312,7 +3320,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	  if (where)
 	    gfc_error ("Expected a procedure pointer for argument %qs at %L",
 		       f->sym->name, &a->expr->where);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       /* Satisfy F03:12.4.1.3 by ensuring that a procedure actual argument is
@@ -3328,7 +3337,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	  if (where)
 	    gfc_error ("Expected a procedure for argument %qs at %L",
 		       f->sym->name, &a->expr->where);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       /* Class array variables and expressions store array info in a
@@ -3392,7 +3402,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	  if (where)
 	    gfc_error ("Actual argument for %qs cannot be an assumed-size"
 		       " array at %L", f->sym->name, where);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       /* Diagnose F2018 C839 (TS29113 C535c).  Here the problem is
@@ -3421,7 +3432,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	    gfc_error ("Actual argument to assumed-rank INTENT(OUT) "
 		       "dummy %qs at %L cannot be of unknown size",
 		       f->sym->name, where);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       if (a->expr->expr_type != EXPR_NULL
@@ -3430,7 +3442,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	  if (where)
 	    gfc_error ("Actual argument for %qs must be a pointer at %L",
 		       f->sym->name, &a->expr->where);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       if (a->expr->expr_type != EXPR_NULL
@@ -3440,7 +3453,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	  if (where)
 	    gfc_error ("Fortran 2008: Non-pointer actual argument at %L to "
 		       "pointer dummy %qs", &a->expr->where,f->sym->name);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
 
@@ -3451,7 +3465,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	    gfc_error ("Coindexed actual argument at %L to pointer "
 		       "dummy %qs",
 		       &a->expr->where, f->sym->name);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       /* Fortran 2008, 12.5.2.5 (no constraint).  */
@@ -3464,7 +3479,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	    gfc_error ("Coindexed actual argument at %L to allocatable "
 		       "dummy %qs requires INTENT(IN)",
 		       &a->expr->where, f->sym->name);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       /* Fortran 2008, C1237.  */
@@ -3479,7 +3495,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 		       "%L requires that dummy %qs has neither "
 		       "ASYNCHRONOUS nor VOLATILE", &a->expr->where,
 		       f->sym->name);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       /* Fortran 2008, 12.5.2.4 (no constraint).  */
@@ -3492,7 +3509,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	    gfc_error ("Coindexed actual argument at %L with allocatable "
 		       "ultimate component to dummy %qs requires either VALUE "
 		       "or INTENT(IN)", &a->expr->where, f->sym->name);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
      if (f->sym->ts.type == BT_CLASS
@@ -3503,7 +3521,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	  if (where)
 	    gfc_error ("Actual CLASS array argument for %qs must be a full "
 		       "array at %L", f->sym->name, &a->expr->where);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
 
@@ -3513,7 +3532,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 	  if (where)
 	    gfc_error ("Actual argument for %qs must be ALLOCATABLE at %L",
 		       f->sym->name, &a->expr->where);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       /* Check intent = OUT/INOUT for definable actual argument.  */
@@ -3529,9 +3549,15 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 		&& CLASS_DATA (f->sym)->attr.class_pointer)
 	       || (f->sym->ts.type != BT_CLASS && f->sym->attr.pointer))
 	      && !gfc_check_vardef_context (a->expr, true, false, false, context))
-	    return false;
+	    {
+	      ok = false;
+	      goto match;
+	    }
 	  if (!gfc_check_vardef_context (a->expr, false, false, false, context))
-	    return false;
+	    {
+	      ok = false;
+	      goto match;
+	    }
 	}
 
       if ((f->sym->attr.intent == INTENT_OUT
@@ -3546,7 +3572,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 		       "INTENT(INOUT), VOLATILE or ASYNCHRONOUS attribute "
 		       "of the dummy argument %qs",
 		       &a->expr->where, f->sym->name);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       /* C1232 (R1221) For an actual argument which is an array section or
@@ -3564,7 +3591,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 		       "incompatible with the non-assumed-shape "
 		       "dummy argument %qs due to VOLATILE attribute",
 		       &a->expr->where,f->sym->name);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       /* Find the last array_ref.  */
@@ -3581,7 +3609,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 		       "incompatible with the non-assumed-shape "
 		       "dummy argument %qs due to VOLATILE attribute",
 		       &a->expr->where, f->sym->name);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
       /* C1233 (R1221) For an actual argument which is a pointer array, the
@@ -3601,7 +3630,8 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
 		       "an assumed-shape or pointer-array dummy "
 		       "argument %qs due to VOLATILE attribute",
 		       &a->expr->where,f->sym->name);
-	  return false;
+	  ok = false;
+	  goto match;
 	}
 
     match:
@@ -3611,6 +3641,10 @@ gfc_compare_actual_formal (gfc_actual_arglist **ap, gfc_formal_arglist *formal,
       new_arg[i++] = a;
     }
 
+  /* Give up now if we saw any bad argument.  */
+  if (!ok)
+    return false;
+
   /* Make sure missing actual arguments are optional.  */
   i = 0;
   for (f = formal; f; f = f->next, i++)
diff --git a/gcc/fortran/intrinsic.c b/gcc/fortran/intrinsic.c
index f5c88d9..2d7d246 100644
--- a/gcc/fortran/intrinsic.c
+++ b/gcc/fortran/intrinsic.c
@@ -223,6 +223,7 @@ static bool
 do_ts29113_check (gfc_intrinsic_sym *specific, gfc_actual_arglist *arg)
 {
   gfc_actual_arglist *a;
+  bool ok = true;
 
   for (a = arg; a; a = a->next)
     {
@@ -238,7 +239,7 @@ do_ts29113_check (gfc_intrinsic_sym *specific, gfc_actual_arglist *arg)
 	  gfc_error ("Variable with NO_ARG_CHECK attribute at %L is only "
 		     "permitted as argument to the intrinsic functions "
 		     "C_LOC and PRESENT", &a->expr->where);
-	  return false;
+	  ok = false;
 	}
       else if (a->expr->ts.type == BT_ASSUMED
 	       && specific->id != GFC_ISYM_LBOUND
@@ -254,32 +255,32 @@ do_ts29113_check (gfc_intrinsic_sym *specific, gfc_actual_arglist *arg)
 	  gfc_error ("Assumed-type argument at %L is not permitted as actual"
 		     " argument to the intrinsic %s", &a->expr->where,
 		     gfc_current_intrinsic);
-	  return false;
+	  ok = false;
 	}
       else if (a->expr->ts.type == BT_ASSUMED && a != arg)
 	{
 	  gfc_error ("Assumed-type argument at %L is only permitted as "
 		     "first actual argument to the intrinsic %s",
 		     &a->expr->where, gfc_current_intrinsic);
-	  return false;
+	  ok = false;
 	}
-      if (a->expr->rank == -1 && !specific->inquiry)
+      else if (a->expr->rank == -1 && !specific->inquiry)
 	{
 	  gfc_error ("Assumed-rank argument at %L is only permitted as actual "
 		     "argument to intrinsic inquiry functions",
 		     &a->expr->where);
-	  return false;
+	  ok = false;
 	}
-      if (a->expr->rank == -1 && arg != a)
+      else if (a->expr->rank == -1 && arg != a)
 	{
 	  gfc_error ("Assumed-rank argument at %L is only permitted as first "
 		     "actual argument to the intrinsic inquiry function %s",
 		     &a->expr->where, gfc_current_intrinsic);
-	  return false;
+	  ok = false;
 	}
     }
 
-  return true;
+  return ok;
 }
 
 
@@ -888,39 +889,6 @@ add_sym_4 (const char *name, gfc_isym_id id, enum klass cl, int actual_ok, bt ty
 	   (void *) 0);
 }
 
-/* Add a symbol to the function list where the function takes 4
-   arguments and resolution may need to change the number or
-   arrangement of arguments. This is the case for INDEX, which needs
-   its KIND argument removed.  */
-
-static void
-add_sym_4ind (const char *name, gfc_isym_id id, enum klass cl, int actual_ok,
-	      bt type, int kind, int standard,
-	      bool (*check) (gfc_expr *, gfc_expr *, gfc_expr *, gfc_expr *),
-	      gfc_expr *(*simplify) (gfc_expr *, gfc_expr *, gfc_expr *,
-				     gfc_expr *),
-	      void (*resolve) (gfc_expr *, gfc_actual_arglist *),
-	      const char *a1, bt type1, int kind1, int optional1,
-	      const char *a2, bt type2, int kind2, int optional2,
-	      const char *a3, bt type3, int kind3, int optional3,
-	      const char *a4, bt type4, int kind4, int optional4 )
-{
-  gfc_check_f cf;
-  gfc_simplify_f sf;
-  gfc_resolve_f rf;
-
-  cf.f4 = check;
-  sf.f4 = simplify;
-  rf.f1m = resolve;
-
-  add_sym (name, id, cl, actual_ok, type, kind, standard, cf, sf, rf,
-	   a1, type1, kind1, optional1, INTENT_IN,
-	   a2, type2, kind2, optional2, INTENT_IN,
-	   a3, type3, kind3, optional3, INTENT_IN,
-	   a4, type4, kind4, optional4, INTENT_IN,
-	   (void *) 0);
-}
-
 
 /* Add a symbol to the subroutine list where the subroutine takes
    4 arguments.  */
@@ -2223,11 +2191,11 @@ add_functions (void)
 
   /* The resolution function for INDEX is called gfc_resolve_index_func
      because the name gfc_resolve_index is already used in resolve.c.  */
-  add_sym_4ind ("index", GFC_ISYM_INDEX, CLASS_ELEMENTAL, ACTUAL_YES,
-		BT_INTEGER, di, GFC_STD_F77,
-		gfc_check_index, gfc_simplify_index, gfc_resolve_index_func,
-		stg, BT_CHARACTER, dc, REQUIRED, ssg, BT_CHARACTER, dc, REQUIRED,
-		bck, BT_LOGICAL, dl, OPTIONAL, kind, BT_INTEGER, di, OPTIONAL);
+  add_sym_4 ("index", GFC_ISYM_INDEX, CLASS_ELEMENTAL, ACTUAL_YES,
+	     BT_INTEGER, di, GFC_STD_F77,
+	     gfc_check_index, gfc_simplify_index, gfc_resolve_index_func,
+	     stg, BT_CHARACTER, dc, REQUIRED, ssg, BT_CHARACTER, dc, REQUIRED,
+	     bck, BT_LOGICAL, dl, OPTIONAL, kind, BT_INTEGER, di, OPTIONAL);
 
   make_generic ("index", GFC_ISYM_INDEX, GFC_STD_F77);
 
@@ -3805,7 +3773,7 @@ add_subroutines (void)
 	      BT_UNKNOWN, 0, GFC_STD_F2018,
 	      gfc_check_co_reduce, NULL, NULL,
 	      a, BT_REAL, dr, REQUIRED, INTENT_INOUT,
-	      "operator", BT_INTEGER, di, REQUIRED, INTENT_IN,
+	      "operation", BT_INTEGER, di, REQUIRED, INTENT_IN,
 	      result_image, BT_INTEGER, di, OPTIONAL, INTENT_IN,
 	      stat, BT_INTEGER, di, OPTIONAL, INTENT_OUT,
 	      errmsg, BT_CHARACTER, dc, OPTIONAL, INTENT_INOUT);
@@ -4530,10 +4498,9 @@ resolve_intrinsic (gfc_intrinsic_sym *specific, gfc_expr *e)
 
   arg = e->value.function.actual;
 
-  /* Special case hacks for MIN, MAX and INDEX.  */
+  /* Special case hacks for MIN and MAX.  */
   if (specific->resolve.f1m == gfc_resolve_max
-      || specific->resolve.f1m == gfc_resolve_min
-      || specific->resolve.f1m == gfc_resolve_index_func)
+      || specific->resolve.f1m == gfc_resolve_min)
     {
       (*specific->resolve.f1m) (e, arg);
       return;
diff --git a/gcc/fortran/intrinsic.h b/gcc/fortran/intrinsic.h
index 2148f89..fb655fb 100644
--- a/gcc/fortran/intrinsic.h
+++ b/gcc/fortran/intrinsic.h
@@ -168,7 +168,6 @@ bool gfc_check_spread (gfc_expr *, gfc_expr *, gfc_expr *);
 bool gfc_check_srand (gfc_expr *);
 bool gfc_check_stat (gfc_expr *, gfc_expr *);
 bool gfc_check_storage_size (gfc_expr *, gfc_expr *);
-bool gfc_check_sum (gfc_expr *, gfc_expr *, gfc_expr *);
 bool gfc_check_symlnk (gfc_expr *, gfc_expr *);
 bool gfc_check_team_number (gfc_expr *);
 bool gfc_check_transf_bit_intrins (gfc_actual_arglist *);
@@ -459,7 +458,6 @@ void gfc_resolve_asinh (gfc_expr *, gfc_expr *);
 void gfc_resolve_atan (gfc_expr *, gfc_expr *);
 void gfc_resolve_atanh (gfc_expr *, gfc_expr *);
 void gfc_resolve_atan2 (gfc_expr *, gfc_expr *, gfc_expr *);
-void gfc_resolve_atan2d (gfc_expr *, gfc_expr *, gfc_expr *);
 void gfc_resolve_atomic_def (gfc_code *);
 void gfc_resolve_atomic_ref (gfc_code *);
 void gfc_resolve_besn (gfc_expr *, gfc_expr *, gfc_expr *);
@@ -521,7 +519,8 @@ void gfc_resolve_ibits (gfc_expr *, gfc_expr *, gfc_expr *, gfc_expr *);
 void gfc_resolve_ibset (gfc_expr *, gfc_expr *, gfc_expr *);
 void gfc_resolve_image_index (gfc_expr *, gfc_expr *, gfc_expr *);
 void gfc_resolve_image_status (gfc_expr *, gfc_expr *, gfc_expr *);
-void gfc_resolve_index_func (gfc_expr *, gfc_actual_arglist *);
+void gfc_resolve_index_func (gfc_expr *, gfc_expr *, gfc_expr *, gfc_expr *,
+			     gfc_expr *);
 void gfc_resolve_ierrno (gfc_expr *);
 void gfc_resolve_ieor (gfc_expr *, gfc_expr *, gfc_expr *);
 void gfc_resolve_ichar (gfc_expr *, gfc_expr *, gfc_expr *);
@@ -542,7 +541,6 @@ void gfc_resolve_rshift (gfc_expr *, gfc_expr *, gfc_expr *);
 void gfc_resolve_lshift (gfc_expr *, gfc_expr *, gfc_expr *);
 void gfc_resolve_ishft (gfc_expr *, gfc_expr *, gfc_expr *);
 void gfc_resolve_ishftc (gfc_expr *, gfc_expr *, gfc_expr *, gfc_expr *);
-void gfc_resolve_kill (gfc_expr *, gfc_expr *, gfc_expr *);
 void gfc_resolve_lbound (gfc_expr *, gfc_expr *, gfc_expr *, gfc_expr *);
 void gfc_resolve_lcobound (gfc_expr *, gfc_expr *, gfc_expr *, gfc_expr *);
 void gfc_resolve_len (gfc_expr *, gfc_expr *, gfc_expr *);
@@ -658,7 +656,6 @@ void gfc_resolve_gmtime (gfc_code *);
 void gfc_resolve_hostnm_sub (gfc_code *);
 void gfc_resolve_idate (gfc_code *);
 void gfc_resolve_itime (gfc_code *);
-void gfc_resolve_kill_sub (gfc_code *);
 void gfc_resolve_lstat_sub (gfc_code *);
 void gfc_resolve_ltime (gfc_code *);
 void gfc_resolve_mvbits (gfc_code *);
diff --git a/gcc/fortran/intrinsic.texi b/gcc/fortran/intrinsic.texi
index e3916ba..04d4cb5 100644
--- a/gcc/fortran/intrinsic.texi
+++ b/gcc/fortran/intrinsic.texi
@@ -202,10 +202,9 @@ Some basic guidelines for editing this document:
 * @code{LNBLNK}:        LNBLNK,    Index of the last non-blank character in a string
 * @code{LOC}:           LOC,       Returns the address of a variable
 * @code{LOG}:           LOG,       Logarithm function
-* @code{LOG10}:         LOG10,     Base 10 logarithm function 
+* @code{LOG10}:         LOG10,     Base 10 logarithm function
 * @code{LOG_GAMMA}:     LOG_GAMMA, Logarithm of the Gamma function
 * @code{LOGICAL}:       LOGICAL,   Convert to logical type
-* @code{LONG}:          LONG,      Convert to integer type
 * @code{LSHIFT}:        LSHIFT,    Left shift bits
 * @code{LSTAT}:         LSTAT,     Get file status
 * @code{LTIME}:         LTIME,     Convert time to local time info
@@ -314,14 +313,11 @@ Some basic guidelines for editing this document:
 @node Introduction to Intrinsics
 @section Introduction to intrinsic procedures
 
-The intrinsic procedures provided by GNU Fortran include all of the
-intrinsic procedures required by the Fortran 95 standard, a set of
-intrinsic procedures for backwards compatibility with G77, and a
-selection of intrinsic procedures from the Fortran 2003 and Fortran 2008
-standards.  Any conflict between a description here and a description in
-either the Fortran 95 standard, the Fortran 2003 standard or the Fortran
-2008 standard is unintentional, and the standard(s) should be considered
-authoritative.
+The intrinsic procedures provided by GNU Fortran include procedures required
+by the Fortran 95 and later supported standards, and a set of intrinsic
+procedures for backwards compatibility with G77.  Any conflict between
+a description here and a description in the Fortran standards is
+unintentional, and the standard(s) should be considered authoritative.
 
 The enumeration of the @code{KIND} type parameter is processor defined in
 the Fortran 95 standard.  GNU Fortran defines the default integer type and
@@ -340,7 +336,7 @@ Many of the intrinsic procedures take one or more optional arguments.
 This document follows the convention used in the Fortran 95 standard,
 and denotes such arguments by square brackets.
 
-GNU Fortran offers the @option{-std=f95} and @option{-std=gnu} options,
+GNU Fortran offers the @option{-std=} command-line option,
 which can be used to restrict the set of intrinsic procedures to a 
 given standard.  By default, @command{gfortran} sets the @option{-std=gnu}
 option, and so all intrinsic procedures described here are accepted.  There
@@ -445,7 +441,7 @@ end program test_abs
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{ABS(A)}   @tab @code{REAL(4) A}    @tab @code{REAL(4)}    @tab Fortran 77 and later
 @item @code{CABS(A)}  @tab @code{COMPLEX(4) A} @tab @code{REAL(4)}    @tab Fortran 77 and later
@@ -610,7 +606,7 @@ end program test_acos
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument         @tab Return type     @tab Standard
 @item @code{ACOS(X)}  @tab @code{REAL(4) X} @tab @code{REAL(4)}  @tab Fortran 77 and later
 @item @code{DACOS(X)} @tab @code{REAL(8) X} @tab @code{REAL(8)}  @tab Fortran 77 and later
@@ -669,7 +665,7 @@ end program test_acosd
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument         @tab Return type     @tab Standard
 @item @code{ACOSD(X)}  @tab @code{REAL(4) X} @tab @code{REAL(4)}  @tab GNU extension
 @item @code{DACOSD(X)} @tab @code{REAL(8) X} @tab @code{REAL(8)}  @tab GNU extension
@@ -725,7 +721,7 @@ END PROGRAM
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument          @tab Return type       @tab Standard
 @item @code{DACOSH(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab GNU extension
 @end multitable
@@ -874,7 +870,7 @@ end program test_aimag
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name               @tab Argument            @tab Return type     @tab Standard
 @item @code{AIMAG(Z)}    @tab @code{COMPLEX Z}    @tab @code{REAL}     @tab Fortran 77 and later
 @item @code{DIMAG(Z)}    @tab @code{COMPLEX(8) Z} @tab @code{REAL(8)}  @tab GNU extension
@@ -934,7 +930,7 @@ end program test_aint
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name           @tab Argument         @tab Return type      @tab Standard
 @item @code{AINT(A)} @tab @code{REAL(4) A} @tab @code{REAL(4)}   @tab Fortran 77 and later
 @item @code{DINT(A)} @tab @code{REAL(8) A} @tab @code{REAL(8)}   @tab Fortran 77 and later
@@ -1214,7 +1210,7 @@ end program test_anint
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument         @tab Return type      @tab Standard
 @item @code{ANINT(A)}  @tab @code{REAL(4) A} @tab @code{REAL(4)}   @tab Fortran 77 and later
 @item @code{DNINT(A)} @tab @code{REAL(8) A} @tab @code{REAL(8)}   @tab Fortran 77 and later
@@ -1330,7 +1326,7 @@ end program test_asin
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{ASIN(X)}  @tab @code{REAL(4) X}  @tab @code{REAL(4)}    @tab Fortran 77 and later
 @item @code{DASIN(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab Fortran 77 and later
@@ -1389,7 +1385,7 @@ end program test_asind
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{ASIND(X)}  @tab @code{REAL(4) X}  @tab @code{REAL(4)}    @tab GNU extension
 @item @code{DASIND(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab GNU extension
@@ -1445,7 +1441,7 @@ END PROGRAM
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument          @tab Return type       @tab Standard
 @item @code{DASINH(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab GNU extension.
 @end multitable
@@ -1581,7 +1577,7 @@ end program test_atan
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{ATAN(X)}  @tab @code{REAL(4) X}  @tab @code{REAL(4)}    @tab Fortran 77 and later
 @item @code{DATAN(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab Fortran 77 and later
@@ -1646,7 +1642,7 @@ end program test_atand
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .23 .23 .20 .30
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{ATAND(X)}  @tab @code{REAL(4) X}  @tab @code{REAL(4)}    @tab GNU extension
 @item @code{DATAND(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab GNU extension
@@ -1711,7 +1707,7 @@ end program test_atan2
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .22 .22 .20 .32
 @headitem Name                @tab Argument            @tab Return type    @tab Standard
 @item @code{ATAN2(X, Y)}  @tab @code{REAL(4) X, Y} @tab @code{REAL(4)} @tab Fortran 77 and later
 @item @code{DATAN2(X, Y)} @tab @code{REAL(8) X, Y} @tab @code{REAL(8)} @tab Fortran 77 and later
@@ -1779,7 +1775,7 @@ end program test_atan2d
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .23 .23 .20 .30
 @headitem Name                @tab Argument            @tab Return type    @tab Standard
 @item @code{ATAN2D(X, Y)}  @tab @code{REAL(4) X, Y} @tab @code{REAL(4)} @tab GNU extension
 @item @code{DATAN2D(X, Y)} @tab @code{REAL(8) X, Y} @tab @code{REAL(8)} @tab GNU extension
@@ -1835,7 +1831,7 @@ END PROGRAM
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument          @tab Return type       @tab Standard
 @item @code{DATANH(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab GNU extension
 @end multitable
@@ -2524,7 +2520,7 @@ end program test_besj0
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .21 .22 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{DBESJ0(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}   @tab GNU extension
 @end multitable
@@ -2573,7 +2569,7 @@ end program test_besj1
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument          @tab Return type       @tab Standard
 @item @code{DBESJ1(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab GNU extension
 @end multitable
@@ -2638,7 +2634,7 @@ end program test_besjn
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .22 .22 .20 .32
 @headitem Name                @tab Argument            @tab Return type       @tab Standard
 @item @code{DBESJN(N, X)} @tab @code{INTEGER N}    @tab @code{REAL(8)}    @tab GNU extension
 @item                     @tab @code{REAL(8) X}    @tab                   @tab
@@ -2686,7 +2682,7 @@ end program test_besy0
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{DBESY0(X)}@tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab GNU extension
 @end multitable
@@ -2733,7 +2729,7 @@ end program test_besy1
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{DBESY1(X)}@tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab GNU extension
 @end multitable
@@ -2798,10 +2794,10 @@ end program test_besyn
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name               @tab Argument            @tab Return type     @tab Standard
 @item @code{DBESYN(N,X)} @tab @code{INTEGER N} @tab @code{REAL(8)}  @tab GNU extension
-@item                    @tab @code{REAL(8) X} @tab                 @tab 
+@item                    @tab @code{REAL(8) X} @tab                 @tab
 @end multitable
 @end table
 
@@ -3041,7 +3037,7 @@ end program test_btest
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .21 .28 .18 .30
 @headitem Name           @tab Argument         @tab Return type             @tab Standard
 @item @code{BTEST(I,POS)} @tab @code{INTEGER I,POS} @tab @code{LOGICAL} @tab Fortran 95 and later
 @item @code{BBTEST(I,POS)} @tab @code{INTEGER(1) I,POS} @tab @code{LOGICAL(1)} @tab GNU extension
@@ -3458,7 +3454,7 @@ end program test_char
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .18 .18 .24 .25
+@multitable @columnfractions .19 .19 .25 .33
 @headitem Name           @tab Argument         @tab Return type             @tab Standard
 @item @code{CHAR(I)} @tab @code{INTEGER I} @tab @code{CHARACTER(LEN=1)} @tab Fortran 77 and later
 @end multitable
@@ -3829,7 +3825,7 @@ end program test
 @table @asis
 @item @emph{Description}:
 @code{CO_REDUCE} determines element-wise the reduction of the value of @var{A}
-on all images of the current team.  The pure function passed as @var{OPERATOR}
+on all images of the current team.  The pure function passed as @var{OPERATION}
 is used to pairwise reduce the values of @var{A} by passing either the value
 of @var{A} of different images or the result values of such a reduction as
 argument.  If @var{A} is an array, the deduction is done element wise. If
@@ -3848,7 +3844,7 @@ Technical Specification (TS) 18508 or later
 Collective subroutine
 
 @item @emph{Syntax}:
-@code{CALL CO_REDUCE(A, OPERATOR, [, RESULT_IMAGE, STAT, ERRMSG])}
+@code{CALL CO_REDUCE(A, OPERATION, [, RESULT_IMAGE, STAT, ERRMSG])}
 
 @item @emph{Arguments}:
 @multitable @columnfractions .20 .65
@@ -3857,12 +3853,12 @@ nonpolymorphic. If it is allocatable, it shall be allocated; if it is a pointer,
 it shall be associated.  @var{A} shall have the same type and type parameters on
 all images of the team; if it is an array, it shall have the same shape on all
 images.
-@item @var{OPERATOR}     @tab pure function with two scalar nonallocatable
+@item @var{OPERATION}     @tab pure function with two scalar nonallocatable
 arguments, which shall be nonpolymorphic and have the same type and type
 parameters as @var{A}.  The function shall return a nonallocatable scalar of
 the same type and type parameters as @var{A}.  The function shall be the same on
 all images and with regards to the arguments mathematically commutative and
-associative.  Note that @var{OPERATOR} may not be an elemental function, unless
+associative.  Note that @var{OPERATION} may not be an elemental function, unless
 it is an intrisic function.
 @item @var{RESULT_IMAGE} @tab (optional) a scalar integer expression; if
 present, it shall have the same value on all images and refer to an
@@ -3876,7 +3872,7 @@ image of the current team.
 program test
   integer :: val
   val = this_image ()
-  call co_reduce (val, result_image=1, operator=myprod)
+  call co_reduce (val, result_image=1, operation=myprod)
   if (this_image() == 1) then
     write(*,*) "Product value", val  ! prints num_images() factorial
   end if
@@ -4187,7 +4183,7 @@ end program test_conjg
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument             @tab Return type       @tab Standard
 @item @code{DCONJG(Z)} @tab @code{COMPLEX(8) Z}  @tab @code{COMPLEX(8)} @tab GNU extension
 @end multitable
@@ -4238,7 +4234,7 @@ end program test_cos
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{COS(X)}   @tab @code{REAL(4) X}    @tab @code{REAL(4)}    @tab Fortran 77 and later
 @item @code{DCOS(X)}  @tab @code{REAL(8) X}    @tab @code{REAL(8)}    @tab Fortran 77 and later
@@ -4302,7 +4298,7 @@ end program test_cosd
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{COSD(X)}   @tab @code{REAL(4) X}    @tab @code{REAL(4)}    @tab GNU extension
 @item @code{DCOSD(X)}  @tab @code{REAL(8) X}    @tab @code{REAL(8)}    @tab GNU extension
@@ -4361,7 +4357,7 @@ end program test_cosh
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{COSH(X)}  @tab @code{REAL(4) X}  @tab @code{REAL(4)}    @tab Fortran 77 and later
 @item @code{DCOSH(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab Fortran 77 and later
@@ -4415,7 +4411,7 @@ end program test_cotan
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type     @tab Standard
 @item @code{COTAN(X)}   @tab @code{REAL(4) X}  @tab @code{REAL(4)}  @tab GNU extension
 @item @code{DCOTAN(X)}  @tab @code{REAL(8) X}  @tab @code{REAL(8)}  @tab GNU extension
@@ -4471,7 +4467,7 @@ end program test_cotand
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type     @tab Standard
 @item @code{COTAND(X)}   @tab @code{REAL(4) X}  @tab @code{REAL(4)}  @tab GNU extension
 @item @code{DCOTAND(X)}  @tab @code{REAL(8) X}  @tab @code{REAL(8)}  @tab GNU extension
@@ -4986,7 +4982,7 @@ end program test_dim
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .26 .20 .30
 @headitem Name             @tab Argument               @tab Return type       @tab Standard
 @item @code{DIM(X,Y)}  @tab @code{REAL(4) X, Y}    @tab @code{REAL(4)}    @tab Fortran 77 and later
 @item @code{IDIM(X,Y)} @tab @code{INTEGER(4) X, Y} @tab @code{INTEGER(4)} @tab Fortran 77 and later
@@ -5089,7 +5085,7 @@ end program test_dprod
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name              @tab Argument               @tab Return type       @tab Standard
 @item @code{DPROD(X,Y)} @tab @code{REAL(4) X, Y}    @tab @code{REAL(8)}    @tab Fortran 77 and later
 @end multitable
@@ -5458,7 +5454,7 @@ end program test_erf
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{DERF(X)}  @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab GNU extension
 @end multitable
@@ -5502,7 +5498,7 @@ end program test_erfc
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{DERFC(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab GNU extension
 @end multitable
@@ -5846,7 +5842,7 @@ end program test_exp
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument             @tab Return type         @tab Standard
 @item @code{EXP(X)}   @tab @code{REAL(4) X}     @tab @code{REAL(4)}      @tab Fortran 77 and later
 @item @code{DEXP(X)}  @tab @code{REAL(8) X}     @tab @code{REAL(8)}      @tab Fortran 77 and later
@@ -6786,7 +6782,7 @@ end program test_gamma
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument         @tab Return type       @tab Standard
 @item @code{DGAMMA(X)} @tab @code{REAL(8) X} @tab @code{REAL(8)}    @tab GNU extension
 @end multitable
@@ -7675,7 +7671,7 @@ END PROGRAM
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{IAND(A)}   @tab @code{INTEGER A}    @tab @code{INTEGER}    @tab Fortran 90 and later
 @item @code{BIAND(A)} @tab @code{INTEGER(1) A} @tab @code{INTEGER(1)} @tab GNU extension
@@ -7838,7 +7834,7 @@ The return value is of type @code{INTEGER} and of the same kind as
 @var{I}.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{IBCLR(A)}   @tab @code{INTEGER A}    @tab @code{INTEGER}    @tab Fortran 90 and later
 @item @code{BBCLR(A)} @tab @code{INTEGER(1) A} @tab @code{INTEGER(1)} @tab GNU extension
@@ -7897,7 +7893,7 @@ The return value is of type @code{INTEGER} and of the same kind as
 @var{I}.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{IBITS(A)}   @tab @code{INTEGER A}    @tab @code{INTEGER}    @tab Fortran 90 and later
 @item @code{BBITS(A)} @tab @code{INTEGER(1) A} @tab @code{INTEGER(1)} @tab GNU extension
@@ -7951,7 +7947,7 @@ The return value is of type @code{INTEGER} and of the same kind as
 @var{I}.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{IBSET(A)}   @tab @code{INTEGER A}    @tab @code{INTEGER}    @tab Fortran 90 and later
 @item @code{BBSET(A)} @tab @code{INTEGER(1) A} @tab @code{INTEGER(1)} @tab GNU extension
@@ -8012,7 +8008,7 @@ end program test_ichar
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument             @tab Return type       @tab Standard
 @item @code{ICHAR(C)}  @tab @code{CHARACTER C}   @tab @code{INTEGER(4)}    @tab Fortran 77 and later
 @end multitable
@@ -8139,7 +8135,7 @@ A boz-literal-constant is converted to an @code{INTEGER} with the kind
 type parameter of the other argument as-if a call to @ref{INT} occurred.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{IEOR(A)}  @tab @code{INTEGER A}    @tab @code{INTEGER}    @tab Fortran 90 and later
 @item @code{BIEOR(A)} @tab @code{INTEGER(1) A} @tab @code{INTEGER(1)} @tab GNU extension
@@ -8276,9 +8272,9 @@ The return value is of type @code{INTEGER} and of kind @var{KIND}. If
 @var{KIND} is absent, the return value is of default integer kind.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .35 .15 .17 .30
 @headitem Name                            @tab Argument           @tab Return type       @tab Standard
-@item @code{INDEX(STRING, SUBSTRING)} @tab @code{CHARACTER}   @tab @code{INTEGER(4)} @tab Fortran 77 and later
+@item @code{INDEX(STRING,SUBSTRING)} @tab @code{CHARACTER}   @tab @code{INTEGER(4)} @tab Fortran 77 and later
 @end multitable
 
 @item @emph{See also}:
@@ -8343,7 +8339,7 @@ end program
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{INT(A)}   @tab @code{REAL(4) A}  @tab @code{INTEGER}    @tab Fortran 77 and later
 @item @code{IFIX(A)}  @tab @code{REAL(4) A}  @tab @code{INTEGER}    @tab Fortran 77 and later
@@ -8356,7 +8352,6 @@ end program
 @node INT2
 @section @code{INT2} --- Convert to 16-bit integer type
 @fnindex INT2
-@fnindex SHORT
 @cindex conversion, to integer
 
 @table @asis
@@ -8365,8 +8360,6 @@ Convert to a @code{KIND=2} integer type. This is equivalent to the
 standard @code{INT} intrinsic with an optional argument of
 @code{KIND=2}, and is only included for backwards compatibility.
 
-The @code{SHORT} intrinsic is equivalent to @code{INT2}.
-
 @item @emph{Standard}:
 GNU extension
 
@@ -8387,8 +8380,7 @@ The return value is a @code{INTEGER(2)} variable.
 
 @item @emph{See also}:
 @ref{INT}, @gol
-@ref{INT8}, @gol
-@ref{LONG}
+@ref{INT8}
 @end table
 
 
@@ -8424,8 +8416,7 @@ The return value is a @code{INTEGER(8)} variable.
 
 @item @emph{See also}:
 @ref{INT}, @gol
-@ref{INT2}, @gol
-@ref{LONG}
+@ref{INT2}
 @end table
 
 
@@ -8469,7 +8460,7 @@ A boz-literal-constant is converted to an @code{INTEGER} with the kind
 type parameter of the other argument as-if a call to @ref{INT} occurred.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{IOR(A)}   @tab @code{INTEGER A}    @tab @code{INTEGER}    @tab Fortran 90 and later
 @item @code{BIOR(A)} @tab @code{INTEGER(1) A} @tab @code{INTEGER(1)} @tab GNU extension
@@ -8823,7 +8814,7 @@ The return value is of type @code{INTEGER} and of the same kind as
 @var{I}.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{ISHFT(A)}   @tab @code{INTEGER A}    @tab @code{INTEGER}    @tab Fortran 90 and later
 @item @code{BSHFT(A)} @tab @code{INTEGER(1) A} @tab @code{INTEGER(1)} @tab GNU extension
@@ -8881,7 +8872,7 @@ The return value is of type @code{INTEGER} and of the same kind as
 @var{I}.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{ISHFTC(A)}   @tab @code{INTEGER A}    @tab @code{INTEGER}    @tab Fortran 90 and later
 @item @code{BSHFTC(A)} @tab @code{INTEGER(1) A} @tab @code{INTEGER(1)} @tab GNU extension
@@ -9242,7 +9233,7 @@ The return value is of type @code{INTEGER} and of kind @var{KIND}. If
 
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name               @tab Argument          @tab Return type       @tab Standard
 @item @code{LEN(STRING)} @tab @code{CHARACTER}  @tab @code{INTEGER}    @tab Fortran 77 and later
 @end multitable
@@ -9335,9 +9326,9 @@ Returns @code{.TRUE.} if @code{STRING_A >= STRING_B}, and @code{.FALSE.}
 otherwise, based on the ASCII ordering.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .34 .16 .17 .30
 @headitem Name                           @tab Argument          @tab Return type       @tab Standard
-@item @code{LGE(STRING_A, STRING_B)} @tab @code{CHARACTER}  @tab @code{LOGICAL}    @tab Fortran 77 and later
+@item @code{LGE(STRING_A,STRING_B)} @tab @code{CHARACTER}  @tab @code{LOGICAL}    @tab Fortran 77 and later
 @end multitable
 
 @item @emph{See also}:
@@ -9389,9 +9380,9 @@ Returns @code{.TRUE.} if @code{STRING_A > STRING_B}, and @code{.FALSE.}
 otherwise, based on the ASCII ordering.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .34 .16 .17 .30
 @headitem Name                           @tab Argument          @tab Return type       @tab Standard
-@item @code{LGT(STRING_A, STRING_B)} @tab @code{CHARACTER}  @tab @code{LOGICAL}    @tab Fortran 77 and later
+@item @code{LGT(STRING_A,STRING_B)} @tab @code{CHARACTER}  @tab @code{LOGICAL}    @tab Fortran 77 and later
 @end multitable
 
 @item @emph{See also}:
@@ -9487,9 +9478,9 @@ Returns @code{.TRUE.} if @code{STRING_A <= STRING_B}, and @code{.FALSE.}
 otherwise, based on the ASCII ordering.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .34 .16 .17 .30
 @headitem Name                           @tab Argument          @tab Return type       @tab Standard
-@item @code{LLE(STRING_A, STRING_B)} @tab @code{CHARACTER}  @tab @code{LOGICAL}    @tab Fortran 77 and later
+@item @code{LLE(STRING_A,STRING_B)} @tab @code{CHARACTER}  @tab @code{LOGICAL}    @tab Fortran 77 and later
 @end multitable
 
 @item @emph{See also}:
@@ -9541,9 +9532,9 @@ Returns @code{.TRUE.} if @code{STRING_A < STRING_B}, and @code{.FALSE.}
 otherwise, based on the ASCII ordering.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .34 .16 .17 .30
 @headitem Name                           @tab Argument          @tab Return type       @tab Standard
-@item @code{LLT(STRING_A, STRING_B)} @tab @code{CHARACTER}  @tab @code{LOGICAL}    @tab Fortran 77 and later
+@item @code{LLT(STRING_A,STRING_B)} @tab @code{CHARACTER}  @tab @code{LOGICAL}    @tab Fortran 77 and later
 @end multitable
 
 @item @emph{See also}:
@@ -9680,7 +9671,7 @@ end program test_log
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{ALOG(X)}  @tab @code{REAL(4) X}  @tab @code{REAL(4)}    @tab Fortran 77 or later
 @item @code{DLOG(X)}  @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab Fortran 77 or later
@@ -9732,7 +9723,7 @@ end program test_log10
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{ALOG10(X)}  @tab @code{REAL(4) X}  @tab @code{REAL(4)}    @tab Fortran 77 and later
 @item @code{DLOG10(X)}  @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab Fortran 77 and later
@@ -9781,7 +9772,7 @@ end program test_log_gamma
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument         @tab Return type       @tab Standard
 @item @code{LGAMMA(X)} @tab @code{REAL(4) X} @tab @code{REAL(4)}    @tab GNU extension
 @item @code{ALGAMA(X)} @tab @code{REAL(4) X} @tab @code{REAL(4)}    @tab GNU extension
@@ -9833,44 +9824,6 @@ kind corresponding to @var{KIND}, or of the default logical kind if
 
 
 
-@node LONG
-@section @code{LONG} --- Convert to integer type
-@fnindex LONG
-@cindex conversion, to integer
-
-@table @asis
-@item @emph{Description}:
-Convert to a @code{KIND=4} integer type, which is the same size as a C
-@code{long} integer.  This is equivalent to the standard @code{INT}
-intrinsic with an optional argument of @code{KIND=4}, and is only
-included for backwards compatibility.
-
-@item @emph{Standard}:
-GNU extension
-
-@item @emph{Class}:
-Elemental function
-
-@item @emph{Syntax}:
-@code{RESULT = LONG(A)}
-
-@item @emph{Arguments}:
-@multitable @columnfractions .15 .70
-@item @var{A}    @tab Shall be of type @code{INTEGER},
-@code{REAL}, or @code{COMPLEX}.
-@end multitable
-
-@item @emph{Return value}:
-The return value is a @code{INTEGER(4)} variable.
-
-@item @emph{See also}:
-@ref{INT}, @gol
-@ref{INT2}, @gol
-@ref{INT8}
-@end table
-
-
-
 @node LSHIFT
 @section @code{LSHIFT} --- Left shift bits
 @fnindex LSHIFT
@@ -10240,7 +10193,7 @@ The return value corresponds to the maximum value among the arguments,
 and has the same type and kind as the first argument.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument             @tab Return type         @tab Standard
 @item @code{MAX0(A1)}  @tab @code{INTEGER(4) A1} @tab @code{INTEGER(4)}   @tab Fortran 77 and later
 @item @code{AMAX0(A1)} @tab @code{INTEGER(4) A1} @tab @code{REAL(MAX(X))} @tab Fortran 77 and later
@@ -10615,7 +10568,7 @@ The return value corresponds to the minimum value among the arguments,
 and has the same type and kind as the first argument.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name              @tab Argument             @tab Return type        @tab Standard
 @item @code{MIN0(A1)}   @tab @code{INTEGER(4) A1} @tab @code{INTEGER(4)}  @tab Fortran 77 and later
 @item @code{AMIN0(A1)}  @tab @code{INTEGER(4) A1} @tab @code{REAL(4)}     @tab Fortran 77 and later
@@ -10848,7 +10801,7 @@ end program test_mod
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .25 .20 .31
 @headitem Name             @tab Arguments          @tab Return type    @tab Standard
 @item @code{MOD(A,P)}  @tab @code{INTEGER A,P} @tab @code{INTEGER} @tab Fortran 77 and later
 @item @code{AMOD(A,P)} @tab @code{REAL(4) A,P} @tab @code{REAL(4)} @tab Fortran 77 and later
@@ -11013,7 +10966,7 @@ same kind as @var{FROM}.
 @end multitable
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{MVBITS(A)}   @tab @code{INTEGER A}    @tab @code{INTEGER}    @tab Fortran 90 and later
 @item @code{BMVBITS(A)} @tab @code{INTEGER(1) A} @tab @code{INTEGER(1)} @tab GNU extension
@@ -11163,7 +11116,7 @@ end program test_nint
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument           @tab Return Type     @tab Standard
 @item @code{NINT(A)}   @tab @code{REAL(4) A}   @tab  @code{INTEGER} @tab Fortran 77 and later
 @item @code{IDNINT(A)} @tab @code{REAL(8) A}   @tab  @code{INTEGER} @tab Fortran 77 and later
@@ -11261,7 +11214,7 @@ The return type is @code{INTEGER}, of the same kind as the
 argument.
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument            @tab Return type       @tab Standard
 @item @code{NOT(A)}   @tab @code{INTEGER A}    @tab @code{INTEGER}    @tab Fortran 95 and later
 @item @code{BNOT(A)} @tab @code{INTEGER(1) A} @tab @code{INTEGER(1)} @tab GNU extension
@@ -12277,14 +12230,14 @@ end program test_real
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument           @tab Return type     @tab Standard
-@item @code{FLOAT(A)}  @tab @code{INTEGER(4)}  @tab @code{REAL(4)}  @tab GNU extension
+@item @code{FLOAT(A)}  @tab @code{INTEGER(4)}  @tab @code{REAL(4)}  @tab Fortran 77 and later
 @item @code{DFLOAT(A)} @tab @code{INTEGER(4)}  @tab @code{REAL(8)}  @tab GNU extension
-@item @code{FLOATI(A)} @tab @code{INTEGER(2)}  @tab @code{REAL(4)}  @tab GNU extension
-@item @code{FLOATJ(A)} @tab @code{INTEGER(4)}  @tab @code{REAL(4)}  @tab GNU extension
-@item @code{FLOATK(A)} @tab @code{INTEGER(8)}  @tab @code{REAL(4)}  @tab GNU extension
-@item @code{SNGL(A)}   @tab @code{INTEGER(8)}  @tab @code{REAL(4)}  @tab GNU extension
+@item @code{FLOATI(A)} @tab @code{INTEGER(2)}  @tab @code{REAL(4)}  @tab GNU extension (-fdec)
+@item @code{FLOATJ(A)} @tab @code{INTEGER(4)}  @tab @code{REAL(4)}  @tab GNU extension (-fdec)
+@item @code{FLOATK(A)} @tab @code{INTEGER(8)}  @tab @code{REAL(4)}  @tab GNU extension (-fdec)
+@item @code{SNGL(A)}   @tab @code{REAL(8)}     @tab @code{REAL(4)}  @tab Fortran 77 and later
 @end multitable
 
 
@@ -13164,7 +13117,7 @@ Elemental function
 
 @item @emph{Return value}:
 The kind of the return value is that of @var{A} and @var{B}.
-If @math{B\ge 0} then the result is @code{ABS(A)}, else
+If @math{B \ge 0} then the result is @code{ABS(A)}, else
 it is @code{-ABS(A)}.
 
 @item @emph{Example}:
@@ -13181,7 +13134,7 @@ end program test_sign
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .26 .20 .30
 @headitem Name              @tab Arguments              @tab Return type       @tab Standard
 @item @code{SIGN(A,B)}  @tab @code{REAL(4) A, B}    @tab @code{REAL(4)}    @tab Fortran 77 and later
 @item @code{ISIGN(A,B)} @tab @code{INTEGER(4) A, B} @tab @code{INTEGER(4)} @tab Fortran 77 and later
@@ -13290,7 +13243,7 @@ end program test_sin
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument             @tab Return type       @tab Standard
 @item @code{SIN(X)}   @tab @code{REAL(4) X}     @tab @code{REAL(4)}    @tab Fortran 77 and later
 @item @code{DSIN(X)}  @tab @code{REAL(8) X}     @tab @code{REAL(8)}    @tab Fortran 77 and later
@@ -13352,7 +13305,7 @@ end program test_sind
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument             @tab Return type       @tab Standard
 @item @code{SIND(X)}   @tab @code{REAL(4) X}     @tab @code{REAL(4)}    @tab GNU extension
 @item @code{DSIND(X)}  @tab @code{REAL(8) X}     @tab @code{REAL(8)}    @tab GNU extension
@@ -13409,7 +13362,7 @@ end program test_sinh
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{DSINH(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab Fortran 90 and later
 @end multitable
@@ -13694,7 +13647,7 @@ end program test_sqrt
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name             @tab Argument             @tab Return type          @tab Standard
 @item @code{SQRT(X)}   @tab @code{REAL(4) X}     @tab @code{REAL(4)}       @tab Fortran 77 and later
 @item @code{DSQRT(X)}  @tab @code{REAL(8) X}     @tab @code{REAL(8)}       @tab Fortran 77 and later
@@ -14145,7 +14098,7 @@ end program test_tan
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type     @tab Standard
 @item @code{TAN(X)}   @tab @code{REAL(4) X}  @tab @code{REAL(4)}  @tab Fortran 77 and later
 @item @code{DTAN(X)}  @tab @code{REAL(8) X}  @tab @code{REAL(8)}  @tab Fortran 77 and later
@@ -14200,7 +14153,7 @@ end program test_tand
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type     @tab Standard
 @item @code{TAND(X)}   @tab @code{REAL(4) X}  @tab @code{REAL(4)}  @tab GNU extension
 @item @code{DTAND(X)}  @tab @code{REAL(8) X}  @tab @code{REAL(8)}  @tab GNU extension
@@ -14256,7 +14209,7 @@ end program test_tanh
 @end smallexample
 
 @item @emph{Specific names}:
-@multitable @columnfractions .20 .20 .20 .25
+@multitable @columnfractions .20 .23 .20 .33
 @headitem Name            @tab Argument          @tab Return type       @tab Standard
 @item @code{TANH(X)}  @tab @code{REAL(4) X}  @tab @code{REAL(4)}    @tab Fortran 77 and later
 @item @code{DTANH(X)} @tab @code{REAL(8) X}  @tab @code{REAL(8)}    @tab Fortran 77 and later
@@ -15180,7 +15133,7 @@ C compiler: @code{C_INT128_T, C_INT_LEAST128_T, C_INT_FAST128_T}.
 Furthermore, if @code{_Float128} is supported in C, the named constants
 @code{C_FLOAT128} and @code{C_FLOAT128_COMPLEX} are defined.
 
-@multitable @columnfractions .15 .35 .35 .35
+@multitable @columnfractions .19 .32 .34 .15
 @headitem Fortran Type  @tab Named constant         @tab C type                                @tab Extension
 @item @code{INTEGER}@tab @code{C_INT}           @tab @code{int}
 @item @code{INTEGER}@tab @code{C_SHORT}         @tab @code{short int}
diff --git a/gcc/fortran/invoke.texi b/gcc/fortran/invoke.texi
index 5b471ac..4635831 100644
--- a/gcc/fortran/invoke.texi
+++ b/gcc/fortran/invoke.texi
@@ -195,7 +195,7 @@ form is determined by the file extension.
 @item -fall-intrinsics
 @opindex @code{fall-intrinsics}
 This option causes all intrinsic procedures (including the GNU-specific
-extensions) to be accepted.  This can be useful with @option{-std=f95} to
+extensions) to be accepted.  This can be useful with @option{-std=} to
 force standard-compliance but get access to the full range of intrinsics
 available with @command{gfortran}.  As a consequence, @option{-Wintrinsics-std}
 will be ignored and no user-defined procedure with the same name as any
@@ -365,7 +365,7 @@ lines in the source file. The default value is 132.
 @item -fmax-identifier-length=@var{n}
 @opindex @code{fmax-identifier-length=}@var{n}
 Specify the maximum allowed identifier length. Typical values are
-31 (Fortran 95) and 63 (Fortran 2003 and Fortran 2008).
+31 (Fortran 95) and 63 (Fortran 2003 and later).
 
 @item -fimplicit-none
 @opindex @code{fimplicit-none}
@@ -538,10 +538,46 @@ Enhance test coverage by forcing most forall assignments to use temporary.
 @cindex preprocessor
 @cindex options, preprocessor
 @cindex CPP
-
-Preprocessor related options. See section 
-@ref{Preprocessing and conditional compilation} for more detailed
-information on preprocessing in @command{gfortran}.
+@cindex FPP
+@cindex Conditional compilation
+@cindex Preprocessing
+@cindex preprocessor, include file handling
+
+Many Fortran compilers including GNU Fortran allow passing the source code
+through a C preprocessor (CPP; sometimes also called the Fortran preprocessor,
+FPP) to allow for conditional compilation.  In the case of GNU Fortran,
+this is the GNU C Preprocessor in the traditional mode.  On systems with
+case-preserving file names, the preprocessor is automatically invoked if the
+filename extension is @file{.F}, @file{.FOR}, @file{.FTN}, @file{.fpp},
+@file{.FPP}, @file{.F90}, @file{.F95}, @file{.F03} or @file{.F08}.  To manually
+invoke the preprocessor on any file, use @option{-cpp}, to disable
+preprocessing on files where the preprocessor is run automatically, use
+@option{-nocpp}.
+
+If a preprocessed file includes another file with the Fortran @code{INCLUDE}
+statement, the included file is not preprocessed.  To preprocess included
+files, use the equivalent preprocessor statement @code{#include}.
+
+If GNU Fortran invokes the preprocessor, @code{__GFORTRAN__}
+is defined.  The macros @code{__GNUC__}, @code{__GNUC_MINOR__} and
+@code{__GNUC_PATCHLEVEL__} can be used to determine the version of the
+compiler.  See @ref{Top,,Overview,cpp,The C Preprocessor} for details.
+
+GNU Fortran supports a number of @code{INTEGER} and @code{REAL} kind types
+in additional to the kind types required by the Fortran standard.
+The availability of any given kind type is architecture dependent.  The
+following pre-defined preprocessor macros can be used to conditionally
+include code for these additional kind types: @code{__GFC_INT_1__},
+@code{__GFC_INT_2__}, @code{__GFC_INT_8__}, @code{__GFC_INT_16__},
+@code{__GFC_REAL_10__}, and @code{__GFC_REAL_16__}.
+
+While CPP is the de-facto standard for preprocessing Fortran code,
+Part 3 of the Fortran 95 standard (ISO/IEC 1539-3:1998) defines
+Conditional Compilation, which is not widely used and not directly
+supported by the GNU Fortran compiler.  You can use the program coco
+to preprocess such files (@uref{http://www.daniellnagle.com/coco.html}).
+
+The following options control preprocessing of Fortran code:
 
 @table @gcctabopt
 @item -cpp
diff --git a/gcc/fortran/iresolve.c b/gcc/fortran/iresolve.c
index e17fe45f..598c040 100644
--- a/gcc/fortran/iresolve.c
+++ b/gcc/fortran/iresolve.c
@@ -1276,27 +1276,16 @@ gfc_resolve_ior (gfc_expr *f, gfc_expr *i, gfc_expr *j)
 
 
 void
-gfc_resolve_index_func (gfc_expr *f, gfc_actual_arglist *a)
+gfc_resolve_index_func (gfc_expr *f, gfc_expr *str,
+			gfc_expr *sub_str ATTRIBUTE_UNUSED, gfc_expr *back,
+			gfc_expr *kind)
 {
   gfc_typespec ts;
   gfc_clear_ts (&ts);
-  gfc_expr *str, *back, *kind;
-  gfc_actual_arglist *a_sub_str, *a_back, *a_kind;
-
-  if (f->do_not_resolve_again)
-    return;
-
-  a_sub_str = a->next;
-  a_back = a_sub_str->next;
-  a_kind = a_back->next;
-
-  str = a->expr;
-  back = a_back->expr;
-  kind = a_kind->expr;
 
   f->ts.type = BT_INTEGER;
   if (kind)
-    f->ts.kind = mpz_get_si ((kind)->value.integer);
+    f->ts.kind = mpz_get_si (kind->value.integer);
   else
     f->ts.kind = gfc_default_integer_kind;
 
@@ -1311,8 +1300,6 @@ gfc_resolve_index_func (gfc_expr *f, gfc_actual_arglist *a)
 
   f->value.function.name
     = gfc_get_string ("__index_%d_i%d", str->ts.kind, f->ts.kind);
-
-  f->do_not_resolve_again = 1;
 }
 
 
diff --git a/gcc/fortran/match.c b/gcc/fortran/match.c
index 53a575e..2bf2143 100644
--- a/gcc/fortran/match.c
+++ b/gcc/fortran/match.c
@@ -530,32 +530,6 @@ gfc_match_small_int (int *value)
 }
 
 
-/* This function is the same as the gfc_match_small_int, except that
-   we're keeping the pointer to the expr.  This function could just be
-   removed and the previously mentioned one modified, though all calls
-   to it would have to be modified then (and there were a number of
-   them).  Return MATCH_ERROR if fail to extract the int; otherwise,
-   return the result of gfc_match_expr().  The expr (if any) that was
-   matched is returned in the parameter expr.  */
-
-match
-gfc_match_small_int_expr (int *value, gfc_expr **expr)
-{
-  match m;
-  int i;
-
-  m = gfc_match_expr (expr);
-  if (m != MATCH_YES)
-    return m;
-
-  if (gfc_extract_int (*expr, &i, 1))
-    m = MATCH_ERROR;
-
-  *value = i;
-  return m;
-}
-
-
 /* Matches a statement label.  Uses gfc_match_small_literal_int() to
    do most of the work.  */
 
@@ -599,7 +573,7 @@ cleanup:
    it.  We also make sure the symbol does not refer to another
    (active) block.  A matched label is pointed to by gfc_new_block.  */
 
-match
+static match
 gfc_match_label (void)
 {
   char name[GFC_MAX_SYMBOL_LEN + 1];
@@ -5314,6 +5288,13 @@ gfc_match_common (void)
 		  goto cleanup;
 		}
 
+	      if (as->corank)
+		{
+		  gfc_error ("Symbol %qs in COMMON at %C cannot be a "
+			     "coarray", sym->name);
+		  goto cleanup;
+		}
+
 	      if (!gfc_add_dimension (&sym->attr, sym->name, NULL))
 		goto cleanup;
 
diff --git a/gcc/fortran/match.h b/gcc/fortran/match.h
index 21e94f7..e9368db 100644
--- a/gcc/fortran/match.h
+++ b/gcc/fortran/match.h
@@ -47,11 +47,8 @@ match gfc_match_space (void);
 match gfc_match_eos (void);
 match gfc_match_small_literal_int (int *, int *);
 match gfc_match_st_label (gfc_st_label **);
-match gfc_match_label (void);
 match gfc_match_small_int (int *);
-match gfc_match_small_int_expr (int *, gfc_expr **);
 match gfc_match_name (char *);
-match gfc_match_name_C (const char **buffer);
 match gfc_match_symbol (gfc_symbol **, int);
 match gfc_match_sym_tree (gfc_symtree **, int);
 match gfc_match_intrinsic_op (gfc_intrinsic_op *);
@@ -237,7 +234,6 @@ match gfc_match_omp_end_single (void);
 match gfc_match_data (void);
 match gfc_match_null (gfc_expr **);
 match gfc_match_kind_spec (gfc_typespec *, bool);
-match gfc_match_old_kind_spec (gfc_typespec *);
 match gfc_match_decl_type_spec (gfc_typespec *, int);
 
 match gfc_match_end (gfc_statement *);
@@ -296,14 +292,8 @@ match gfc_match_volatile (void);
 
 /* Fortran 2003 c interop.
    TODO: some of these should be moved to another file rather than decl.c */
-void set_com_block_bind_c (gfc_common_head *, int);
-bool set_verify_bind_c_sym (gfc_symbol *, int);
-bool set_verify_bind_c_com_block (gfc_common_head *, int);
-bool get_bind_c_idents (void);
 match gfc_match_bind_c_stmt (void);
-match gfc_match_suffix (gfc_symbol *, gfc_symbol **);
 match gfc_match_bind_c (gfc_symbol *, bool);
-match gfc_get_type_attr_spec (symbol_attribute *, char*);
 
 /* primary.c.  */
 match gfc_match_structure_constructor (gfc_symbol *, gfc_expr **);
diff --git a/gcc/fortran/misc.c b/gcc/fortran/misc.c
index e6402e8..a553e1e 100644
--- a/gcc/fortran/misc.c
+++ b/gcc/fortran/misc.c
@@ -284,7 +284,7 @@ gfc_dummy_typename (gfc_typespec *ts)
 	{
 	  if (ts->kind == gfc_default_character_kind)
 	    sprintf(buffer, "CHARACTER(*)");
-	  else if (ts->kind < 10)
+	  else if (ts->kind >= 0 && ts->kind < 10)
 	    sprintf(buffer, "CHARACTER(*,%d)", ts->kind);
 	  else
 	    sprintf(buffer, "CHARACTER(*,?)");
diff --git a/gcc/fortran/openmp.c b/gcc/fortran/openmp.c
index dcf22ac..d120be8 100644
--- a/gcc/fortran/openmp.c
+++ b/gcc/fortran/openmp.c
@@ -85,7 +85,8 @@ gfc_free_omp_clauses (gfc_omp_clauses *c)
   gfc_free_expr (c->chunk_size);
   gfc_free_expr (c->safelen_expr);
   gfc_free_expr (c->simdlen_expr);
-  gfc_free_expr (c->num_teams);
+  gfc_free_expr (c->num_teams_lower);
+  gfc_free_expr (c->num_teams_upper);
   gfc_free_expr (c->device);
   gfc_free_expr (c->thread_limit);
   gfc_free_expr (c->dist_chunk_size);
@@ -2420,11 +2421,22 @@ gfc_match_omp_clauses (gfc_omp_clauses **cp, const omp_mask mask,
 	      continue;
 	    }
 	  if ((mask & OMP_CLAUSE_NUM_TEAMS)
-	      && (m = gfc_match_dupl_check (!c->num_teams, "num_teams", true,
-					    &c->num_teams)) != MATCH_NO)
+	      && (m = gfc_match_dupl_check (!c->num_teams_upper, "num_teams",
+					    true)) != MATCH_NO)
 	    {
 	      if (m == MATCH_ERROR)
 		goto error;
+	      if (gfc_match ("%e ", &c->num_teams_upper) != MATCH_YES)
+		goto error;
+	      if (gfc_peek_ascii_char () == ':')
+		{
+		  c->num_teams_lower = c->num_teams_upper;
+		  c->num_teams_upper = NULL;
+		  if (gfc_match (": %e ", &c->num_teams_upper) != MATCH_YES)
+		    goto error;
+		}
+	      if (gfc_match (") ") != MATCH_YES)
+		goto error;
 	      continue;
 	    }
 	  if ((mask & OMP_CLAUSE_NUM_THREADS)
@@ -3551,7 +3563,8 @@ cleanup:
   (omp_mask (OMP_CLAUSE_DEVICE) | OMP_CLAUSE_MAP | OMP_CLAUSE_IF	\
    | OMP_CLAUSE_DEPEND | OMP_CLAUSE_NOWAIT | OMP_CLAUSE_PRIVATE		\
    | OMP_CLAUSE_FIRSTPRIVATE | OMP_CLAUSE_DEFAULTMAP			\
-   | OMP_CLAUSE_IS_DEVICE_PTR | OMP_CLAUSE_IN_REDUCTION)
+   | OMP_CLAUSE_IS_DEVICE_PTR | OMP_CLAUSE_IN_REDUCTION			\
+   | OMP_CLAUSE_THREAD_LIMIT)
 #define OMP_TARGET_DATA_CLAUSES \
   (omp_mask (OMP_CLAUSE_DEVICE) | OMP_CLAUSE_MAP | OMP_CLAUSE_IF	\
    | OMP_CLAUSE_USE_DEVICE_PTR | OMP_CLAUSE_USE_DEVICE_ADDR)
@@ -6220,6 +6233,7 @@ resolve_omp_clauses (gfc_code *code, gfc_omp_clauses *omp_clauses,
 
 	    case EXEC_OMP_PARALLEL:
 	    case EXEC_OMP_PARALLEL_DO:
+	    case EXEC_OMP_PARALLEL_LOOP:
 	    case EXEC_OMP_PARALLEL_MASKED:
 	    case EXEC_OMP_PARALLEL_MASTER:
 	    case EXEC_OMP_PARALLEL_SECTIONS:
@@ -6273,6 +6287,7 @@ resolve_omp_clauses (gfc_code *code, gfc_omp_clauses *omp_clauses,
 	    case EXEC_OMP_TARGET:
 	    case EXEC_OMP_TARGET_TEAMS:
 	    case EXEC_OMP_TARGET_TEAMS_DISTRIBUTE:
+	    case EXEC_OMP_TARGET_TEAMS_LOOP:
 	      ok = ifc == OMP_IF_TARGET;
 	      break;
 
@@ -6300,6 +6315,7 @@ resolve_omp_clauses (gfc_code *code, gfc_omp_clauses *omp_clauses,
 	    case EXEC_OMP_TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO:
 	    case EXEC_OMP_TARGET_PARALLEL:
 	    case EXEC_OMP_TARGET_PARALLEL_DO:
+	    case EXEC_OMP_TARGET_PARALLEL_LOOP:
 	      ok = ifc == OMP_IF_TARGET || ifc == OMP_IF_PARALLEL;
 	      break;
 
@@ -7293,8 +7309,18 @@ resolve_omp_clauses (gfc_code *code, gfc_omp_clauses *omp_clauses,
     resolve_positive_int_expr (omp_clauses->safelen_expr, "SAFELEN");
   if (omp_clauses->simdlen_expr)
     resolve_positive_int_expr (omp_clauses->simdlen_expr, "SIMDLEN");
-  if (omp_clauses->num_teams)
-    resolve_positive_int_expr (omp_clauses->num_teams, "NUM_TEAMS");
+  if (omp_clauses->num_teams_lower)
+    resolve_positive_int_expr (omp_clauses->num_teams_lower, "NUM_TEAMS");
+  if (omp_clauses->num_teams_upper)
+    resolve_positive_int_expr (omp_clauses->num_teams_upper, "NUM_TEAMS");
+  if (omp_clauses->num_teams_lower
+      && omp_clauses->num_teams_lower->expr_type == EXPR_CONSTANT
+      && omp_clauses->num_teams_upper->expr_type == EXPR_CONSTANT
+      && mpz_cmp (omp_clauses->num_teams_lower->value.integer,
+		  omp_clauses->num_teams_upper->value.integer) > 0)
+    gfc_warning (0, "NUM_TEAMS lower bound at %L larger than upper bound at %L",
+		 &omp_clauses->num_teams_lower->where,
+		 &omp_clauses->num_teams_upper->where);
   if (omp_clauses->device)
     resolve_nonnegative_int_expr (omp_clauses->device, "DEVICE");
   if (omp_clauses->filter)
diff --git a/gcc/fortran/parse.c b/gcc/fortran/parse.c
index b1e73ee..94b677f 100644
--- a/gcc/fortran/parse.c
+++ b/gcc/fortran/parse.c
@@ -924,6 +924,7 @@ decode_omp_directive (void)
       matcho ("end distribute", gfc_match_omp_eos_error, ST_OMP_END_DISTRIBUTE);
       matchs ("end do simd", gfc_match_omp_end_nowait, ST_OMP_END_DO_SIMD);
       matcho ("end do", gfc_match_omp_end_nowait, ST_OMP_END_DO);
+      matcho ("end loop", gfc_match_omp_eos_error, ST_OMP_END_LOOP);
       matchs ("end simd", gfc_match_omp_eos_error, ST_OMP_END_SIMD);
       matcho ("end masked taskloop simd", gfc_match_omp_eos_error,
 	      ST_OMP_END_MASKED_TASKLOOP_SIMD);
@@ -939,6 +940,8 @@ decode_omp_directive (void)
       matchs ("end parallel do simd", gfc_match_omp_eos_error,
 	      ST_OMP_END_PARALLEL_DO_SIMD);
       matcho ("end parallel do", gfc_match_omp_eos_error, ST_OMP_END_PARALLEL_DO);
+      matcho ("end parallel loop", gfc_match_omp_eos_error,
+	      ST_OMP_END_PARALLEL_LOOP);
       matcho ("end parallel masked taskloop simd", gfc_match_omp_eos_error,
 	      ST_OMP_END_PARALLEL_MASKED_TASKLOOP_SIMD);
       matcho ("end parallel masked taskloop", gfc_match_omp_eos_error,
@@ -960,24 +963,29 @@ decode_omp_directive (void)
       matcho ("end sections", gfc_match_omp_end_nowait, ST_OMP_END_SECTIONS);
       matcho ("end single", gfc_match_omp_end_single, ST_OMP_END_SINGLE);
       matcho ("end target data", gfc_match_omp_eos_error, ST_OMP_END_TARGET_DATA);
-      matchs ("end target parallel do simd", gfc_match_omp_eos_error,
+      matchs ("end target parallel do simd", gfc_match_omp_end_nowait,
 	      ST_OMP_END_TARGET_PARALLEL_DO_SIMD);
-      matcho ("end target parallel do", gfc_match_omp_eos_error,
+      matcho ("end target parallel do", gfc_match_omp_end_nowait,
 	      ST_OMP_END_TARGET_PARALLEL_DO);
-      matcho ("end target parallel", gfc_match_omp_eos_error,
+      matcho ("end target parallel loop", gfc_match_omp_end_nowait,
+	      ST_OMP_END_TARGET_PARALLEL_LOOP);
+      matcho ("end target parallel", gfc_match_omp_end_nowait,
 	      ST_OMP_END_TARGET_PARALLEL);
-      matchs ("end target simd", gfc_match_omp_eos_error, ST_OMP_END_TARGET_SIMD);
+      matchs ("end target simd", gfc_match_omp_end_nowait, ST_OMP_END_TARGET_SIMD);
       matchs ("end target teams distribute parallel do simd",
-	      gfc_match_omp_eos_error,
+	      gfc_match_omp_end_nowait,
 	      ST_OMP_END_TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO_SIMD);
-      matcho ("end target teams distribute parallel do", gfc_match_omp_eos_error,
+      matcho ("end target teams distribute parallel do", gfc_match_omp_end_nowait,
 	      ST_OMP_END_TARGET_TEAMS_DISTRIBUTE_PARALLEL_DO);
-      matchs ("end target teams distribute simd", gfc_match_omp_eos_error,
+      matchs ("end target teams distribute simd", gfc_match_omp_end_nowait,
 	      ST_OMP_END_TARGET_TEAMS_DISTRIBUTE_SIMD);
-      matcho ("end target teams distribute", gfc_match_omp_eos_error,
+      matcho ("end target teams distribute", gfc_match_omp_end_nowait,
 	      ST_OMP_END_TARGET_TEAMS_DISTRIBUTE);
-      matcho ("end target teams", gfc_match_omp_eos_error, ST_OMP_END_TARGET_TEAMS);
-      matcho ("end target", gfc_match_omp_eos_error, ST_OMP_END_TARGET);
+      matcho ("end target teams loop", gfc_match_omp_end_nowait,
+	      ST_OMP_END_TARGET_TEAMS_LOOP);
+      matcho ("end target teams", gfc_match_omp_end_nowait,
+	      ST_OMP_END_TARGET_TEAMS);
+      matcho ("end target", gfc_match_omp_end_nowait, ST_OMP_END_TARGET);
       matcho ("end taskgroup", gfc_match_omp_eos_error, ST_OMP_END_TASKGROUP);
       matchs ("end taskloop simd", gfc_match_omp_eos_error,
 	      ST_OMP_END_TASKLOOP_SIMD);
@@ -991,6 +999,7 @@ decode_omp_directive (void)
 	      ST_OMP_END_TEAMS_DISTRIBUTE_SIMD);
       matcho ("end teams distribute", gfc_match_omp_eos_error,
 	      ST_OMP_END_TEAMS_DISTRIBUTE);
+      matcho ("end teams loop", gfc_match_omp_eos_error, ST_OMP_END_TEAMS_LOOP);
       matcho ("end teams", gfc_match_omp_eos_error, ST_OMP_END_TEAMS);
       matcho ("end workshare", gfc_match_omp_end_nowait,
 	      ST_OMP_END_WORKSHARE);
@@ -2553,7 +2562,7 @@ gfc_ascii_statement (gfc_statement st)
       p = "!$OMP END TEAMS DISTRIBUTE SIMD";
       break;
     case ST_OMP_END_TEAMS_LOOP:
-      p = "!$OMP END TEAMS LOP";
+      p = "!$OMP END TEAMS LOOP";
       break;
     case ST_OMP_END_WORKSHARE:
       p = "!$OMP END WORKSHARE";
@@ -6569,7 +6578,7 @@ resolve_all_program_units (gfc_namespace *gfc_global_ns_list)
 
 
 static void
-clean_up_modules (gfc_gsymbol *gsym)
+clean_up_modules (gfc_gsymbol *&gsym)
 {
   if (gsym == NULL)
     return;
@@ -6577,14 +6586,18 @@ clean_up_modules (gfc_gsymbol *gsym)
   clean_up_modules (gsym->left);
   clean_up_modules (gsym->right);
 
-  if (gsym->type != GSYM_MODULE || !gsym->ns)
+  if (gsym->type != GSYM_MODULE)
     return;
 
-  gfc_current_ns = gsym->ns;
-  gfc_derived_types = gfc_current_ns->derived_types;
-  gfc_done_2 ();
-  gsym->ns = NULL;
-  return;
+  if (gsym->ns)
+    {
+      gfc_current_ns = gsym->ns;
+      gfc_derived_types = gfc_current_ns->derived_types;
+      gfc_done_2 ();
+      gsym->ns = NULL;
+    }
+  free (gsym);
+  gsym = NULL;
 }
 
 
diff --git a/gcc/fortran/resolve.c b/gcc/fortran/resolve.c
index 5ccd907..705d232 100644
--- a/gcc/fortran/resolve.c
+++ b/gcc/fortran/resolve.c
@@ -1463,8 +1463,15 @@ resolve_structure_cons (gfc_expr *expr, int init)
 	  mpz_init (len);
 	  for (int n = 0; n < rank; n++)
 	    {
-	      gcc_assert (comp->as->upper[n]->expr_type == EXPR_CONSTANT
-			  && comp->as->lower[n]->expr_type == EXPR_CONSTANT);
+	      if (comp->as->upper[n]->expr_type != EXPR_CONSTANT
+		  || comp->as->lower[n]->expr_type != EXPR_CONSTANT)
+		{
+		  gfc_error ("Bad array spec of component %qs referenced in "
+			     "structure constructor at %L",
+			     comp->name, &cons->expr->where);
+		  t = false;
+		  break;
+		};
 	      mpz_set_ui (len, 1);
 	      mpz_add (len, len, comp->as->upper[n]->value.integer);
 	      mpz_sub (len, len, comp->as->lower[n]->value.integer);
@@ -2967,6 +2974,19 @@ resolve_unknown_f (gfc_expr *expr)
       return false;
     }
 
+  /* IMPLICIT NONE (external) procedures require an explicit EXTERNAL attr.  */
+  /* Intrinsics were handled above, only non-intrinsics left here.  */
+  if (sym->attr.flavor == FL_PROCEDURE
+      && sym->attr.implicit_type
+      && sym->ns
+      && sym->ns->has_implicit_none_export)
+    {
+	  gfc_error ("Missing explicit declaration with EXTERNAL attribute "
+	      "for symbol %qs at %L", sym->name, &sym->declared_at);
+	  sym->error = 1;
+	  return false;
+    }
+
   /* The reference is to an external name.  */
 
   sym->attr.proc = PROC_EXTERNAL;
@@ -4044,7 +4064,7 @@ resolve_operator (gfc_expr *e)
     {
     default:
       if (!gfc_resolve_expr (e->value.op.op2))
-	return false;
+	t = false;
 
     /* Fall through.  */
 
@@ -4071,6 +4091,9 @@ resolve_operator (gfc_expr *e)
   op2 = e->value.op.op2;
   if (op1 == NULL && op2 == NULL)
     return false;
+  /* Error out if op2 did not resolve. We already diagnosed op1.  */
+  if (t == false)
+    return false;
 
   dual_locus_error = false;
 
@@ -8763,11 +8786,11 @@ resolve_select (gfc_code *code, bool select_type)
 
 	      if (cp->low != NULL
 		  && case_expr->ts.kind != gfc_kind_max(case_expr, cp->low))
-		gfc_convert_type_warn (case_expr, &cp->low->ts, 2, 0);
+		gfc_convert_type_warn (case_expr, &cp->low->ts, 1, 0);
 
 	      if (cp->high != NULL
 		  && case_expr->ts.kind != gfc_kind_max(case_expr, cp->high))
-		gfc_convert_type_warn (case_expr, &cp->high->ts, 2, 0);
+		gfc_convert_type_warn (case_expr, &cp->high->ts, 1, 0);
 	    }
 	 }
     }
@@ -13172,7 +13195,7 @@ static bool
 resolve_fl_procedure (gfc_symbol *sym, int mp_flag)
 {
   gfc_formal_arglist *arg;
-  bool allocatable_or_pointer;
+  bool allocatable_or_pointer = false;
 
   if (sym->attr.function
       && !resolve_fl_var_and_proc (sym, mp_flag))
diff --git a/gcc/fortran/scanner.c b/gcc/fortran/scanner.c
index 5a45069..69b81ab 100644
--- a/gcc/fortran/scanner.c
+++ b/gcc/fortran/scanner.c
@@ -78,8 +78,8 @@ static struct gfc_file_change
   gfc_linebuf *lb;
   int line;
 } *file_changes;
-size_t file_changes_cur, file_changes_count;
-size_t file_changes_allocated;
+static size_t file_changes_cur, file_changes_count;
+static size_t file_changes_allocated;
 
 static gfc_char_t *last_error_char;
 
diff --git a/gcc/fortran/simplify.c b/gcc/fortran/simplify.c
index d675f2c..6a6b3fb 100644
--- a/gcc/fortran/simplify.c
+++ b/gcc/fortran/simplify.c
@@ -2109,6 +2109,9 @@ gfc_simplify_cshift (gfc_expr *array, gfc_expr *shift, gfc_expr *dim)
   else
     which = 0;
 
+  if (array->shape == NULL)
+    return NULL;
+
   gfc_array_size (array, &size);
   arraysize = mpz_get_ui (size);
   mpz_clear (size);
@@ -8174,6 +8177,9 @@ gfc_simplify_transpose (gfc_expr *matrix)
 
   gcc_assert (matrix->rank == 2);
 
+  if (matrix->shape == NULL)
+    return NULL;
+
   result = gfc_get_array_expr (matrix->ts.type, matrix->ts.kind,
 			       &matrix->where);
   result->rank = 2;
diff --git a/gcc/fortran/symbol.c b/gcc/fortran/symbol.c
index 2c4acd5..179f602 100644
--- a/gcc/fortran/symbol.c
+++ b/gcc/fortran/symbol.c
@@ -720,6 +720,7 @@ gfc_check_conflict (symbol_attribute *attr, const char *name, locus *where)
   conf (pdt_len, pointer)
   conf (pdt_len, dimension)
   conf (pdt_len, codimension)
+  conf (pdt_len, pdt_kind)
 
   if (attr->access == ACCESS_PRIVATE)
     {
@@ -2609,7 +2610,7 @@ free_components (gfc_component *p)
       if (p->param_list)
 	gfc_free_actual_arglist (p->param_list);
       free (p->tb);
-
+      p->tb = NULL;
       free (p);
     }
 }
@@ -3066,7 +3067,7 @@ set_symbol_common_block (gfc_symbol *sym, gfc_common_head *common_block)
 /* Remove a gfc_symbol structure and everything it points to.  */
 
 void
-gfc_free_symbol (gfc_symbol *sym)
+gfc_free_symbol (gfc_symbol *&sym)
 {
 
   if (sym == NULL)
@@ -3096,13 +3097,14 @@ gfc_free_symbol (gfc_symbol *sym)
     gfc_free_actual_arglist (sym->param_list);
 
   free (sym);
+  sym = NULL;
 }
 
 
 /* Decrease the reference counter and free memory when we reach zero.  */
 
 void
-gfc_release_symbol (gfc_symbol *sym)
+gfc_release_symbol (gfc_symbol *&sym)
 {
   if (sym == NULL)
     return;
@@ -3829,9 +3831,9 @@ free_tb_tree (gfc_symtree *t)
   free_tb_tree (t->left);
   free_tb_tree (t->right);
 
-  /* TODO: Free type-bound procedure structs themselves; probably needs some
-     sort of ref-counting mechanism.  */
-
+  /* TODO: Free type-bound procedure u.generic  */
+  free (t->n.tb);
+  t->n.tb = NULL;
   free (t);
 }
 
@@ -3984,7 +3986,7 @@ gfc_new_charlen (gfc_namespace *ns, gfc_charlen *old_cl)
 /* Free the charlen list from cl to end (end is not freed).
    Free the whole list if end is NULL.  */
 
-void
+static void
 gfc_free_charlen (gfc_charlen *cl, gfc_charlen *end)
 {
   gfc_charlen *cl2;
@@ -4021,7 +4023,7 @@ free_entry_list (gfc_entry_list *el)
    taken care of when a specific name is freed.  */
 
 void
-gfc_free_namespace (gfc_namespace *ns)
+gfc_free_namespace (gfc_namespace *&ns)
 {
   gfc_namespace *p, *q;
   int i;
@@ -4072,6 +4074,7 @@ gfc_free_namespace (gfc_namespace *ns)
 
   p = ns->contained;
   free (ns);
+  ns = NULL;
 
   /* Recursively free any contained namespaces.  */
   while (p != NULL)
@@ -5105,23 +5108,6 @@ gfc_get_derived_super_type (gfc_symbol* derived)
 }
 
 
-/* Get the ultimate super-type of a given derived type.  */
-
-gfc_symbol*
-gfc_get_ultimate_derived_super_type (gfc_symbol* derived)
-{
-  if (!derived->attr.extension)
-    return NULL;
-
-  derived = gfc_get_derived_super_type (derived);
-
-  if (derived->attr.extension)
-    return gfc_get_ultimate_derived_super_type (derived);
-  else
-    return derived;
-}
-
-
 /* Check if a derived type t2 is an extension of (or equal to) a type t1.  */
 
 bool
diff --git a/gcc/fortran/trans-array.c b/gcc/fortran/trans-array.c
index bceb8b2..7932185 100644
--- a/gcc/fortran/trans-array.c
+++ b/gcc/fortran/trans-array.c
@@ -4161,7 +4161,7 @@ gfc_start_scalarized_body (gfc_loopinfo * loop, stmtblock_t * pbody)
 
 /* Generates the actual loop code for a scalarization loop.  */
 
-void
+static void
 gfc_trans_scalarized_loop_end (gfc_loopinfo * loop, int n,
 			       stmtblock_t * pbody)
 {
@@ -11460,6 +11460,59 @@ gfc_get_proc_ifc_for_expr (gfc_expr *procedure_ref)
 }
 
 
+/* Given an expression referring to an intrinsic function call,
+   return the intrinsic symbol.  */
+
+gfc_intrinsic_sym *
+gfc_get_intrinsic_for_expr (gfc_expr *call)
+{
+  if (call == NULL)
+    return NULL;
+
+  /* Normal procedure case.  */
+  if (call->expr_type == EXPR_FUNCTION)
+    return call->value.function.isym;
+  else
+    return NULL;
+}
+
+
+/* Indicates whether an argument to an intrinsic function should be used in
+   scalarization.  It is usually the case, except for some intrinsics
+   requiring the value to be constant, and using the value at compile time only.
+   As the value is not used at runtime in those cases, we don’t produce code
+   for it, and it should not be visible to the scalarizer.
+   FUNCTION is the intrinsic function being called, ACTUAL_ARG is the actual
+   argument being examined in that call, and ARG_NUM the index number
+   of ACTUAL_ARG in the list of arguments.
+   The intrinsic procedure’s dummy argument associated with ACTUAL_ARG is
+   identified using the name in ACTUAL_ARG if it is present (that is: if it’s
+   a keyword argument), otherwise using ARG_NUM.  */
+
+static bool
+arg_evaluated_for_scalarization (gfc_intrinsic_sym *function,
+				 gfc_actual_arglist &actual_arg, int arg_num)
+{
+  if (function != NULL)
+    {
+      switch (function->id)
+	{
+	  case GFC_ISYM_INDEX:
+	    if ((actual_arg.name == NULL && arg_num == 3)
+		|| (actual_arg.name != NULL
+		    && strcmp ("kind", actual_arg.name) == 0))
+	      return false;
+	  /* Fallthrough.  */
+
+	  default:
+	    break;
+	}
+    }
+
+  return true;
+}
+
+
 /* Walk the arguments of an elemental function.
    PROC_EXPR is used to check whether an argument is permitted to be absent.  If
    it is NULL, we don't do the check and the argument is assumed to be present.
@@ -11467,6 +11520,7 @@ gfc_get_proc_ifc_for_expr (gfc_expr *procedure_ref)
 
 gfc_ss *
 gfc_walk_elemental_function_args (gfc_ss * ss, gfc_actual_arglist *arg,
+				  gfc_intrinsic_sym *intrinsic_sym,
 				  gfc_symbol *proc_ifc, gfc_ss_type type)
 {
   gfc_formal_arglist *dummy_arg;
@@ -11483,10 +11537,13 @@ gfc_walk_elemental_function_args (gfc_ss * ss, gfc_actual_arglist *arg,
   else
     dummy_arg = NULL;
 
+  int arg_num = 0;
   scalar = 1;
   for (; arg; arg = arg->next)
     {
-      if (!arg->expr || arg->expr->expr_type == EXPR_NULL)
+      if (!arg->expr
+	  || arg->expr->expr_type == EXPR_NULL
+	  || !arg_evaluated_for_scalarization (intrinsic_sym, *arg, arg_num))
 	goto loop_continue;
 
       newss = gfc_walk_subexpr (head, arg->expr);
@@ -11519,6 +11576,7 @@ gfc_walk_elemental_function_args (gfc_ss * ss, gfc_actual_arglist *arg,
         }
 
 loop_continue:
+      arg_num++;
       if (dummy_arg != NULL)
 	dummy_arg = dummy_arg->next;
     }
@@ -11579,6 +11637,7 @@ gfc_walk_function_expr (gfc_ss * ss, gfc_expr * expr)
 
       ss = gfc_walk_elemental_function_args (old_ss,
 					     expr->value.function.actual,
+					     gfc_get_intrinsic_for_expr (expr),
 					     gfc_get_proc_ifc_for_expr (expr),
 					     GFC_SS_REFERENCE);
       if (ss != old_ss
diff --git a/gcc/fortran/trans-array.h b/gcc/fortran/trans-array.h
index 1d3dc48..8f806c3 100644
--- a/gcc/fortran/trans-array.h
+++ b/gcc/fortran/trans-array.h
@@ -76,6 +76,8 @@ void gfc_trans_static_array_pointer (gfc_symbol *);
 
 /* Get the procedure interface for a function call.  */
 gfc_symbol *gfc_get_proc_ifc_for_expr (gfc_expr *);
+/* Get the intrinsic symbol for an intrinsic function call.  */
+gfc_intrinsic_sym *gfc_get_intrinsic_for_expr (gfc_expr *);
 /* Generate scalarization information for an expression.  */
 gfc_ss *gfc_walk_expr (gfc_expr *);
 /* Workhorse for gfc_walk_expr.  */
@@ -84,6 +86,7 @@ gfc_ss *gfc_walk_subexpr (gfc_ss *, gfc_expr *);
 gfc_ss *gfc_walk_array_ref (gfc_ss *, gfc_expr *, gfc_ref * ref);
 /* Walk the arguments of an elemental function.  */
 gfc_ss *gfc_walk_elemental_function_args (gfc_ss *, gfc_actual_arglist *,
+					  gfc_intrinsic_sym *,
 					  gfc_symbol *, gfc_ss_type);
 /* Walk an intrinsic function.  */
 gfc_ss *gfc_walk_intrinsic_function (gfc_ss *, gfc_expr *,
@@ -118,8 +121,6 @@ void gfc_copy_loopinfo_to_se (gfc_se *, gfc_loopinfo *);
 
 /* Marks the start of a scalarized expression, and declares loop variables.  */
 void gfc_start_scalarized_body (gfc_loopinfo *, stmtblock_t *);
-/* Generates one actual loop for a scalarized expression.  */
-void  gfc_trans_scalarized_loop_end (gfc_loopinfo *, int, stmtblock_t *);
 /* Generates the actual loops for a scalarized expression.  */
 void gfc_trans_scalarizing_loops (gfc_loopinfo *, stmtblock_t *);
 /* Mark the end of the main loop body and the start of the copying loop.  */
@@ -137,8 +138,6 @@ tree gfc_build_null_descriptor (tree);
 void gfc_conv_array_ref (gfc_se *, gfc_array_ref *, gfc_expr *, locus *);
 /* Translate a reference to a temporary array.  */
 void gfc_conv_tmp_array_ref (gfc_se * se);
-/* Translate a reference to an array temporary.  */
-void gfc_conv_tmp_ref (gfc_se *);
 
 /* Calculate the overall offset, including subreferences.  */
 void gfc_get_dataptr_offset (stmtblock_t*, tree, tree, tree, bool, gfc_expr*);
@@ -149,8 +148,6 @@ void gfc_conv_expr_descriptor (gfc_se *, gfc_expr *);
 /* Convert an array for passing as an actual function parameter.  */
 void gfc_conv_array_parameter (gfc_se *, gfc_expr *, bool,
 			       const gfc_symbol *, const char *, tree *);
-/* Evaluate and transpose a matrix expression.  */
-void gfc_conv_array_transpose (gfc_se *, gfc_expr *);
 
 /* These work with both descriptors and descriptorless arrays.  */
 tree gfc_conv_array_data (tree);
diff --git a/gcc/fortran/trans-decl.c b/gcc/fortran/trans-decl.c
index de624c8..cb7f684 100644
--- a/gcc/fortran/trans-decl.c
+++ b/gcc/fortran/trans-decl.c
@@ -42,7 +42,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "trans-types.h"
 #include "trans-array.h"
 #include "trans-const.h"
-#include "intrinsic.h" 		/* For gfc_resolve_index_func.  */
 /* Only for gfc_trans_code.  Shouldn't need to include this.  */
 #include "trans-stmt.h"
 #include "gomp-constants.h"
@@ -2267,28 +2266,7 @@ module_sym:
 		{
 		  /* All specific intrinsics take less than 5 arguments.  */
 		  gcc_assert (isym->formal->next->next->next->next == NULL);
-		  if (isym->resolve.f1m == gfc_resolve_index_func)
-		    {
-		      /* gfc_resolve_index_func is special because it takes a
-			 gfc_actual_arglist instead of individual arguments.  */
-		      gfc_actual_arglist *a, *n;
-		      int i;
-		      a = gfc_get_actual_arglist();
-		      n = a;
-
-		      for (i = 0; i < 4; i++)
-			{
-			  n->next = gfc_get_actual_arglist();
-			  n = n->next;
-			}
-
-		      a->expr = &argexpr;
-		      isym->resolve.f1m (&e, a);
-		      a->expr = NULL;
-		      gfc_free_actual_arglist (a);
-		    }
-		  else
-		    isym->resolve.f4 (&e, &argexpr, NULL, NULL, NULL);
+		  isym->resolve.f4 (&e, &argexpr, NULL, NULL, NULL);
 		}
 	    }
 	}
@@ -6668,7 +6646,7 @@ gfc_conv_cfi_to_gfc (stmtblock_t *init, stmtblock_t *finally,
   stmtblock_t block;
   gfc_init_block (&block);
   tree cfi = build_fold_indirect_ref_loc (input_location, cfi_desc);
-  tree rank, idx, etype, tmp, tmp2, size_var = NULL_TREE;
+  tree idx, etype, tmp, tmp2, size_var = NULL_TREE, rank = NULL_TREE;
   bool do_copy_inout = false;
 
   /* When allocatable + intent out, free the cfi descriptor.  */
@@ -6837,6 +6815,13 @@ gfc_conv_cfi_to_gfc (stmtblock_t *init, stmtblock_t *finally,
       gfc_add_modify (&block, sym->ts.u.cl->backend_decl, tmp);
     }
 
+  if (sym->ts.type == BT_CHARACTER
+      && !INTEGER_CST_P (sym->ts.u.cl->backend_decl))
+    {
+      gfc_conv_string_length (sym->ts.u.cl, NULL, init);
+      gfc_trans_vla_type_sizes (sym, init);
+    }
+
   /* gfc->data = cfi->base_addr - or for scalars: gfc = cfi->base_addr.
      assumed-size/explicit-size arrays end up here for character(len=*)
      only. */
diff --git a/gcc/fortran/trans-expr.c b/gcc/fortran/trans-expr.c
index 29697e6..e7aec38 100644
--- a/gcc/fortran/trans-expr.c
+++ b/gcc/fortran/trans-expr.c
@@ -45,7 +45,7 @@ along with GCC; see the file COPYING3.  If not see
 
 /* Calculate the number of characters in a string.  */
 
-tree
+static tree
 gfc_get_character_len (tree type)
 {
   tree len;
@@ -278,7 +278,7 @@ gfc_class_len_get (tree decl)
 /* Try to get the _len component of a class.  When the class is not unlimited
    poly, i.e. no _len field exists, then return a zero node.  */
 
-tree
+static tree
 gfc_class_len_or_zero_get (tree decl)
 {
   tree len;
@@ -382,7 +382,7 @@ VTAB_GET_FIELD_GEN (def_init, VTABLE_DEF_INIT_FIELD)
 VTAB_GET_FIELD_GEN (copy, VTABLE_COPY_FIELD)
 VTAB_GET_FIELD_GEN (final, VTABLE_FINAL_FIELD)
 VTAB_GET_FIELD_GEN (deallocate, VTABLE_DEALLOCATE_FIELD)
-
+#undef VTAB_GET_FIELD_GEN
 
 /* The size field is returned as an array index type.  Therefore treat
    it and only it specially.  */
@@ -1367,7 +1367,7 @@ gfc_conv_class_to_class (gfc_se *parmse, gfc_expr *e, gfc_typespec class_ts,
 /* Given a class array declaration and an index, returns the address
    of the referenced element.  */
 
-tree
+static tree
 gfc_get_class_array_ref (tree index, tree class_decl, tree data_comp,
 			 bool unlimited)
 {
@@ -4531,7 +4531,7 @@ gfc_add_interface_mapping (gfc_interface_mapping * mapping,
    the length of each argument, adding any initialization code to PRE and
    any finalization code to POST.  */
 
-void
+static void
 gfc_finish_interface_mapping (gfc_interface_mapping * mapping,
 			      stmtblock_t * pre, stmtblock_t * post)
 {
@@ -11727,6 +11727,7 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
 
 	  tmp = INDIRECT_REF_P (lse.expr)
 	      ? gfc_build_addr_expr (NULL_TREE, lse.expr) : lse.expr;
+	  STRIP_NOPS (tmp);
 
 	  /* We should only get array references here.  */
 	  gcc_assert (TREE_CODE (tmp) == POINTER_PLUS_EXPR
diff --git a/gcc/fortran/trans-intrinsic.c b/gcc/fortran/trans-intrinsic.c
index 0d91958..3f86791 100644
--- a/gcc/fortran/trans-intrinsic.c
+++ b/gcc/fortran/trans-intrinsic.c
@@ -11084,6 +11084,7 @@ gfc_walk_intrinsic_function (gfc_ss * ss, gfc_expr * expr,
 
   if (isym->elemental)
     return gfc_walk_elemental_function_args (ss, expr->value.function.actual,
+					     expr->value.function.isym,
 					     NULL, GFC_SS_SCALAR);
 
   if (expr->rank == 0)
diff --git a/gcc/fortran/trans-openmp.c b/gcc/fortran/trans-openmp.c
index e81c558..5b3c310 100644
--- a/gcc/fortran/trans-openmp.c
+++ b/gcc/fortran/trans-openmp.c
@@ -3927,18 +3927,27 @@ gfc_trans_omp_clauses (stmtblock_t *block, gfc_omp_clauses *clauses,
 	}
     }
 
-  if (clauses->num_teams)
+  if (clauses->num_teams_upper)
     {
-      tree num_teams;
+      tree num_teams_lower = NULL_TREE, num_teams_upper;
 
       gfc_init_se (&se, NULL);
-      gfc_conv_expr (&se, clauses->num_teams);
+      gfc_conv_expr (&se, clauses->num_teams_upper);
       gfc_add_block_to_block (block, &se.pre);
-      num_teams = gfc_evaluate_now (se.expr, block);
+      num_teams_upper = gfc_evaluate_now (se.expr, block);
       gfc_add_block_to_block (block, &se.post);
 
+      if (clauses->num_teams_lower)
+	{
+	  gfc_init_se (&se, NULL);
+	  gfc_conv_expr (&se, clauses->num_teams_lower);
+	  gfc_add_block_to_block (block, &se.pre);
+	  num_teams_lower = gfc_evaluate_now (se.expr, block);
+	  gfc_add_block_to_block (block, &se.post);
+	}
       c = build_omp_clause (gfc_get_location (&where), OMP_CLAUSE_NUM_TEAMS);
-      OMP_CLAUSE_NUM_TEAMS_EXPR (c) = num_teams;
+      OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c) = num_teams_lower;
+      OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (c) = num_teams_upper;
       omp_clauses = gfc_trans_add_clause (c, omp_clauses);
     }
 
@@ -5861,6 +5870,8 @@ gfc_split_omp_clauses (gfc_code *code,
 	    = code->ext.omp_clauses->lists[OMP_LIST_IS_DEVICE_PTR];
 	  clausesa[GFC_OMP_SPLIT_TARGET].device
 	    = code->ext.omp_clauses->device;
+	  clausesa[GFC_OMP_SPLIT_TARGET].thread_limit
+	    = code->ext.omp_clauses->thread_limit;
 	  for (int i = 0; i < OMP_DEFAULTMAP_CAT_NUM; i++)
 	    clausesa[GFC_OMP_SPLIT_TARGET].defaultmap[i]
 	      = code->ext.omp_clauses->defaultmap[i];
@@ -5869,12 +5880,16 @@ gfc_split_omp_clauses (gfc_code *code,
 	  /* And this is copied to all.  */
 	  clausesa[GFC_OMP_SPLIT_TARGET].if_expr
 	    = code->ext.omp_clauses->if_expr;
+	  clausesa[GFC_OMP_SPLIT_TARGET].nowait
+	    = code->ext.omp_clauses->nowait;
 	}
       if (mask & GFC_OMP_MASK_TEAMS)
 	{
 	  /* First the clauses that are unique to some constructs.  */
-	  clausesa[GFC_OMP_SPLIT_TEAMS].num_teams
-	    = code->ext.omp_clauses->num_teams;
+	  clausesa[GFC_OMP_SPLIT_TEAMS].num_teams_lower
+	    = code->ext.omp_clauses->num_teams_lower;
+	  clausesa[GFC_OMP_SPLIT_TEAMS].num_teams_upper
+	    = code->ext.omp_clauses->num_teams_upper;
 	  clausesa[GFC_OMP_SPLIT_TEAMS].thread_limit
 	    = code->ext.omp_clauses->thread_limit;
 	  /* Shared and default clauses are allowed on parallel, teams
@@ -6649,7 +6664,7 @@ gfc_trans_omp_target (gfc_code *code)
       break;
     default:
       if (flag_openmp
-	  && (clausesa[GFC_OMP_SPLIT_TEAMS].num_teams
+	  && (clausesa[GFC_OMP_SPLIT_TEAMS].num_teams_upper
 	      || clausesa[GFC_OMP_SPLIT_TEAMS].thread_limit))
 	{
 	  gfc_omp_clauses clausesb;
@@ -6658,9 +6673,13 @@ gfc_trans_omp_target (gfc_code *code)
 	     thread_limit clauses are evaluated before entering the
 	     target construct.  */
 	  memset (&clausesb, '\0', sizeof (clausesb));
-	  clausesb.num_teams = clausesa[GFC_OMP_SPLIT_TEAMS].num_teams;
+	  clausesb.num_teams_lower
+	    = clausesa[GFC_OMP_SPLIT_TEAMS].num_teams_lower;
+	  clausesb.num_teams_upper
+	    = clausesa[GFC_OMP_SPLIT_TEAMS].num_teams_upper;
 	  clausesb.thread_limit = clausesa[GFC_OMP_SPLIT_TEAMS].thread_limit;
-	  clausesa[GFC_OMP_SPLIT_TEAMS].num_teams = NULL;
+	  clausesa[GFC_OMP_SPLIT_TEAMS].num_teams_lower = NULL;
+	  clausesa[GFC_OMP_SPLIT_TEAMS].num_teams_upper = NULL;
 	  clausesa[GFC_OMP_SPLIT_TEAMS].thread_limit = NULL;
 	  teams_clauses
 	    = gfc_trans_omp_clauses (&block, &clausesb, code->loc);
diff --git a/gcc/fortran/trans-stmt.c b/gcc/fortran/trans-stmt.c
index c66a3be..bdf7957 100644
--- a/gcc/fortran/trans-stmt.c
+++ b/gcc/fortran/trans-stmt.c
@@ -356,6 +356,25 @@ gfc_conv_elemental_dependencies (gfc_se * se, gfc_se * loopse,
 }
 
 
+/* Given an executable statement referring to an intrinsic function call,
+   returns the intrinsic symbol.  */
+
+static gfc_intrinsic_sym *
+get_intrinsic_for_code (gfc_code *code)
+{
+  if (code->op == EXEC_CALL)
+    {
+      gfc_intrinsic_sym * const isym = code->resolved_isym;
+      if (isym)
+	return isym;
+      else
+	return gfc_get_intrinsic_for_expr (code->expr1);
+    }
+
+  return NULL;
+}
+
+
 /* Get the interface symbol for the procedure corresponding to the given call.
    We can't get the procedure symbol directly as we have to handle the case
    of (deferred) type-bound procedures.  */
@@ -402,6 +421,7 @@ gfc_trans_call (gfc_code * code, bool dependency_check,
   ss = gfc_ss_terminator;
   if (code->resolved_sym->attr.elemental)
     ss = gfc_walk_elemental_function_args (ss, code->ext.actual,
+					   get_intrinsic_for_code (code),
 					   get_proc_ifc_for_call (code),
 					   GFC_SS_REFERENCE);
 
@@ -3685,7 +3705,7 @@ gfc_trans_select_rank_cases (gfc_code * code)
   rank = gfc_conv_descriptor_rank (se.expr);
   rank = gfc_evaluate_now (rank, &block);
   symbol_attribute attr = gfc_expr_attr (code->expr1);
-  if (!attr.pointer || !attr.allocatable)
+  if (!attr.pointer && !attr.allocatable)
     {
       /* Special case for assumed-rank ('rank(*)', internally -1):
 	 rank = (rank == 0 || ubound[rank-1] != -1) ? rank : -1.  */
diff --git a/gcc/fortran/trans-stmt.h b/gcc/fortran/trans-stmt.h
index 1a24d9b..e824caf 100644
--- a/gcc/fortran/trans-stmt.h
+++ b/gcc/fortran/trans-stmt.h
@@ -66,7 +66,6 @@ tree gfc_trans_sync_team (gfc_code *);
 tree gfc_trans_where (gfc_code *);
 tree gfc_trans_allocate (gfc_code *);
 tree gfc_trans_deallocate (gfc_code *);
-tree gfc_trans_deallocate_array (tree);
 
 /* trans-openmp.c */
 tree gfc_trans_omp_directive (gfc_code *);
diff --git a/gcc/fortran/trans-types.c b/gcc/fortran/trans-types.c
index 4277806..e5d36d5 100644
--- a/gcc/fortran/trans-types.c
+++ b/gcc/fortran/trans-types.c
@@ -61,7 +61,7 @@ tree pvoid_type_node;
 tree prvoid_type_node;
 tree ppvoid_type_node;
 tree pchar_type_node;
-tree pfunc_type_node;
+static tree pfunc_type_node;
 
 tree logical_type_node;
 tree logical_true_node;
@@ -133,7 +133,7 @@ int gfc_size_kind;
 int gfc_numeric_storage_size;
 int gfc_character_storage_size;
 
-tree dtype_type_node = NULL_TREE;
+static tree dtype_type_node = NULL_TREE;
 
 
 /* Build the dtype_type_node if necessary.  */
@@ -175,25 +175,6 @@ tree get_dtype_type_node (void)
   return dtype_type_node;
 }
 
-bool
-gfc_check_any_c_kind (gfc_typespec *ts)
-{
-  int i;
-
-  for (i = 0; i < ISOCBINDING_NUMBER; i++)
-    {
-      /* Check for any C interoperable kind for the given type/kind in ts.
-         This can be used after verify_c_interop to make sure that the
-         Fortran kind being used exists in at least some form for C.  */
-      if (c_interop_kinds_table[i].f90_type == ts->type &&
-          c_interop_kinds_table[i].value == ts->kind)
-        return true;
-    }
-
-  return false;
-}
-
-
 static int
 get_real_kind_from_node (tree type)
 {
@@ -2470,7 +2451,7 @@ gfc_copy_dt_decls_ifequal (gfc_symbol *from, gfc_symbol *to,
 
 /* Build a tree node for a procedure pointer component.  */
 
-tree
+static tree
 gfc_get_ppc_type (gfc_component* c)
 {
   tree t;
@@ -3436,10 +3417,8 @@ gfc_get_array_descr_info (const_tree type, struct array_descr_info *info)
   base_decl = GFC_TYPE_ARRAY_BASE_DECL (type, indirect);
   if (!base_decl)
     {
-      base_decl = make_node (DEBUG_EXPR_DECL);
-      DECL_ARTIFICIAL (base_decl) = 1;
-      TREE_TYPE (base_decl) = indirect ? build_pointer_type (ptype) : ptype;
-      SET_DECL_MODE (base_decl, TYPE_MODE (TREE_TYPE (base_decl)));
+      base_decl = build_debug_expr_decl (indirect
+					 ? build_pointer_type (ptype) : ptype);
       GFC_TYPE_ARRAY_BASE_DECL (type, indirect) = base_decl;
     }
   info->base_decl = base_decl;
diff --git a/gcc/fortran/trans-types.h b/gcc/fortran/trans-types.h
index 15d206b..3bc236c 100644
--- a/gcc/fortran/trans-types.h
+++ b/gcc/fortran/trans-types.h
@@ -65,9 +65,6 @@ enum gfc_packed {
   PACKED_STATIC
 };
 
-/* be-function.c */
-void gfc_convert_function_code (gfc_namespace *);
-
 /* trans-types.c */
 void gfc_init_kinds (void);
 void gfc_init_types (void);
@@ -117,7 +114,6 @@ int gfc_is_nodesc_array (gfc_symbol *);
 tree gfc_get_dtype_rank_type (int, tree);
 tree gfc_get_dtype (tree, int *rank = NULL);
 
-tree gfc_get_ppc_type (gfc_component *);
 tree gfc_get_caf_vector_type (int dim);
 tree gfc_get_caf_reference_type ();
 
diff --git a/gcc/fortran/trans.c b/gcc/fortran/trans.c
index 22f2676..a377d0e 100644
--- a/gcc/fortran/trans.c
+++ b/gcc/fortran/trans.c
@@ -45,7 +45,6 @@ along with GCC; see the file COPYING3.  If not see
 static gfc_file *gfc_current_backend_file;
 
 const char gfc_msg_fault[] = N_("Array reference out of bounds");
-const char gfc_msg_wrong_return[] = N_("Incorrect function return value");
 
 
 /* Return a location_t suitable for 'tree' for a gfortran locus.  The way the
diff --git a/gcc/fortran/trans.h b/gcc/fortran/trans.h
index 7ec4ca53..0d4eed2 100644
--- a/gcc/fortran/trans.h
+++ b/gcc/fortran/trans.h
@@ -425,7 +425,6 @@ tree gfc_class_set_static_fields (tree, tree, tree);
 tree gfc_class_data_get (tree);
 tree gfc_class_vptr_get (tree);
 tree gfc_class_len_get (tree);
-tree gfc_class_len_or_zero_get (tree);
 tree gfc_resize_class_size_with_len (stmtblock_t *, tree, tree);
 gfc_expr * gfc_find_and_cut_at_last_class_ref (gfc_expr *, bool is_mold = false,
 					       gfc_typespec **ts = NULL);
@@ -433,14 +432,12 @@ gfc_expr * gfc_find_and_cut_at_last_class_ref (gfc_expr *, bool is_mold = false,
    available.  */
 tree gfc_class_vtab_hash_get (tree);
 tree gfc_class_vtab_size_get (tree);
-tree gfc_class_vtab_extends_get (tree);
 tree gfc_class_vtab_def_init_get (tree);
 tree gfc_class_vtab_copy_get (tree);
 tree gfc_class_vtab_final_get (tree);
 /* Get an accessor to the vtab's * field, when a vptr handle is present.  */
 tree gfc_vptr_hash_get (tree);
 tree gfc_vptr_size_get (tree);
-tree gfc_vptr_extends_get (tree);
 tree gfc_vptr_def_init_get (tree);
 tree gfc_vptr_copy_get (tree);
 tree gfc_vptr_final_get (tree);
@@ -450,7 +447,6 @@ void gfc_reset_len (stmtblock_t *, gfc_expr *);
 tree gfc_get_class_from_gfc_expr (gfc_expr *);
 tree gfc_get_class_from_expr (tree);
 tree gfc_get_vptr_from_expr (tree);
-tree gfc_get_class_array_ref (tree, tree, tree, bool);
 tree gfc_copy_class_to_class (tree, tree, tree, bool);
 bool gfc_add_finalizer_call (stmtblock_t *, gfc_expr *);
 bool gfc_add_comp_finalizer_call (stmtblock_t *, tree, gfc_component *, bool);
@@ -509,7 +505,6 @@ void gfc_conv_expr_type (gfc_se * se, gfc_expr *, tree);
 
 
 /* trans-expr.c */
-tree gfc_get_character_len (tree);
 tree gfc_get_character_len_in_bytes (tree);
 tree gfc_conv_scalar_to_descriptor (gfc_se *, tree, symbol_attribute);
 tree gfc_get_ultimate_alloc_ptr_comps_caf_token (gfc_se *, gfc_expr *);
@@ -621,9 +616,6 @@ tree gfc_get_extern_function_decl (gfc_symbol *,
 				   gfc_actual_arglist *args = NULL,
 				   const char *fnspec = NULL);
 
-/* Return the decl for a function.  */
-tree gfc_get_function_decl (gfc_symbol *);
-
 /* Build an ADDR_EXPR.  */
 tree gfc_build_addr_expr (tree, tree);
 
@@ -1171,15 +1163,12 @@ void gfc_init_interface_mapping (gfc_interface_mapping *);
 void gfc_free_interface_mapping (gfc_interface_mapping *);
 void gfc_add_interface_mapping (gfc_interface_mapping *,
 				gfc_symbol *, gfc_se *, gfc_expr *);
-void gfc_finish_interface_mapping (gfc_interface_mapping *,
-				   stmtblock_t *, stmtblock_t *);
 void gfc_apply_interface_mapping (gfc_interface_mapping *,
 				  gfc_se *, gfc_expr *);
 
 
 /* Standard error messages used in all the trans-*.c files.  */
 extern const char gfc_msg_fault[];
-extern const char gfc_msg_wrong_return[];
 
 #define OMPWS_WORKSHARE_FLAG	1	/* Set if in a workshare construct.  */
 #define OMPWS_CURR_SINGLEUNIT	2	/* Set if current gfc_code in workshare
diff --git a/gcc/fortran/types.def b/gcc/fortran/types.def
index 85b85ed..850fe97 100644
--- a/gcc/fortran/types.def
+++ b/gcc/fortran/types.def
@@ -117,7 +117,6 @@ DEF_FUNCTION_TYPE_2 (BT_FN_BOOL_VPTR_INT, BT_BOOL, BT_VOLATILE_PTR, BT_INT)
 DEF_FUNCTION_TYPE_2 (BT_FN_BOOL_SIZE_CONST_VPTR, BT_BOOL, BT_SIZE,
 		     BT_CONST_VOLATILE_PTR)
 DEF_FUNCTION_TYPE_2 (BT_FN_BOOL_INT_BOOL, BT_BOOL, BT_INT, BT_BOOL)
-DEF_FUNCTION_TYPE_2 (BT_FN_VOID_UINT_UINT, BT_VOID, BT_UINT, BT_UINT)
 DEF_FUNCTION_TYPE_2 (BT_FN_VOID_PTR_PTRMODE,
 		     BT_VOID, BT_PTR, BT_PTRMODE)
 DEF_FUNCTION_TYPE_2 (BT_FN_VOID_CONST_PTR_SIZE, BT_VOID, BT_CONST_PTR, BT_SIZE)
@@ -173,6 +172,8 @@ DEF_FUNCTION_TYPE_4 (BT_FN_BOOL_UINT_ULLPTR_ULLPTR_ULLPTR,
 		     BT_PTR_ULONGLONG)
 DEF_FUNCTION_TYPE_4 (BT_FN_VOID_UINT_PTR_INT_PTR, BT_VOID, BT_INT, BT_PTR,
 		     BT_INT, BT_PTR)
+DEF_FUNCTION_TYPE_4 (BT_FN_BOOL_UINT_UINT_UINT_BOOL,
+		     BT_BOOL, BT_UINT, BT_UINT, BT_UINT, BT_BOOL)
 
 DEF_FUNCTION_TYPE_5 (BT_FN_VOID_OMPFN_PTR_UINT_UINT_UINT,
 		     BT_VOID, BT_PTR_FN_VOID_PTR, BT_PTR, BT_UINT, BT_UINT,
diff --git a/gcc/function-tests.c b/gcc/function-tests.c
index 0ac1d37..8302049 100644
--- a/gcc/function-tests.c
+++ b/gcc/function-tests.c
@@ -42,6 +42,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "function.h"
 #include "dominance.h"
 #include "cfg.h"
+#include "bitmap.h"
 #include "cfganal.h"
 #include "basic-block.h"
 #include "tree-ssa-alias.h"
diff --git a/gcc/function.c b/gcc/function.c
index af3d57b..61b3bd0 100644
--- a/gcc/function.c
+++ b/gcc/function.c
@@ -4873,8 +4873,6 @@ allocate_struct_function (tree fndecl, bool abstract_p)
      binding annotations among them.  */
   cfun->debug_nonbind_markers = lang_hooks.emits_begin_stmt
     && MAY_HAVE_DEBUG_MARKER_STMTS;
-
-  cfun->x_range_query = &global_ranges;
 }
 
 /* This is like allocate_struct_function, but pushes a new cfun for FNDECL
diff --git a/gcc/function.h b/gcc/function.h
index 36003e7..8994308 100644
--- a/gcc/function.h
+++ b/gcc/function.h
@@ -719,15 +719,4 @@ extern const char *current_function_name (void);
 
 extern void used_types_insert (tree);
 
-/* Returns the currently active range access class.  When there is no active
-   range class, global ranges are used.  Never returns null.  */
-
-ATTRIBUTE_RETURNS_NONNULL inline range_query *
-get_range_query (const struct function *fun)
-{
-  return fun->x_range_query;
-}
-
-extern range_query *get_global_range_query ();
-
 #endif  /* GCC_FUNCTION_H */
diff --git a/gcc/gcov.c b/gcc/gcov.c
index 34f53ac..8c10f69 100644
--- a/gcc/gcov.c
+++ b/gcc/gcov.c
@@ -2912,7 +2912,6 @@ read_line (FILE *file)
   static char *string;
   static size_t string_len;
   size_t pos = 0;
-  char *ptr;
 
   if (!string_len)
     {
@@ -2920,7 +2919,7 @@ read_line (FILE *file)
       string = XNEWVEC (char, string_len);
     }
 
-  while ((ptr = fgets (string + pos, string_len - pos, file)))
+  while (fgets (string + pos, string_len - pos, file))
     {
       size_t len = strlen (string + pos);
 
diff --git a/gcc/genconditions.c b/gcc/genconditions.c
index a237df5..b7f2a5a 100644
--- a/gcc/genconditions.c
+++ b/gcc/genconditions.c
@@ -123,7 +123,7 @@ extern rtx operands[];\n\
 static int
 write_one_condition (void **slot, void * ARG_UNUSED (dummy))
 {
-  const struct c_test *test = * (const struct c_test **) slot;
+  const struct c_test *test = *(const struct c_test *const *) slot;
   const char *p;
 
   rtx_reader_ptr->print_md_ptr_loc (test->expr);
diff --git a/gcc/genmodes.c b/gcc/genmodes.c
index c268ebc..ecc8b44 100644
--- a/gcc/genmodes.c
+++ b/gcc/genmodes.c
@@ -749,12 +749,15 @@ make_partial_integer_mode (const char *base, const char *name,
 
 /* A single vector mode can be specified by naming its component
    mode and the number of components.  */
-#define VECTOR_MODE(C, M, N) \
-  make_vector_mode (MODE_##C, #M, N, __FILE__, __LINE__);
+#define VECTOR_MODE_WITH_PREFIX(PREFIX, C, M, N, ORDER) \
+  make_vector_mode (MODE_##C, #PREFIX, #M, N, ORDER, __FILE__, __LINE__);
+#define VECTOR_MODE(C, M, N) VECTOR_MODE_WITH_PREFIX(V, C, M, N, 0);
 static void ATTRIBUTE_UNUSED
 make_vector_mode (enum mode_class bclass,
+		  const char *prefix,
 		  const char *base,
 		  unsigned int ncomponents,
+		  unsigned int order,
 		  const char *file, unsigned int line)
 {
   struct mode_data *v;
@@ -778,7 +781,7 @@ make_vector_mode (enum mode_class bclass,
       return;
     }
 
-  if ((size_t)snprintf (namebuf, sizeof namebuf, "V%u%s",
+  if ((size_t)snprintf (namebuf, sizeof namebuf, "%s%u%s", prefix,
 			ncomponents, base) >= sizeof namebuf)
     {
       error ("%s:%d: mode name \"%s\" is too long",
@@ -787,6 +790,7 @@ make_vector_mode (enum mode_class bclass,
     }
 
   v = new_mode (vclass, xstrdup (namebuf), file, line);
+  v->order = order;
   v->ncomponents = ncomponents;
   v->component = component;
 }
@@ -1312,6 +1316,19 @@ enum machine_mode\n{");
   NUM_MACHINE_MODES = MAX_MACHINE_MODE\n\
 };\n");
 
+  /* Define a NUM_* macro for each mode class, giving the number of modes
+     in the class.  */
+  for (c = 0; c < MAX_MODE_CLASS; c++)
+    {
+      printf ("#define NUM_%s ", mode_class_names[c]);
+      if (modes[c])
+	printf ("(MAX_%s - MIN_%s + 1)\n", mode_class_names[c],
+		mode_class_names[c]);
+      else
+	printf ("0\n");
+    }
+  printf ("\n");
+
   /* I can't think of a better idea, can you?  */
   printf ("#define CONST_MODE_NUNITS%s\n", adj_nunits ? "" : " const");
   printf ("#define CONST_MODE_PRECISION%s\n", adj_nunits ? "" : " const");
diff --git a/gcc/gimple-array-bounds.cc b/gcc/gimple-array-bounds.cc
index 0517e5d..a353559 100644
--- a/gcc/gimple-array-bounds.cc
+++ b/gcc/gimple-array-bounds.cc
@@ -426,7 +426,7 @@ array_bounds_checker::check_mem_ref (location_t location, tree ref,
       axssize = wi::to_offset (access_size);
 
   access_ref aref;
-  if (!compute_objsize (ref, 0, &aref, ranges))
+  if (!compute_objsize (ref, m_stmt, 0, &aref, ranges))
     return false;
 
   if (aref.offset_in_range (axssize))
@@ -667,7 +667,7 @@ array_bounds_checker::check_addr_expr (location_t location, tree t,
    problems discussed in pr98266 and pr97595.  */
 
 static bool
-inbounds_memaccess_p (tree t)
+inbounds_memaccess_p (tree t, gimple *stmt)
 {
   if (TREE_CODE (t) != COMPONENT_REF)
     return false;
@@ -686,7 +686,7 @@ inbounds_memaccess_p (tree t)
      allocated).  */
   access_ref aref;   // unused
   tree refop = TREE_OPERAND (mref, 0);
-  tree refsize = compute_objsize (refop, 1, &aref);
+  tree refsize = compute_objsize (refop, stmt, 1, &aref);
   if (!refsize || TREE_CODE (refsize) != INTEGER_CST)
     return false;
 
@@ -724,6 +724,7 @@ array_bounds_checker::check_array_bounds (tree *tp, int *walk_subtree,
 {
   tree t = *tp;
   struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
+
   location_t location;
 
   if (EXPR_HAS_LOCATION (t))
@@ -735,6 +736,8 @@ array_bounds_checker::check_array_bounds (tree *tp, int *walk_subtree,
 
   bool warned = false;
   array_bounds_checker *checker = (array_bounds_checker *) wi->info;
+  gcc_assert (checker->m_stmt == wi->stmt);
+
   if (TREE_CODE (t) == ARRAY_REF)
     warned = checker->check_array_ref (location, t, wi->stmt,
 				       false/*ignore_off_by_one*/);
@@ -746,7 +749,7 @@ array_bounds_checker::check_array_bounds (tree *tp, int *walk_subtree,
       checker->check_addr_expr (location, t, wi->stmt);
       *walk_subtree = false;
     }
-  else if (inbounds_memaccess_p (t))
+  else if (inbounds_memaccess_p (t, wi->stmt))
     /* Hack: Skip MEM_REF checks in accesses to a member of a base class
        at an offset that's within the bounds of the enclosing object.
        See pr98266 and pr97595.  */
@@ -794,14 +797,13 @@ check_array_bounds_dom_walker::before_dom_children (basic_block bb)
   for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
     {
       gimple *stmt = gsi_stmt (si);
-      struct walk_stmt_info wi;
       if (!gimple_has_location (stmt)
 	  || is_gimple_debug (stmt))
 	continue;
 
-      memset (&wi, 0, sizeof (wi));
-
+      struct walk_stmt_info wi{ };
       wi.info = checker;
+      checker->m_stmt = stmt;
 
       walk_gimple_op (stmt, array_bounds_checker::check_array_bounds, &wi);
     }
diff --git a/gcc/gimple-array-bounds.h b/gcc/gimple-array-bounds.h
index d8f7ff7..d0e665e 100644
--- a/gcc/gimple-array-bounds.h
+++ b/gcc/gimple-array-bounds.h
@@ -36,8 +36,12 @@ private:
   void check_addr_expr (location_t, tree, gimple *);
   const value_range *get_value_range (const_tree op, gimple *);
 
+  /* Current function.  */
   struct function *fun;
+  /* Ranger instance.  */
   range_query *ranges;
+  /* Current statement.  */
+  gimple *m_stmt;
 };
 
 #endif // GCC_GIMPLE_ARRAY_BOUNDS_H
diff --git a/gcc/gimple-fold.c b/gcc/gimple-fold.c
index 7fcfef4..765726c 100644
--- a/gcc/gimple-fold.c
+++ b/gcc/gimple-fold.c
@@ -1505,6 +1505,7 @@ gimple_fold_builtin_memset (gimple_stmt_iterator *gsi, tree c, tree len)
   var = fold_build2 (MEM_REF, etype, dest, build_int_cst (ptr_type_node, 0));
   gimple *store = gimple_build_assign (var, build_int_cst_type (etype, cval));
   gimple_move_vops (store, stmt);
+  gimple_set_location (store, gimple_location (stmt));
   gsi_insert_before (gsi, store, GSI_SAME_STMT);
   if (gimple_call_lhs (stmt))
     {
@@ -2030,6 +2031,28 @@ get_maxval_strlen (tree arg, strlen_range_kind rkind, tree *nonstr = NULL)
   return lendata.decl ? NULL_TREE : lendata.maxlen;
 }
 
+/* Return true if LEN is known to be less than or equal to (or if STRICT is
+   true, strictly less than) the lower bound of SIZE at compile time and false
+   otherwise.  */
+
+static bool
+known_lower (gimple *stmt, tree len, tree size, bool strict = false)
+{
+  if (len == NULL_TREE)
+    return false;
+
+  wide_int size_range[2];
+  wide_int len_range[2];
+  if (get_range (len, stmt, len_range) && get_range (size, stmt, size_range))
+    {
+      if (strict)
+	return wi::ltu_p (len_range[1], size_range[0]);
+      else
+       return wi::leu_p (len_range[1], size_range[0]);
+    }
+
+  return false;
+}
 
 /* Fold function call to builtin strcpy with arguments DEST and SRC.
    If LEN is not NULL, it represents the length of the string to be
@@ -2462,72 +2485,73 @@ gimple_fold_builtin_strncat (gimple_stmt_iterator *gsi)
   tree dst = gimple_call_arg (stmt, 0);
   tree src = gimple_call_arg (stmt, 1);
   tree len = gimple_call_arg (stmt, 2);
-
-  const char *p = c_getstr (src);
+  tree src_len = c_strlen (src, 1);
 
   /* If the requested length is zero, or the src parameter string
      length is zero, return the dst parameter.  */
-  if (integer_zerop (len) || (p && *p == '\0'))
+  if (integer_zerop (len) || (src_len && integer_zerop (src_len)))
     {
       replace_call_with_value (gsi, dst);
       return true;
     }
 
-  if (TREE_CODE (len) != INTEGER_CST || !p)
-    return false;
-
-  unsigned srclen = strlen (p);
-
-  int cmpsrc = compare_tree_int (len, srclen);
-
   /* Return early if the requested len is less than the string length.
      Warnings will be issued elsewhere later.  */
-  if (cmpsrc < 0)
+  if (!src_len || known_lower (stmt, len, src_len, true))
     return false;
 
   unsigned HOST_WIDE_INT dstsize;
+  bool found_dstsize = compute_builtin_object_size (dst, 1, &dstsize);
 
-  bool nowarn = warning_suppressed_p (stmt, OPT_Wstringop_overflow_);
-
-  if (!nowarn && compute_builtin_object_size (dst, 1, &dstsize))
+  /* Warn on constant LEN.  */
+  if (TREE_CODE (len) == INTEGER_CST)
     {
-      int cmpdst = compare_tree_int (len, dstsize);
+      bool nowarn = warning_suppressed_p (stmt, OPT_Wstringop_overflow_);
 
-      if (cmpdst >= 0)
+      if (!nowarn && found_dstsize)
 	{
-	  tree fndecl = gimple_call_fndecl (stmt);
+	  int cmpdst = compare_tree_int (len, dstsize);
+
+	  if (cmpdst >= 0)
+	    {
+	      tree fndecl = gimple_call_fndecl (stmt);
+
+	      /* Strncat copies (at most) LEN bytes and always appends
+		 the terminating NUL so the specified bound should never
+		 be equal to (or greater than) the size of the destination.
+		 If it is, the copy could overflow.  */
+	      location_t loc = gimple_location (stmt);
+	      nowarn = warning_at (loc, OPT_Wstringop_overflow_,
+				   cmpdst == 0
+				   ? G_("%qD specified bound %E equals "
+					"destination size")
+				   : G_("%qD specified bound %E exceeds "
+					"destination size %wu"),
+				   fndecl, len, dstsize);
+	      if (nowarn)
+		suppress_warning (stmt, OPT_Wstringop_overflow_);
+	    }
+	}
 
-	  /* Strncat copies (at most) LEN bytes and always appends
-	     the terminating NUL so the specified bound should never
-	     be equal to (or greater than) the size of the destination.
-	     If it is, the copy could overflow.  */
+      if (!nowarn && TREE_CODE (src_len) == INTEGER_CST
+	  && tree_int_cst_compare (src_len, len) == 0)
+	{
+	  tree fndecl = gimple_call_fndecl (stmt);
 	  location_t loc = gimple_location (stmt);
-	  nowarn = warning_at (loc, OPT_Wstringop_overflow_,
-			       cmpdst == 0
-			       ? G_("%qD specified bound %E equals "
-				    "destination size")
-			       : G_("%qD specified bound %E exceeds "
-				    "destination size %wu"),
-			       fndecl, len, dstsize);
-	  if (nowarn)
+
+	  /* To avoid possible overflow the specified bound should also
+	     not be equal to the length of the source, even when the size
+	     of the destination is unknown (it's not an uncommon mistake
+	     to specify as the bound to strncpy the length of the source).  */
+	  if (warning_at (loc, OPT_Wstringop_overflow_,
+			  "%qD specified bound %E equals source length",
+			  fndecl, len))
 	    suppress_warning (stmt, OPT_Wstringop_overflow_);
 	}
     }
 
-  if (!nowarn && cmpsrc == 0)
-    {
-      tree fndecl = gimple_call_fndecl (stmt);
-      location_t loc = gimple_location (stmt);
-
-      /* To avoid possible overflow the specified bound should also
-	 not be equal to the length of the source, even when the size
-	 of the destination is unknown (it's not an uncommon mistake
-	 to specify as the bound to strncpy the length of the source).  */
-      if (warning_at (loc, OPT_Wstringop_overflow_,
-		      "%qD specified bound %E equals source length",
-		      fndecl, len))
-	suppress_warning (stmt, OPT_Wstringop_overflow_);
-    }
+  if (!known_lower (stmt, src_len, len))
+    return false;
 
   tree fn = builtin_decl_implicit (BUILT_IN_STRCAT);
 
@@ -2565,16 +2589,10 @@ gimple_fold_builtin_strncat_chk (gimple_stmt_iterator *gsi)
       return true;
     }
 
-  if (! tree_fits_uhwi_p (size))
-    return false;
-
   if (! integer_all_onesp (size))
     {
       tree src_len = c_strlen (src, 1);
-      if (src_len
-	  && tree_fits_uhwi_p (src_len)
-	  && tree_fits_uhwi_p (len)
-	  && ! tree_int_cst_lt (len, src_len))
+      if (known_lower (stmt, src_len, len))
 	{
 	  /* If LEN >= strlen (SRC), optimize into __strcat_chk.  */
 	  fn = builtin_decl_explicit (BUILT_IN_STRCAT_CHK);
@@ -3023,39 +3041,25 @@ gimple_fold_builtin_memory_chk (gimple_stmt_iterator *gsi,
 	}
     }
 
-  if (! tree_fits_uhwi_p (size))
-    return false;
-
   tree maxlen = get_maxval_strlen (len, SRK_INT_VALUE);
-  if (! integer_all_onesp (size))
+  if (! integer_all_onesp (size)
+      && !known_lower (stmt, len, size)
+      && !known_lower (stmt, maxlen, size))
     {
-      if (! tree_fits_uhwi_p (len))
+      /* MAXLEN and LEN both cannot be proved to be less than SIZE, at
+	 least try to optimize (void) __mempcpy_chk () into
+	 (void) __memcpy_chk () */
+      if (fcode == BUILT_IN_MEMPCPY_CHK && ignore)
 	{
-	  /* If LEN is not constant, try MAXLEN too.
-	     For MAXLEN only allow optimizing into non-_ocs function
-	     if SIZE is >= MAXLEN, never convert to __ocs_fail ().  */
-	  if (maxlen == NULL_TREE || ! tree_fits_uhwi_p (maxlen))
-	    {
-	      if (fcode == BUILT_IN_MEMPCPY_CHK && ignore)
-		{
-		  /* (void) __mempcpy_chk () can be optimized into
-		     (void) __memcpy_chk ().  */
-		  fn = builtin_decl_explicit (BUILT_IN_MEMCPY_CHK);
-		  if (!fn)
-		    return false;
+	  fn = builtin_decl_explicit (BUILT_IN_MEMCPY_CHK);
+	  if (!fn)
+	    return false;
 
-		  gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
-		  replace_call_with_call_and_fold (gsi, repl);
-		  return true;
-		}
-	      return false;
-	    }
+	  gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
+	  replace_call_with_call_and_fold (gsi, repl);
+	  return true;
 	}
-      else
-	maxlen = len;
-
-      if (tree_int_cst_lt (size, maxlen))
-	return false;
+      return false;
     }
 
   fn = NULL_TREE;
@@ -3087,6 +3091,16 @@ gimple_fold_builtin_memory_chk (gimple_stmt_iterator *gsi,
   return true;
 }
 
+/* Print a message in the dump file recording transformation of FROM to TO.  */
+
+static void
+dump_transformation (gcall *from, gcall *to)
+{
+  if (dump_enabled_p ())
+    dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, from, "simplified %T to %T\n",
+		     gimple_call_fn (from), gimple_call_fn (to));
+}
+
 /* Fold a call to the __st[rp]cpy_chk builtin.
    DEST, SRC, and SIZE are the arguments to the call.
    IGNORE is true if return value can be ignored.  FCODE is the BUILT_IN_*
@@ -3099,7 +3113,7 @@ gimple_fold_builtin_stxcpy_chk (gimple_stmt_iterator *gsi,
 				tree src, tree size,
 				enum built_in_function fcode)
 {
-  gimple *stmt = gsi_stmt (*gsi);
+  gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
   location_t loc = gimple_location (stmt);
   bool ignore = gimple_call_lhs (stmt) == NULL_TREE;
   tree len, fn;
@@ -3125,70 +3139,58 @@ gimple_fold_builtin_stxcpy_chk (gimple_stmt_iterator *gsi,
       return true;
     }
 
-  if (! tree_fits_uhwi_p (size))
-    return false;
-
   tree maxlen = get_maxval_strlen (src, SRK_STRLENMAX);
   if (! integer_all_onesp (size))
     {
       len = c_strlen (src, 1);
-      if (! len || ! tree_fits_uhwi_p (len))
+      if (!known_lower (stmt, len, size, true)
+	  && !known_lower (stmt, maxlen, size, true))
 	{
-	  /* If LEN is not constant, try MAXLEN too.
-	     For MAXLEN only allow optimizing into non-_ocs function
-	     if SIZE is >= MAXLEN, never convert to __ocs_fail ().  */
-	  if (maxlen == NULL_TREE || ! tree_fits_uhwi_p (maxlen))
+	  if (fcode == BUILT_IN_STPCPY_CHK)
 	    {
-	      if (fcode == BUILT_IN_STPCPY_CHK)
-		{
-		  if (! ignore)
-		    return false;
-
-		  /* If return value of __stpcpy_chk is ignored,
-		     optimize into __strcpy_chk.  */
-		  fn = builtin_decl_explicit (BUILT_IN_STRCPY_CHK);
-		  if (!fn)
-		    return false;
-
-		  gimple *repl = gimple_build_call (fn, 3, dest, src, size);
-		  replace_call_with_call_and_fold (gsi, repl);
-		  return true;
-		}
-
-	      if (! len || TREE_SIDE_EFFECTS (len))
+	      if (! ignore)
 		return false;
 
-	      /* If c_strlen returned something, but not a constant,
-		 transform __strcpy_chk into __memcpy_chk.  */
-	      fn = builtin_decl_explicit (BUILT_IN_MEMCPY_CHK);
+	      /* If return value of __stpcpy_chk is ignored,
+		 optimize into __strcpy_chk.  */
+	      fn = builtin_decl_explicit (BUILT_IN_STRCPY_CHK);
 	      if (!fn)
 		return false;
 
-	      gimple_seq stmts = NULL;
-	      len = force_gimple_operand (len, &stmts, true, NULL_TREE);
-	      len = gimple_convert (&stmts, loc, size_type_node, len);
-	      len = gimple_build (&stmts, loc, PLUS_EXPR, size_type_node, len,
-				  build_int_cst (size_type_node, 1));
-	      gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
-	      gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
+	      gimple *repl = gimple_build_call (fn, 3, dest, src, size);
 	      replace_call_with_call_and_fold (gsi, repl);
 	      return true;
 	    }
-	}
-      else
-	maxlen = len;
 
-      if (! tree_int_cst_lt (maxlen, size))
-	return false;
+	  if (! len || TREE_SIDE_EFFECTS (len))
+	    return false;
+
+	  /* If c_strlen returned something, but not provably less than size,
+	     transform __strcpy_chk into __memcpy_chk.  */
+	  fn = builtin_decl_explicit (BUILT_IN_MEMCPY_CHK);
+	  if (!fn)
+	    return false;
+
+	  gimple_seq stmts = NULL;
+	  len = force_gimple_operand (len, &stmts, true, NULL_TREE);
+	  len = gimple_convert (&stmts, loc, size_type_node, len);
+	  len = gimple_build (&stmts, loc, PLUS_EXPR, size_type_node, len,
+			      build_int_cst (size_type_node, 1));
+	  gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
+	  gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
+	  replace_call_with_call_and_fold (gsi, repl);
+	  return true;
+	}
     }
 
   /* If __builtin_st{r,p}cpy_chk is used, assume st{r,p}cpy is available.  */
-  fn = builtin_decl_explicit (fcode == BUILT_IN_STPCPY_CHK
+  fn = builtin_decl_explicit (fcode == BUILT_IN_STPCPY_CHK && !ignore
 			      ? BUILT_IN_STPCPY : BUILT_IN_STRCPY);
   if (!fn)
     return false;
 
-  gimple *repl = gimple_build_call (fn, 2, dest, src);
+  gcall *repl = gimple_build_call (fn, 2, dest, src);
+  dump_transformation (stmt, repl);
   replace_call_with_call_and_fold (gsi, repl);
   return true;
 }
@@ -3204,51 +3206,37 @@ gimple_fold_builtin_stxncpy_chk (gimple_stmt_iterator *gsi,
 				 tree len, tree size,
 				 enum built_in_function fcode)
 {
-  gimple *stmt = gsi_stmt (*gsi);
+  gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
   bool ignore = gimple_call_lhs (stmt) == NULL_TREE;
   tree fn;
 
-  if (fcode == BUILT_IN_STPNCPY_CHK && ignore)
-    {
-       /* If return value of __stpncpy_chk is ignored,
-          optimize into __strncpy_chk.  */
-       fn = builtin_decl_explicit (BUILT_IN_STRNCPY_CHK);
-       if (fn)
-	 {
-	   gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
-	   replace_call_with_call_and_fold (gsi, repl);
-	   return true;
-	 }
-    }
-
-  if (! tree_fits_uhwi_p (size))
-    return false;
-
   tree maxlen = get_maxval_strlen (len, SRK_INT_VALUE);
-  if (! integer_all_onesp (size))
+  if (! integer_all_onesp (size)
+      && !known_lower (stmt, len, size) && !known_lower (stmt, maxlen, size))
     {
-      if (! tree_fits_uhwi_p (len))
+      if (fcode == BUILT_IN_STPNCPY_CHK && ignore)
 	{
-	  /* If LEN is not constant, try MAXLEN too.
-	     For MAXLEN only allow optimizing into non-_ocs function
-	     if SIZE is >= MAXLEN, never convert to __ocs_fail ().  */
-	  if (maxlen == NULL_TREE || ! tree_fits_uhwi_p (maxlen))
-	    return false;
+	  /* If return value of __stpncpy_chk is ignored,
+	     optimize into __strncpy_chk.  */
+	  fn = builtin_decl_explicit (BUILT_IN_STRNCPY_CHK);
+	  if (fn)
+	    {
+	      gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
+	      replace_call_with_call_and_fold (gsi, repl);
+	      return true;
+	    }
 	}
-      else
-	maxlen = len;
-
-      if (tree_int_cst_lt (size, maxlen))
-	return false;
+      return false;
     }
 
   /* If __builtin_st{r,p}ncpy_chk is used, assume st{r,p}ncpy is available.  */
-  fn = builtin_decl_explicit (fcode == BUILT_IN_STPNCPY_CHK
+  fn = builtin_decl_explicit (fcode == BUILT_IN_STPNCPY_CHK && !ignore
 			      ? BUILT_IN_STPNCPY : BUILT_IN_STRNCPY);
   if (!fn)
     return false;
 
-  gimple *repl = gimple_build_call (fn, 3, dest, src, len);
+  gcall *repl = gimple_build_call (fn, 3, dest, src, len);
+  dump_transformation (stmt, repl);
   replace_call_with_call_and_fold (gsi, repl);
   return true;
 }
@@ -3358,27 +3346,11 @@ gimple_fold_builtin_snprintf_chk (gimple_stmt_iterator *gsi,
   size = gimple_call_arg (stmt, 3);
   fmt = gimple_call_arg (stmt, 4);
 
-  if (! tree_fits_uhwi_p (size))
+  tree maxlen = get_maxval_strlen (len, SRK_INT_VALUE);
+  if (! integer_all_onesp (size)
+      && !known_lower (stmt, len, size) && !known_lower (stmt, maxlen, size))
     return false;
 
-  if (! integer_all_onesp (size))
-    {
-      tree maxlen = get_maxval_strlen (len, SRK_INT_VALUE);
-      if (! tree_fits_uhwi_p (len))
-	{
-	  /* If LEN is not constant, try MAXLEN too.
-	     For MAXLEN only allow optimizing into non-_ocs function
-	     if SIZE is >= MAXLEN, never convert to __ocs_fail ().  */
-	  if (maxlen == NULL_TREE || ! tree_fits_uhwi_p (maxlen))
-	    return false;
-	}
-      else
-	maxlen = len;
-
-      if (tree_int_cst_lt (size, maxlen))
-	return false;
-    }
-
   if (!init_target_chars ())
     return false;
 
@@ -3437,9 +3409,6 @@ gimple_fold_builtin_sprintf_chk (gimple_stmt_iterator *gsi,
   size = gimple_call_arg (stmt, 2);
   fmt = gimple_call_arg (stmt, 3);
 
-  if (! tree_fits_uhwi_p (size))
-    return false;
-
   len = NULL_TREE;
 
   if (!init_target_chars ())
@@ -3466,20 +3435,13 @@ gimple_fold_builtin_sprintf_chk (gimple_stmt_iterator *gsi,
 	    {
 	      arg = gimple_call_arg (stmt, 4);
 	      if (POINTER_TYPE_P (TREE_TYPE (arg)))
-		{
-		  len = c_strlen (arg, 1);
-		  if (! len || ! tree_fits_uhwi_p (len))
-		    len = NULL_TREE;
-		}
+		len = c_strlen (arg, 1);
 	    }
 	}
     }
 
-  if (! integer_all_onesp (size))
-    {
-      if (! len || ! tree_int_cst_lt (len, size))
-	return false;
-    }
+  if (! integer_all_onesp (size) && !known_lower (stmt, len, size, true))
+    return false;
 
   /* Only convert __{,v}sprintf_chk to {,v}sprintf if flag is 0
      or if format doesn't contain % chars or is "%s".  */
@@ -3663,10 +3625,6 @@ gimple_fold_builtin_snprintf (gimple_stmt_iterator *gsi)
   if (gimple_call_num_args (stmt) == 4)
     orig = gimple_call_arg (stmt, 3);
 
-  if (!tree_fits_uhwi_p (destsize))
-    return false;
-  unsigned HOST_WIDE_INT destlen = tree_to_uhwi (destsize);
-
   /* Check whether the format is a literal string constant.  */
   fmt_str = c_getstr (fmt);
   if (fmt_str == NULL)
@@ -3686,6 +3644,8 @@ gimple_fold_builtin_snprintf (gimple_stmt_iterator *gsi)
       if (orig)
 	return false;
 
+      tree len = build_int_cstu (TREE_TYPE (destsize), strlen (fmt_str));
+
       /* We could expand this as
 	 memcpy (str, fmt, cst - 1); str[cst - 1] = '\0';
 	 or to
@@ -3693,8 +3653,7 @@ gimple_fold_builtin_snprintf (gimple_stmt_iterator *gsi)
 	 but in the former case that might increase code size
 	 and in the latter case grow .rodata section too much.
 	 So punt for now.  */
-      size_t len = strlen (fmt_str);
-      if (len >= destlen)
+      if (!known_lower (stmt, len, destsize, true))
 	return false;
 
       gimple_seq stmts = NULL;
@@ -3703,7 +3662,7 @@ gimple_fold_builtin_snprintf (gimple_stmt_iterator *gsi)
       if (tree lhs = gimple_call_lhs (stmt))
 	{
 	  repl = gimple_build_assign (lhs,
-				      build_int_cst (TREE_TYPE (lhs), len));
+				      fold_convert (TREE_TYPE (lhs), len));
 	  gimple_seq_add_stmt_without_update (&stmts, repl);
 	  gsi_replace_with_seq_vops (gsi, stmts);
 	  /* gsi now points at the assignment to the lhs, get a
@@ -3734,8 +3693,6 @@ gimple_fold_builtin_snprintf (gimple_stmt_iterator *gsi)
 	return false;
 
       tree orig_len = get_maxval_strlen (orig, SRK_STRLEN);
-      if (!orig_len || TREE_CODE (orig_len) != INTEGER_CST)
-	return false;
 
       /* We could expand this as
 	 memcpy (str1, str2, cst - 1); str1[cst - 1] = '\0';
@@ -3744,7 +3701,7 @@ gimple_fold_builtin_snprintf (gimple_stmt_iterator *gsi)
 	 but in the former case that might increase code size
 	 and in the latter case grow .rodata section too much.
 	 So punt for now.  */
-      if (compare_tree_int (orig_len, destlen) >= 0)
+      if (!known_lower (stmt, orig_len, destsize, true))
 	return false;
 
       /* Convert snprintf (str1, cst, "%s", str2) into
diff --git a/gcc/gimple-harden-conditionals.cc b/gcc/gimple-harden-conditionals.cc
new file mode 100644
index 0000000..8916420
--- /dev/null
+++ b/gcc/gimple-harden-conditionals.cc
@@ -0,0 +1,439 @@
+/* Harden conditionals.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   Contributed by Alexandre Oliva <oliva@adacore.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "backend.h"
+#include "tree.h"
+#include "fold-const.h"
+#include "gimple.h"
+#include "gimplify.h"
+#include "tree-pass.h"
+#include "ssa.h"
+#include "gimple-iterator.h"
+#include "tree-cfg.h"
+#include "basic-block.h"
+#include "cfghooks.h"
+#include "cfgloop.h"
+#include "diagnostic.h"
+#include "intl.h"
+
+namespace {
+
+/* These passes introduces redundant, but reversed conditionals at
+   compares, such as those used in conditional branches, and those
+   that compute boolean results.  This doesn't make much sense for
+   abstract CPUs, but this kind of hardening may avoid undesirable
+   execution paths on actual CPUs under such attacks as of power
+   deprivation.  */
+
+/* Define a pass to harden conditionals other than branches.  */
+
+const pass_data pass_data_harden_compares = {
+  GIMPLE_PASS,
+  "hardcmp",
+  OPTGROUP_NONE,
+  TV_NONE,
+  PROP_cfg | PROP_ssa, // properties_required
+  0,	    // properties_provided
+  0,	    // properties_destroyed
+  0,	    // properties_start
+  TODO_update_ssa
+  | TODO_cleanup_cfg
+  | TODO_verify_il, // properties_finish
+};
+
+class pass_harden_compares : public gimple_opt_pass
+{
+public:
+  pass_harden_compares (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_harden_compares, ctxt)
+  {}
+  opt_pass *clone () { return new pass_harden_compares (m_ctxt); }
+  virtual bool gate (function *) {
+    return flag_harden_compares;
+  }
+  virtual unsigned int execute (function *);
+};
+
+/* Define a pass to harden conditionals in branches.  This pass must
+   run after the above, otherwise it will re-harden the checks
+   introduced by the above.  */
+
+const pass_data pass_data_harden_conditional_branches = {
+  GIMPLE_PASS,
+  "hardcbr",
+  OPTGROUP_NONE,
+  TV_NONE,
+  PROP_cfg | PROP_ssa, // properties_required
+  0,	    // properties_provided
+  0,	    // properties_destroyed
+  0,	    // properties_start
+  TODO_update_ssa
+  | TODO_cleanup_cfg
+  | TODO_verify_il, // properties_finish
+};
+
+class pass_harden_conditional_branches : public gimple_opt_pass
+{
+public:
+  pass_harden_conditional_branches (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_harden_conditional_branches, ctxt)
+  {}
+  opt_pass *clone () { return new pass_harden_conditional_branches (m_ctxt); }
+  virtual bool gate (function *) {
+    return flag_harden_conditional_branches;
+  }
+  virtual unsigned int execute (function *);
+};
+
+}
+
+/* If VAL is an SSA name, return an SSA name holding the same value,
+   but without the compiler's knowing that it holds the same value, so
+   that uses thereof can't be optimized the way VAL might.  Insert
+   stmts that initialize it before *GSIP, with LOC.
+
+   Otherwise, VAL must be an invariant, returned unchanged.  */
+
+static inline tree
+detach_value (location_t loc, gimple_stmt_iterator *gsip, tree val)
+{
+  if (TREE_CONSTANT (val) || TREE_CODE (val) != SSA_NAME)
+    {
+      gcc_checking_assert (is_gimple_min_invariant (val));
+      return val;
+    }
+
+  tree ret = copy_ssa_name (val);
+
+  /* Output asm ("" : "=g" (ret) : "0" (val));  */
+  vec<tree, va_gc> *inputs = NULL;
+  vec<tree, va_gc> *outputs = NULL;
+  vec_safe_push (outputs,
+		 build_tree_list
+		 (build_tree_list
+		  (NULL_TREE, build_string (2, "=g")),
+		  ret));
+  vec_safe_push (inputs,
+		 build_tree_list
+		 (build_tree_list
+		  (NULL_TREE, build_string (1, "0")),
+		  val));
+  gasm *detach = gimple_build_asm_vec ("", inputs, outputs,
+				       NULL, NULL);
+  gimple_set_location (detach, loc);
+  gsi_insert_before (gsip, detach, GSI_SAME_STMT);
+
+  SSA_NAME_DEF_STMT (ret) = detach;
+
+  return ret;
+}
+
+/* Build a cond stmt out of COP, LHS, RHS, insert it before *GSIP with
+   location LOC.  *GSIP must be at the end of a basic block.  The succ
+   edge out of the block becomes the true or false edge opposite to
+   that in FLAGS.  Create a new block with a single trap stmt, in the
+   cold partition if the function is partitioned,, and a new edge to
+   it as the other edge for the cond.  */
+
+static inline void
+insert_check_and_trap (location_t loc, gimple_stmt_iterator *gsip,
+		       int flags, enum tree_code cop, tree lhs, tree rhs)
+{
+  basic_block chk = gsi_bb (*gsip);
+
+  gcond *cond = gimple_build_cond (cop, lhs, rhs, NULL, NULL);
+  gimple_set_location (cond, loc);
+  gsi_insert_before (gsip, cond, GSI_SAME_STMT);
+
+  basic_block trp = create_empty_bb (chk);
+
+  gimple_stmt_iterator gsit = gsi_after_labels (trp);
+  gcall *trap = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
+  gimple_set_location (trap, loc);
+  gsi_insert_before (&gsit, trap, GSI_SAME_STMT);
+
+  if (dump_file)
+    fprintf (dump_file,
+	     "Adding reversed compare to block %i, and trap to block %i\n",
+	     chk->index, trp->index);
+
+  if (BB_PARTITION (chk))
+    BB_SET_PARTITION (trp, BB_COLD_PARTITION);
+
+  int true_false_flag = flags & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
+  gcc_assert (true_false_flag);
+  int neg_true_false_flag = (~flags) & (EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
+
+  /* Remove the fallthru bit, and set the truth value for the
+     preexisting edge and for the newly-created one.  In hardcbr,
+     FLAGS is taken from the edge of the original cond expr that we're
+     dealing with, so the reversed compare is expected to yield the
+     negated result, and the same result calls for a trap.  In
+     hardcmp, we're comparing the boolean results of the original and
+     of the reversed compare, so we're passed FLAGS to trap on
+     equality.  */
+  single_succ_edge (chk)->flags &= ~EDGE_FALLTHRU;
+  single_succ_edge (chk)->flags |= neg_true_false_flag;
+  edge e = make_edge (chk, trp, true_false_flag);
+  e->goto_locus = loc;
+
+  if (dom_info_available_p (CDI_DOMINATORS))
+    set_immediate_dominator (CDI_DOMINATORS, trp, chk);
+  if (current_loops)
+    add_bb_to_loop (trp, current_loops->tree_root);
+}
+
+/* Split edge E, and insert_check_and_trap (see above) in the
+   newly-created block, using detached copies of LHS's and RHS's
+   values (see detach_value above) for the COP compare.  */
+
+static inline void
+insert_edge_check_and_trap (location_t loc, edge e,
+			    enum tree_code cop, tree lhs, tree rhs)
+{
+  int flags = e->flags;
+  basic_block src = e->src;
+  basic_block dest = e->dest;
+  location_t eloc = e->goto_locus;
+
+  basic_block chk = split_edge (e);
+  e = NULL;
+
+  single_pred_edge (chk)->goto_locus = loc;
+  single_succ_edge (chk)->goto_locus = eloc;
+
+  if (dump_file)
+    fprintf (dump_file,
+	     "Splitting edge %i->%i into block %i\n",
+	     src->index, dest->index, chk->index);
+
+  gimple_stmt_iterator gsik = gsi_after_labels (chk);
+
+  bool same_p = (lhs == rhs);
+  lhs = detach_value (loc, &gsik, lhs);
+  rhs = same_p ? lhs : detach_value (loc, &gsik, rhs);
+
+  insert_check_and_trap (loc, &gsik, flags, cop, lhs, rhs);
+}
+
+/* Harden cond stmts at the end of FUN's blocks.  */
+
+unsigned int
+pass_harden_conditional_branches::execute (function *fun)
+{
+  basic_block bb;
+  FOR_EACH_BB_REVERSE_FN (bb, fun)
+    {
+      gimple_stmt_iterator gsi = gsi_last_bb (bb);
+
+      if (gsi_end_p (gsi))
+	continue;
+
+      gcond *cond = dyn_cast <gcond *> (gsi_stmt (gsi));
+      if (!cond)
+	continue;
+
+      /* Turn:
+
+	 if (x op y) goto l1; else goto l2;
+
+	 into:
+
+	 if (x op y) goto l1'; else goto l2';
+	 l1': if (x' cop y') goto l1'trap; else goto l1;
+	 l1'trap: __builtin_trap ();
+	 l2': if (x' cop y') goto l2; else goto l2'trap;
+	 l2'trap: __builtin_trap ();
+
+	 where cop is a complementary boolean operation to op; l1', l1'trap,
+	 l2' and l2'trap are newly-created labels; and x' and y' hold the same
+	 value as x and y, but in a way that does not enable the compiler to
+	 optimize the redundant compare away.
+      */
+
+      enum tree_code op = gimple_cond_code (cond);
+      tree lhs = gimple_cond_lhs (cond);
+      tree rhs = gimple_cond_rhs (cond);
+      location_t loc = gimple_location (cond);
+
+      enum tree_code cop = invert_tree_comparison (op, HONOR_NANS (lhs));
+
+      if (cop == ERROR_MARK)
+	/* ??? Can we do better?  */
+	continue;
+
+      insert_edge_check_and_trap (loc, EDGE_SUCC (bb, 0), cop, lhs, rhs);
+      insert_edge_check_and_trap (loc, EDGE_SUCC (bb, 1), cop, lhs, rhs);
+    }
+
+  return 0;
+}
+
+/* Instantiate a hardcbr pass.  */
+
+gimple_opt_pass *
+make_pass_harden_conditional_branches (gcc::context *ctxt)
+{
+  return new pass_harden_conditional_branches (ctxt);
+}
+
+/* Harden boolean-yielding compares in FUN.  */
+
+unsigned int
+pass_harden_compares::execute (function *fun)
+{
+  basic_block bb;
+  /* Go backwards over BBs and stmts, so that, even if we split the
+     block multiple times to insert a cond_expr after each compare we
+     find, we remain in the same block, visiting every preexisting
+     stmt exactly once, and not visiting newly-added blocks or
+     stmts.  */
+  FOR_EACH_BB_REVERSE_FN (bb, fun)
+    for (gimple_stmt_iterator gsi = gsi_last_bb (bb);
+	 !gsi_end_p (gsi); gsi_prev (&gsi))
+      {
+	gassign *asgn = dyn_cast <gassign *> (gsi_stmt (gsi));
+	if (!asgn)
+	  continue;
+
+	/* Turn:
+
+	   z = x op y;
+
+	   into:
+
+	   z = x op y;
+	   z' = x' cop y';
+	   if (z == z') __builtin_trap ();
+
+	   where cop is a complementary boolean operation to op; and x'
+	   and y' hold the same value as x and y, but in a way that does
+	   not enable the compiler to optimize the redundant compare
+	   away.
+	*/
+
+	enum tree_code op = gimple_assign_rhs_code (asgn);
+
+	enum tree_code cop;
+
+	switch (op)
+	  {
+	  case EQ_EXPR:
+	  case NE_EXPR:
+	  case GT_EXPR:
+	  case GE_EXPR:
+	  case LT_EXPR:
+	  case LE_EXPR:
+	  case LTGT_EXPR:
+	  case UNEQ_EXPR:
+	  case UNGT_EXPR:
+	  case UNGE_EXPR:
+	  case UNLT_EXPR:
+	  case UNLE_EXPR:
+	  case ORDERED_EXPR:
+	  case UNORDERED_EXPR:
+	    cop = invert_tree_comparison (op,
+					  HONOR_NANS
+					  (gimple_assign_rhs1 (asgn)));
+
+	    if (cop == ERROR_MARK)
+	      /* ??? Can we do better?  */
+	      continue;
+
+	    break;
+
+	    /* ??? Maybe handle these too?  */
+	  case TRUTH_NOT_EXPR:
+	    /* ??? The code below assumes binary ops, it would have to
+	       be adjusted for TRUTH_NOT_EXPR, since it's unary.  */
+	  case TRUTH_ANDIF_EXPR:
+	  case TRUTH_ORIF_EXPR:
+	  case TRUTH_AND_EXPR:
+	  case TRUTH_OR_EXPR:
+	  case TRUTH_XOR_EXPR:
+	  default:
+	    continue;
+	  }
+
+	/* These are the operands for the verification.  */
+	tree lhs = gimple_assign_lhs (asgn);
+	tree op1 = gimple_assign_rhs1 (asgn);
+	tree op2 = gimple_assign_rhs2 (asgn);
+	location_t loc = gimple_location (asgn);
+
+	/* Vector booleans can't be used in conditional branches.  ???
+	   Can we do better?  How to reduce compare and
+	   reversed-compare result vectors to a single boolean?  */
+	if (VECTOR_TYPE_P (TREE_TYPE (op1)))
+	  continue;
+
+	gcc_checking_assert (TREE_CODE (TREE_TYPE (lhs)) == BOOLEAN_TYPE);
+
+	tree rhs = copy_ssa_name (lhs);
+
+	gimple_stmt_iterator gsi_split = gsi;
+	/* Don't separate the original assignment from debug stmts
+	   that might be associated with it, and arrange to split the
+	   block after debug stmts, so as to make sure the split block
+	   won't be debug stmts only.  */
+	gsi_next_nondebug (&gsi_split);
+
+	bool same_p = (op1 == op2);
+	op1 = detach_value (loc, &gsi_split, op1);
+	op2 = same_p ? op1 : detach_value (loc, &gsi_split, op2);
+
+	gassign *asgnck = gimple_build_assign (rhs, cop, op1, op2);
+	gimple_set_location (asgnck, loc);
+	gsi_insert_before (&gsi_split, asgnck, GSI_SAME_STMT);
+
+	/* We wish to insert a cond_expr after the compare, so arrange
+	   for it to be at the end of a block if it isn't.  */
+	if (!gsi_end_p (gsi_split))
+	  {
+	    gsi_prev (&gsi_split);
+	    split_block (bb, gsi_stmt (gsi_split));
+	    gsi_next (&gsi_split);
+	    gcc_checking_assert (gsi_end_p (gsi_split));
+
+	    single_succ_edge (bb)->goto_locus = loc;
+
+	    if (dump_file)
+	      fprintf (dump_file, "Splitting block %i\n", bb->index);
+	  }
+
+	gcc_checking_assert (single_succ_p (bb));
+
+	insert_check_and_trap (loc, &gsi_split, EDGE_TRUE_VALUE,
+			       EQ_EXPR, lhs, rhs);
+      }
+
+  return 0;
+}
+
+/* Instantiate a hardcmp pass.  */
+
+gimple_opt_pass *
+make_pass_harden_compares (gcc::context *ctxt)
+{
+  return new pass_harden_compares (ctxt);
+}
diff --git a/gcc/gimple-loop-jam.c b/gcc/gimple-loop-jam.c
index 611d380..666f740 100644
--- a/gcc/gimple-loop-jam.c
+++ b/gcc/gimple-loop-jam.c
@@ -593,7 +593,7 @@ tree_loop_unroll_and_jam (void)
 	  todo |= TODO_cleanup_cfg;
 
 	  auto_bitmap exit_bbs;
-	  bitmap_set_bit (exit_bbs, single_dom_exit (outer)->dest->index);
+	  bitmap_set_bit (exit_bbs, single_exit (outer)->dest->index);
 	  todo |= do_rpo_vn (cfun, loop_preheader_edge (outer), exit_bbs);
 	}
 
diff --git a/gcc/gimple-predicate-analysis.cc b/gcc/gimple-predicate-analysis.cc
index f0c8444..6dde020 100644
--- a/gcc/gimple-predicate-analysis.cc
+++ b/gcc/gimple-predicate-analysis.cc
@@ -45,36 +45,6 @@
 
 #define DEBUG_PREDICATE_ANALYZER 1
 
-/* Find the immediate postdominator of the specified basic block BB.  */
-
-static inline basic_block
-find_pdom (basic_block bb)
-{
-  basic_block exit_bb = EXIT_BLOCK_PTR_FOR_FN (cfun);
-  if (bb == exit_bb)
-    return exit_bb;
-
-  if (basic_block pdom = get_immediate_dominator (CDI_POST_DOMINATORS, bb))
-    return pdom;
-
-  return exit_bb;
-}
-
-/* Find the immediate dominator of the specified basic block BB.  */
-
-static inline basic_block
-find_dom (basic_block bb)
-{
-  basic_block entry_bb = ENTRY_BLOCK_PTR_FOR_FN (cfun);
-  if (bb == entry_bb)
-    return entry_bb;
-
-  if (basic_block dom = get_immediate_dominator (CDI_DOMINATORS, bb))
-    return dom;
-
-  return entry_bb;
-}
-
 /* Return true if BB1 is postdominating BB2 and BB1 is not a loop exit
    bb.  The loop exit bb check is simple and does not cover all cases.  */
 
@@ -96,7 +66,7 @@ is_non_loop_exit_postdominating (basic_block bb1, basic_block bb2)
 static inline basic_block
 find_control_equiv_block (basic_block bb)
 {
-  basic_block pdom = find_pdom (bb);
+  basic_block pdom = get_immediate_dominator (CDI_POST_DOMINATORS, bb);
 
   /* Skip the postdominating bb that is also a loop exit.  */
   if (!is_non_loop_exit_postdominating (pdom, bb))
@@ -1167,7 +1137,7 @@ compute_control_dep_chain (basic_block dom_bb, const_basic_block dep_bb,
 	      break;
 	    }
 
-	  cd_bb = find_pdom (cd_bb);
+	  cd_bb = get_immediate_dominator (CDI_POST_DOMINATORS, cd_bb);
 	  post_dom_check++;
 	  if (cd_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
 	      || post_dom_check > MAX_POSTDOM_CHECK)
@@ -1788,7 +1758,7 @@ predicate::init_from_phi_def (gphi *phi)
 
   basic_block phi_bb = gimple_bb (phi);
   /* Find the closest dominating bb to be the control dependence root.  */
-  basic_block cd_root = find_dom (phi_bb);
+  basic_block cd_root = get_immediate_dominator (CDI_DOMINATORS, phi_bb);
   if (!cd_root)
     return false;
 
@@ -2135,67 +2105,6 @@ predicate::normalize (gimple *use_or_def, bool is_use)
     }
 }
 
-/* Add a predicate for the condition or logical assignment STMT to CHAIN.
-   Expand SSA_NAME into constituent subexpressions.  Invert the result
-   if INVERT is true.  Return true if the predicate has been added.  */
-
-static bool
-add_pred (pred_chain *chain, gimple *stmt, bool invert)
-{
-  if (gimple_code (stmt) == GIMPLE_COND)
-    {
-      tree lhs = gimple_cond_lhs (stmt);
-      if (TREE_CODE (lhs) == SSA_NAME)
-	{
-	  gimple *def = SSA_NAME_DEF_STMT (lhs);
-	  if (is_gimple_assign (def)
-	      && add_pred (chain, def, invert))
-	    return true;
-	}
-
-      pred_info pred;
-      pred.pred_lhs = lhs;
-      pred.pred_rhs = gimple_cond_rhs (stmt);
-      pred.cond_code = gimple_cond_code (stmt);
-      pred.invert = invert;
-      chain->safe_push (pred);
-      return true;
-    }
-
-  if (!is_gimple_assign (stmt))
-    return false;
-
-  if (gimple_assign_single_p (stmt))
-    // FIXME: handle this?
-    return false;
-
-  if (TREE_TYPE (gimple_assign_lhs (stmt)) != boolean_type_node)
-    return false;
-
-  tree rhs1 = gimple_assign_rhs1 (stmt);
-  tree rhs2 = gimple_assign_rhs2 (stmt);
-  tree_code code = gimple_assign_rhs_code (stmt);
-  if (code == BIT_AND_EXPR)
-    {
-      if (TREE_CODE (rhs1) == SSA_NAME
-	  && add_pred (chain, SSA_NAME_DEF_STMT (rhs1), invert)
-	  && TREE_CODE (rhs2) == SSA_NAME
-	  /* FIXME: Need to handle failure below! */
-	  && add_pred (chain, SSA_NAME_DEF_STMT (rhs2), invert))
-	return true;
-    }
-  else if (TREE_CODE_CLASS (code) != tcc_comparison)
-    return false;
-
-  pred_info pred;
-  pred.pred_lhs = rhs1;
-  pred.pred_rhs = rhs2;
-  pred.cond_code = code;
-  pred.invert = invert;
-  chain->safe_push (pred);
-  return true;
-}
-
 /* Convert the chains of control dependence edges into a set of predicates.
    A control dependence chain is represented by a vector edges.  DEP_CHAINS
    points to an array of NUM_CHAINS dependence chains. One edge in
diff --git a/gcc/gimple-range-cache.cc b/gcc/gimple-range-cache.cc
index 9cbc63d..b347ede 100644
--- a/gcc/gimple-range-cache.cc
+++ b/gcc/gimple-range-cache.cc
@@ -30,8 +30,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-range.h"
 #include "tree-cfg.h"
 
-#define DEBUG_RANGE_CACHE (dump_file && (param_evrp_mode & EVRP_MODE_CACHE) \
-					 == EVRP_MODE_CACHE)
+#define DEBUG_RANGE_CACHE (dump_file					\
+			   && (param_ranger_debug & RANGER_DEBUG_CACHE))
 
 // During contructor, allocate the vector of ssa_names.
 
@@ -210,6 +210,7 @@ protected:
   int_range<2> m_undefined;
   tree m_type;
   irange_allocator *m_irange_allocator;
+  void grow ();
 };
 
 
@@ -229,13 +230,37 @@ sbr_vector::sbr_vector (tree t, irange_allocator *allocator)
   m_undefined.set_undefined ();
 }
 
+// Grow the vector when the CFG has increased in size.
+
+void
+sbr_vector::grow ()
+{
+  int curr_bb_size = last_basic_block_for_fn (cfun);
+  gcc_checking_assert (curr_bb_size > m_tab_size);
+
+  // Increase the max of a)128, b)needed increase * 2, c)10% of current_size.
+  int inc = MAX ((curr_bb_size - m_tab_size) * 2, 128);
+  inc = MAX (inc, curr_bb_size / 10);
+  int new_size = inc + curr_bb_size;
+
+  // Allocate new memory, copy the old vector and clear the new space.
+  irange **t = (irange **)m_irange_allocator->get_memory (new_size
+							  * sizeof (irange *));
+  memcpy (t, m_tab, m_tab_size * sizeof (irange *));
+  memset (t + m_tab_size, 0, (new_size - m_tab_size) * sizeof (irange *));
+
+  m_tab = t;
+  m_tab_size = new_size;
+}
+
 // Set the range for block BB to be R.
 
 bool
 sbr_vector::set_bb_range (const_basic_block bb, const irange &r)
 {
   irange *m;
-  gcc_checking_assert (bb->index < m_tab_size);
+  if (bb->index >= m_tab_size)
+    grow ();
   if (r.varying_p ())
     m = &m_varying;
   else if (r.undefined_p ())
@@ -252,7 +277,8 @@ sbr_vector::set_bb_range (const_basic_block bb, const irange &r)
 bool
 sbr_vector::get_bb_range (irange &r, const_basic_block bb)
 {
-  gcc_checking_assert (bb->index < m_tab_size);
+  if (bb->index >= m_tab_size)
+    return false;
   irange *m = m_tab[bb->index];
   if (m)
     {
@@ -267,8 +293,9 @@ sbr_vector::get_bb_range (irange &r, const_basic_block bb)
 bool
 sbr_vector::bb_range_p (const_basic_block bb)
 {
-  gcc_checking_assert (bb->index < m_tab_size);
-  return m_tab[bb->index] != NULL;
+  if (bb->index < m_tab_size)
+    return m_tab[bb->index] != NULL;
+  return false;
 }
 
 // This class implements the on entry cache via a sparse bitmap.
@@ -624,7 +651,8 @@ ssa_global_cache::clear_global_range (tree name)
 void
 ssa_global_cache::clear ()
 {
-  memset (m_tab.address(), 0, m_tab.length () * sizeof (irange *));
+  if (m_tab.address ())
+    memset (m_tab.address(), 0, m_tab.length () * sizeof (irange *));
 }
 
 // Dump the contents of the global cache to F.
@@ -754,14 +782,96 @@ temporal_cache::set_always_current (tree name)
 
 // --------------------------------------------------------------------------
 
+// This class provides an abstraction of a list of blocks to be updated
+// by the cache.  It is currently a stack but could be changed.  It also
+// maintains a list of blocks which have failed propagation, and does not
+// enter any of those blocks into the list.
+
+// A vector over the BBs is maintained, and an entry of 0 means it is not in
+// a list.  Otherwise, the entry is the next block in the list. -1 terminates
+// the list.  m_head points to the top of the list, -1 if the list is empty.
+
+class update_list
+{
+public:
+  update_list ();
+  ~update_list ();
+  void add (basic_block bb);
+  basic_block pop ();
+  inline bool empty_p () { return m_update_head == -1; }
+  inline void clear_failures () { bitmap_clear (m_propfail); }
+  inline void propagation_failed (basic_block bb)
+				  { bitmap_set_bit (m_propfail, bb->index); }
+private:
+  vec<int> m_update_list;
+  int m_update_head;
+  bitmap m_propfail;
+};
+
+// Create an update list.
+
+update_list::update_list ()
+{
+  m_update_list.create (0);
+  m_update_list.safe_grow_cleared (last_basic_block_for_fn (cfun) + 64);
+  m_update_head = -1;
+  m_propfail = BITMAP_ALLOC (NULL);
+}
+
+// Destroy an update list.
+
+update_list::~update_list ()
+{
+  m_update_list.release ();
+  BITMAP_FREE (m_propfail);
+}
+
+// Add BB to the list of blocks to update, unless it's already in the list.
+
+void
+update_list::add (basic_block bb)
+{
+  int i = bb->index;
+  // If propagation has failed for BB, or its already in the list, don't
+  // add it again.
+  if ((unsigned)i >= m_update_list.length ())
+    m_update_list.safe_grow_cleared (i + 64);
+  if (!m_update_list[i] && !bitmap_bit_p (m_propfail, i))
+    {
+      if (empty_p ())
+	{
+	  m_update_head = i;
+	  m_update_list[i] = -1;
+	}
+      else
+	{
+	  gcc_checking_assert (m_update_head > 0);
+	  m_update_list[i] = m_update_head;
+	  m_update_head = i;
+	}
+    }
+}
+
+// Remove a block from the list.
+
+basic_block
+update_list::pop ()
+{
+  gcc_checking_assert (!empty_p ());
+  basic_block bb = BASIC_BLOCK_FOR_FN (cfun, m_update_head);
+  int pop = m_update_head;
+  m_update_head = m_update_list[pop];
+  m_update_list[pop] = 0;
+  return bb;
+}
+
+// --------------------------------------------------------------------------
+
 ranger_cache::ranger_cache (int not_executable_flag)
 						: m_gori (not_executable_flag)
 {
   m_workback.create (0);
   m_workback.safe_grow_cleared (last_basic_block_for_fn (cfun));
-  m_update_list.create (0);
-  m_update_list.safe_grow_cleared (last_basic_block_for_fn (cfun));
-  m_update_list.truncate (0);
   m_temporal = new temporal_cache;
   // If DOM info is available, spawn an oracle as well.
   if (dom_info_available_p (CDI_DOMINATORS))
@@ -779,17 +889,16 @@ ranger_cache::ranger_cache (int not_executable_flag)
       if (bb)
 	m_gori.exports (bb);
     }
-  m_propfail = BITMAP_ALLOC (NULL);
+  m_update = new update_list ();
 }
 
 ranger_cache::~ranger_cache ()
 {
-  BITMAP_FREE (m_propfail);
+  delete m_update;
   if (m_oracle)
     delete m_oracle;
   delete m_temporal;
   m_workback.release ();
-  m_update_list.release ();
 }
 
 // Dump the global caches to file F.  if GORI_DUMP is true, dump the
@@ -1029,17 +1138,6 @@ ranger_cache::block_range (irange &r, basic_block bb, tree name, bool calc)
   return m_on_entry.get_bb_range (r, name, bb);
 }
 
-// Add BB to the list of blocks to update, unless it's already in the list.
-
-void
-ranger_cache::add_to_update (basic_block bb)
-{
-  // If propagation has failed for BB, or its already in the list, don't
-  // add it again.
-  if (!bitmap_bit_p (m_propfail, bb->index) &&  !m_update_list.contains (bb))
-    m_update_list.quick_push (bb);
-}
-
 // If there is anything in the propagation update_list, continue
 // processing NAME until the list of blocks is empty.
 
@@ -1053,16 +1151,15 @@ ranger_cache::propagate_cache (tree name)
   int_range_max current_range;
   int_range_max e_range;
 
-  gcc_checking_assert (bitmap_empty_p (m_propfail));
   // Process each block by seeing if its calculated range on entry is
   // the same as its cached value. If there is a difference, update
   // the cache to reflect the new value, and check to see if any
   // successors have cache entries which may need to be checked for
   // updates.
 
-  while (m_update_list.length () > 0)
+  while (!m_update->empty_p ())
     {
-      bb = m_update_list.pop ();
+      bb = m_update->pop ();
       gcc_checking_assert (m_on_entry.bb_range_p (name, bb));
       m_on_entry.get_bb_range (current_range, name, bb);
 
@@ -1097,7 +1194,7 @@ ranger_cache::propagate_cache (tree name)
 	  bool ok_p = m_on_entry.set_bb_range (name, bb, new_range);
 	  // If the cache couldn't set the value, mark it as failed.
 	  if (!ok_p)
-	    bitmap_set_bit (m_propfail, bb->index);
+	    m_update->propagation_failed (bb);
 	  if (DEBUG_RANGE_CACHE) 
 	    {
 	      if (!ok_p)
@@ -1119,7 +1216,7 @@ ranger_cache::propagate_cache (tree name)
 	      {
 		if (DEBUG_RANGE_CACHE) 
 		  fprintf (dump_file, " bb%d",e->dest->index);
-		add_to_update (e->dest);
+		m_update->add (e->dest);
 	      }
 	  if (DEBUG_RANGE_CACHE) 
 	    fprintf (dump_file, "\n");
@@ -1131,7 +1228,7 @@ ranger_cache::propagate_cache (tree name)
       print_generic_expr (dump_file, name, TDF_SLIM);
       fprintf (dump_file, "\n");
     }
-  bitmap_clear (m_propfail);
+  m_update->clear_failures ();
 }
 
 // Check to see if an update to the value for NAME in BB has any effect
@@ -1146,7 +1243,7 @@ ranger_cache::propagate_updated_value (tree name, basic_block bb)
   edge_iterator ei;
 
   // The update work list should be empty at this point.
-  gcc_checking_assert (m_update_list.length () == 0);
+  gcc_checking_assert (m_update->empty_p ());
   gcc_checking_assert (bb);
 
   if (DEBUG_RANGE_CACHE)
@@ -1160,12 +1257,12 @@ ranger_cache::propagate_updated_value (tree name, basic_block bb)
       // Only update active cache entries.
       if (m_on_entry.bb_range_p (name, e->dest))
 	{
-	  add_to_update (e->dest);
+	  m_update->add (e->dest);
 	  if (DEBUG_RANGE_CACHE)
 	    fprintf (dump_file, " UPDATE: bb%d", e->dest->index);
 	}
     }
-    if (m_update_list.length () != 0)
+    if (!m_update->empty_p ())
       {
 	if (DEBUG_RANGE_CACHE)
 	  fprintf (dump_file, "\n");
@@ -1205,7 +1302,7 @@ ranger_cache::fill_block_cache (tree name, basic_block bb, basic_block def_bb)
   m_workback.quick_push (bb);
   undefined.set_undefined ();
   m_on_entry.set_bb_range (name, bb, undefined);
-  gcc_checking_assert (m_update_list.length () == 0);
+  gcc_checking_assert (m_update->empty_p ());
 
   if (DEBUG_RANGE_CACHE)
     {
@@ -1235,7 +1332,7 @@ ranger_cache::fill_block_cache (tree name, basic_block bb, basic_block def_bb)
 	  // If the pred block is the def block add this BB to update list.
 	  if (pred == def_bb)
 	    {
-	      add_to_update (node);
+	      m_update->add (node);
 	      continue;
 	    }
 
@@ -1255,7 +1352,7 @@ ranger_cache::fill_block_cache (tree name, basic_block bb, basic_block def_bb)
 	    {
 	      if (DEBUG_RANGE_CACHE)
 		fprintf (dump_file, "nonnull: update ");
-	      add_to_update (node);
+	      m_update->add (node);
 	    }
 
 	  // If the pred block already has a range, or if it can contribute
@@ -1270,7 +1367,7 @@ ranger_cache::fill_block_cache (tree name, basic_block bb, basic_block def_bb)
 		}
 	      if (!r.undefined_p () || m_gori.has_edge_range_p (name, e))
 		{
-		  add_to_update (node);
+		  m_update->add (node);
 		  if (DEBUG_RANGE_CACHE)
 		    fprintf (dump_file, "update. ");
 		}
diff --git a/gcc/gimple-range-cache.h b/gcc/gimple-range-cache.h
index 4937a0b..49c13d1 100644
--- a/gcc/gimple-range-cache.h
+++ b/gcc/gimple-range-cache.h
@@ -103,6 +103,8 @@ public:
   bool get_non_stale_global_range (irange &r, tree name);
   void set_global_range (tree name, const irange &r);
 
+  void propagate_updated_value (tree name, basic_block bb);
+
   non_null_ref m_non_null;
   gori_compute m_gori;
 
@@ -112,7 +114,6 @@ private:
   ssa_global_cache m_globals;
   block_range_cache m_on_entry;
   class temporal_cache *m_temporal;
-  void add_to_update (basic_block bb);
   void fill_block_cache (tree name, basic_block bb, basic_block def_bb);
   void propagate_cache (tree name);
 
@@ -120,11 +121,8 @@ private:
   void entry_range (irange &r, tree expr, basic_block bb);
   void exit_range (irange &r, tree expr, basic_block bb);
 
-  void propagate_updated_value (tree name, basic_block bb);
-
-  bitmap m_propfail;
   vec<basic_block> m_workback;
-  vec<basic_block> m_update_list;
+  class update_list *m_update;
 };
 
 #endif // GCC_SSA_RANGE_CACHE_H
diff --git a/gcc/gimple-range-fold.cc b/gcc/gimple-range-fold.cc
index ed2fbe1..608d98b 100644
--- a/gcc/gimple-range-fold.cc
+++ b/gcc/gimple-range-fold.cc
@@ -620,7 +620,9 @@ fold_using_range::range_of_range_op (irange &r, gimple *s, fur_source &src)
 	  if (dump_file && (dump_flags & TDF_DETAILS) && rel != VREL_NONE)
 	    {
 	      fprintf (dump_file, " folding with relation ");
+	      print_generic_expr (dump_file, op1, TDF_SLIM);
 	      print_relation (dump_file, rel);
+	      print_generic_expr (dump_file, op2, TDF_SLIM);
 	      fputc ('\n', dump_file);
 	    }
 	  // Fold range, and register any dependency if available.
@@ -777,6 +779,10 @@ fold_using_range::range_of_phi (irange &r, gphi *phi, fur_source &src)
   for (x = 0; x < gimple_phi_num_args (phi); x++)
     {
       tree arg = gimple_phi_arg_def (phi, x);
+      // An argument that is the same as the def provides no new range.
+      if (arg == phi_def)
+	continue;
+
       edge e = gimple_phi_arg_edge (phi, x);
 
       // Get the range of the argument on its edge.
@@ -958,18 +964,18 @@ fold_using_range::range_of_builtin_call (irange &r, gcall *call,
   switch (func)
     {
     case CFN_BUILT_IN_CONSTANT_P:
-      if (cfun->after_inlining)
-	{
-	  r.set_zero (type);
-	  // r.equiv_clear ();
-	  return true;
-	}
       arg = gimple_call_arg (call, 0);
       if (src.get_operand (r, arg) && r.singleton_p ())
 	{
 	  r.set (build_one_cst (type), build_one_cst (type));
 	  return true;
 	}
+      if (cfun->after_inlining)
+	{
+	  r.set_zero (type);
+	  // r.equiv_clear ();
+	  return true;
+	}
       break;
 
     case CFN_BUILT_IN_TOUPPER:
diff --git a/gcc/gimple-range-gori.cc b/gcc/gimple-range-gori.cc
index 6946fa6..fb2d571 100644
--- a/gcc/gimple-range-gori.cc
+++ b/gcc/gimple-range-gori.cc
@@ -37,16 +37,13 @@ bool
 gimple_range_calc_op1 (irange &r, const gimple *stmt, const irange &lhs_range)
 {
   gcc_checking_assert (gimple_num_ops (stmt) < 3);
-
-  // An empty range is viral.
-  tree type = TREE_TYPE (gimple_range_operand1 (stmt));
+  // Give up on empty ranges.
   if (lhs_range.undefined_p ())
-    {
-      r.set_undefined ();
-      return true;
-    }
+    return false;
+
   // Unary operations require the type of the first operand in the
   // second range position.
+  tree type = TREE_TYPE (gimple_range_operand1 (stmt));
   int_range<2> type_range (type);
   return gimple_range_handler (stmt)->op1_range (r, type, lhs_range,
 						 type_range);
@@ -61,15 +58,23 @@ bool
 gimple_range_calc_op1 (irange &r, const gimple *stmt,
 		       const irange &lhs_range, const irange &op2_range)
 {
+  // Give up on empty ranges.
+  if (lhs_range.undefined_p ())
+    return false;
+
   // Unary operation are allowed to pass a range in for second operand
   // as there are often additional restrictions beyond the type which
   // can be imposed.  See operator_cast::op1_range().
   tree type = TREE_TYPE (gimple_range_operand1 (stmt));
-  // An empty range is viral.
-  if (op2_range.undefined_p () || lhs_range.undefined_p ())
+  // If op2 is undefined, solve as if it is varying.
+  if (op2_range.undefined_p ())
     {
-      r.set_undefined ();
-      return true;
+      // This is sometimes invoked on single operand stmts.
+      if (gimple_num_ops (stmt) < 3)
+	return false;
+      int_range<2> trange (TREE_TYPE (gimple_range_operand2 (stmt)));
+      return gimple_range_handler (stmt)->op1_range (r, type, lhs_range,
+						     trange);
     }
   return gimple_range_handler (stmt)->op1_range (r, type, lhs_range,
 						 op2_range);
@@ -84,12 +89,17 @@ bool
 gimple_range_calc_op2 (irange &r, const gimple *stmt,
 		       const irange &lhs_range, const irange &op1_range)
 {
+  // Give up on empty ranges.
+  if (lhs_range.undefined_p ())
+    return false;
+
   tree type = TREE_TYPE (gimple_range_operand2 (stmt));
-  // An empty range is viral.
-  if (op1_range.undefined_p () || lhs_range.undefined_p ())
+  // If op1 is undefined, solve as if it is varying.
+  if (op1_range.undefined_p ())
     {
-      r.set_undefined ();
-      return true;
+      int_range<2> trange (TREE_TYPE (gimple_range_operand1 (stmt)));
+      return gimple_range_handler (stmt)->op2_range (r, type, lhs_range,
+						     trange);
     }
   return gimple_range_handler (stmt)->op2_range (r, type, lhs_range,
 						 op1_range);
@@ -216,9 +226,6 @@ range_def_chain::get_imports (tree name)
   if (!has_def_chain (name))
     get_def_chain (name);
   bitmap i = m_def_chain[SSA_NAME_VERSION (name)].m_import;
-  // Either this is a default def,  OR imports must be a subset of exports.
-  gcc_checking_assert (!get_def_chain (name) || !i
-		       || !bitmap_intersect_compl_p (i, get_def_chain (name)));
   return i;
 }
 
@@ -644,7 +651,7 @@ gori_compute::gori_compute (int not_executable_flag)
   // Create a boolean_type true and false range.
   m_bool_zero = int_range<2> (boolean_false_node, boolean_false_node);
   m_bool_one = int_range<2> (boolean_true_node, boolean_true_node);
-  if (dump_file && (param_evrp_mode & EVRP_MODE_GORI))
+  if (dump_file && (param_ranger_debug & RANGER_DEBUG_GORI))
     tracer.enable_trace ();
 }
 
diff --git a/gcc/gimple-range-path.cc b/gcc/gimple-range-path.cc
index 6942713..4aa666d 100644
--- a/gcc/gimple-range-path.cc
+++ b/gcc/gimple-range-path.cc
@@ -34,23 +34,38 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-iterator.h"
 
 // Internal construct to help facilitate debugging of solver.
-#define DEBUG_SOLVER (dump_file && dump_flags & TDF_THREADING)
+#define DEBUG_SOLVER (dump_file && (param_threader_debug == THREADER_DEBUG_ALL))
 
-path_range_query::path_range_query (gimple_ranger &ranger, bool resolve)
-  : m_ranger (ranger)
+path_range_query::path_range_query (bool resolve, gimple_ranger *ranger)
+  : m_cache (new ssa_global_cache),
+    m_has_cache_entry (BITMAP_ALLOC (NULL)),
+    m_resolve (resolve),
+    m_alloced_ranger (!ranger)
 {
-  m_cache = new ssa_global_cache;
-  m_has_cache_entry = BITMAP_ALLOC (NULL);
-  m_path = NULL;
-  m_resolve = resolve;
-  m_oracle = new path_oracle (ranger.oracle ());
+  if (m_alloced_ranger)
+    m_ranger = new gimple_ranger;
+  else
+    m_ranger = ranger;
+
+  m_oracle = new path_oracle (m_ranger->oracle ());
 }
 
 path_range_query::~path_range_query ()
 {
+  delete m_oracle;
+  if (m_alloced_ranger)
+    delete m_ranger;
   BITMAP_FREE (m_has_cache_entry);
   delete m_cache;
-  delete m_oracle;
+}
+
+// Return TRUE if NAME is in the import bitmap.
+
+bool
+path_range_query::import_p (tree name)
+{
+  return (TREE_CODE (name) == SSA_NAME
+	  && bitmap_bit_p (m_imports, SSA_NAME_VERSION (name)));
 }
 
 // Mark cache entry for NAME as unused.
@@ -92,14 +107,13 @@ path_range_query::dump (FILE *dump_file)
 {
   push_dump_file save (dump_file, dump_flags & ~TDF_DETAILS);
 
-  if (m_path->is_empty ())
+  if (m_path.is_empty ())
     return;
 
   unsigned i;
   bitmap_iterator bi;
 
-  fprintf (dump_file, "\nPath is (length=%d):\n", m_path->length ());
-  dump_ranger (dump_file, *m_path);
+  dump_ranger (dump_file, m_path);
 
   fprintf (dump_file, "Imports:\n");
   EXECUTE_IF_SET_IN_BITMAP (m_imports, 0, i, bi)
@@ -126,7 +140,7 @@ path_range_query::defined_outside_path (tree name)
   gimple *def = SSA_NAME_DEF_STMT (name);
   basic_block bb = gimple_bb (def);
 
-  return !bb || !m_path->contains (bb);
+  return !bb || !m_path.contains (bb);
 }
 
 // Return the range of NAME on entry to the path.
@@ -135,16 +149,30 @@ void
 path_range_query::range_on_path_entry (irange &r, tree name)
 {
   gcc_checking_assert (defined_outside_path (name));
-  int_range_max tmp;
   basic_block entry = entry_bb ();
-  bool changed = false;
 
+  // Prefer to use range_of_expr if we have a statement to look at,
+  // since it has better caching than range_on_edge.
+  gimple *last = last_stmt (entry);
+  if (last)
+    {
+      if (m_ranger->range_of_expr (r, name, last))
+	return;
+      gcc_unreachable ();
+    }
+
+  // If we have no statement, look at all the incoming ranges to the
+  // block.  This can happen when we're querying a block with only an
+  // outgoing edge (no statement but the fall through edge), but for
+  // which we can determine a range on entry to the block.
+  int_range_max tmp;
+  bool changed = false;
   r.set_undefined ();
   for (unsigned i = 0; i < EDGE_COUNT (entry->preds); ++i)
     {
       edge e = EDGE_PRED (entry, i);
       if (e->src != ENTRY_BLOCK_PTR_FOR_FN (cfun)
-	  && m_ranger.range_on_edge (tmp, e, name))
+	  && m_ranger->range_on_edge (tmp, e, name))
 	{
 	  r.union_ (tmp);
 	  changed = true;
@@ -174,8 +202,8 @@ path_range_query::internal_range_of_expr (irange &r, tree name, gimple *stmt)
       return true;
     }
 
-  basic_block bb = stmt ? gimple_bb (stmt) : exit_bb ();
-  if (stmt && range_defined_in_block (r, name, bb))
+  if (stmt
+      && range_defined_in_block (r, name, gimple_bb (stmt)))
     {
       if (TREE_CODE (name) == SSA_NAME)
 	r.intersect (gimple_range_global (name));
@@ -184,7 +212,7 @@ path_range_query::internal_range_of_expr (irange &r, tree name, gimple *stmt)
       return true;
     }
 
-  r.set_varying (TREE_TYPE (name));
+  r = gimple_range_global (name);
   return true;
 }
 
@@ -217,41 +245,67 @@ void
 path_range_query::set_path (const vec<basic_block> &path)
 {
   gcc_checking_assert (path.length () > 1);
-  m_path = &path;
-  m_pos = m_path->length () - 1;
+  m_path = path.copy ();
+  m_pos = m_path.length () - 1;
   bitmap_clear (m_has_cache_entry);
 }
 
+bool
+path_range_query::ssa_defined_in_bb (tree name, basic_block bb)
+{
+  return (TREE_CODE (name) == SSA_NAME
+	  && SSA_NAME_DEF_STMT (name)
+	  && gimple_bb (SSA_NAME_DEF_STMT (name)) == bb);
+}
+
 // Return the range of the result of PHI in R.
+//
+// Since PHIs are calculated in parallel at the beginning of the
+// block, we must be careful to never save anything to the cache here.
+// It is the caller's responsibility to adjust the cache.  Also,
+// calculating the PHI's range must not trigger additional lookups.
 
 void
 path_range_query::ssa_range_in_phi (irange &r, gphi *phi)
 {
   tree name = gimple_phi_result (phi);
   basic_block bb = gimple_bb (phi);
+  unsigned nargs = gimple_phi_num_args (phi);
 
   if (at_entry ())
     {
-      if (m_resolve && m_ranger.range_of_expr (r, name, phi))
+      if (m_resolve && m_ranger->range_of_expr (r, name, phi))
 	return;
 
-      // Try fold just in case we can resolve simple things like PHI <5(99), 6(88)>.
-      if (!fold_range (r, phi, this))
-	r.set_varying (TREE_TYPE (name));
-
+      // Try to fold the phi exclusively with global or cached values.
+      // This will get things like PHI <5(99), 6(88)>.  We do this by
+      // calling range_of_expr with no context.
+      int_range_max arg_range;
+      r.set_undefined ();
+      for (size_t i = 0; i < nargs; ++i)
+	{
+	  tree arg = gimple_phi_arg_def (phi, i);
+	  if (range_of_expr (arg_range, arg, /*stmt=*/NULL))
+	    r.union_ (arg_range);
+	  else
+	    {
+	      r.set_varying (TREE_TYPE (name));
+	      return;
+	    }
+	}
       return;
     }
 
   basic_block prev = prev_bb ();
   edge e_in = find_edge (prev, bb);
-  unsigned nargs = gimple_phi_num_args (phi);
 
   for (size_t i = 0; i < nargs; ++i)
     if (e_in == gimple_phi_arg_edge (phi, i))
       {
 	tree arg = gimple_phi_arg_def (phi, i);
-
-	if (!get_cache (r, arg))
+	// Avoid using the cache for ARGs defined in this block, as
+	// that could create an ordering problem.
+	if (ssa_defined_in_bb (arg, bb) || !get_cache (r, arg))
 	  {
 	    if (m_resolve)
 	      {
@@ -263,7 +317,7 @@ path_range_query::ssa_range_in_phi (irange &r, gphi *phi)
 		  range_on_path_entry (r, arg);
 		else
 		  r.set_varying (TREE_TYPE (name));
-		m_ranger.range_on_edge (tmp, e_in, arg);
+		m_ranger->range_on_edge (tmp, e_in, arg);
 		r.intersect (tmp);
 		return;
 	      }
@@ -286,10 +340,19 @@ path_range_query::range_defined_in_block (irange &r, tree name, basic_block bb)
   if (def_bb != bb)
     return false;
 
+  if (get_cache (r, name))
+    return true;
+
   if (gimple_code (def_stmt) == GIMPLE_PHI)
     ssa_range_in_phi (r, as_a<gphi *> (def_stmt));
-  else if (!range_of_stmt (r, def_stmt, name))
-    r.set_varying (TREE_TYPE (name));
+  else
+    {
+      if (name)
+	get_path_oracle ()->killing_def (name);
+
+      if (!range_of_stmt (r, def_stmt, name))
+	r.set_varying (TREE_TYPE (name));
+    }
 
   if (bb)
     m_non_null.adjust_range (r, name, bb);
@@ -306,6 +369,38 @@ path_range_query::range_defined_in_block (irange &r, tree name, basic_block bb)
   return true;
 }
 
+// Compute ranges defined in the PHIs in this block.
+
+void
+path_range_query::compute_ranges_in_phis (basic_block bb)
+{
+  int_range_max r;
+  gphi_iterator iter;
+
+  // PHIs must be resolved simultaneously on entry to the block
+  // because any dependencies must be satistifed with values on entry.
+  // Thus, we calculate all PHIs first, and then update the cache at
+  // the end.
+
+  m_tmp_phi_cache.clear ();
+  for (iter = gsi_start_phis (bb); !gsi_end_p (iter); gsi_next (&iter))
+    {
+      gphi *phi = iter.phi ();
+      tree name = gimple_phi_result (phi);
+
+      if (import_p (name) && range_defined_in_block (r, name, bb))
+	m_tmp_phi_cache.set_global_range (name, r);
+    }
+  for (iter = gsi_start_phis (bb); !gsi_end_p (iter); gsi_next (&iter))
+    {
+      gphi *phi = iter.phi ();
+      tree name = gimple_phi_result (phi);
+
+      if (m_tmp_phi_cache.get_global_range (r, name))
+	set_cache (r, name);
+    }
+}
+
 // Compute ranges defined in the current block, or exported to the
 // next block.
 
@@ -316,24 +411,27 @@ path_range_query::compute_ranges_in_block (basic_block bb)
   int_range_max r, cached_range;
   unsigned i;
 
+  if (m_resolve && !at_entry ())
+    compute_phi_relations (bb, prev_bb ());
+
   // Force recalculation of any names in the cache that are defined in
   // this block.  This can happen on interdependent SSA/phis in loops.
   EXECUTE_IF_SET_IN_BITMAP (m_imports, 0, i, bi)
     {
       tree name = ssa_name (i);
-      gimple *def_stmt = SSA_NAME_DEF_STMT (name);
-      basic_block def_bb = gimple_bb (def_stmt);
-
-      if (def_bb == bb)
+      if (ssa_defined_in_bb (name, bb))
 	clear_cache (name);
     }
 
-  // Solve imports defined in this block.
+  // Solve imports defined in this block, starting with the PHIs...
+  compute_ranges_in_phis (bb);
+  // ...and then the rest of the imports.
   EXECUTE_IF_SET_IN_BITMAP (m_imports, 0, i, bi)
     {
       tree name = ssa_name (i);
 
-      if (range_defined_in_block (r, name, bb))
+      if (gimple_code (SSA_NAME_DEF_STMT (name)) != GIMPLE_PHI
+	  && range_defined_in_block (r, name, bb))
 	set_cache (r, name);
     }
 
@@ -341,11 +439,12 @@ path_range_query::compute_ranges_in_block (basic_block bb)
     return;
 
   // Solve imports that are exported to the next block.
-  edge e = find_edge (bb, next_bb ());
+  basic_block next = next_bb ();
+  edge e = find_edge (bb, next);
   EXECUTE_IF_SET_IN_BITMAP (m_imports, 0, i, bi)
     {
       tree name = ssa_name (i);
-      gori_compute &g = m_ranger.gori ();
+      gori_compute &g = m_ranger->gori ();
       bitmap exports = g.exports (bb);
 
       if (bitmap_bit_p (exports, i))
@@ -369,6 +468,9 @@ path_range_query::compute_ranges_in_block (basic_block bb)
 	    }
 	}
     }
+
+  if (m_resolve)
+    compute_outgoing_relations (bb, next);
 }
 
 // Adjust all pointer imports in BB with non-null information.
@@ -411,26 +513,32 @@ path_range_query::add_to_imports (tree name, bitmap imports)
   return false;
 }
 
-// Add the copies of any SSA names in IMPORTS to IMPORTS.
+// Compute the imports to the path ending in EXIT.  These are
+// essentially the SSA names used to calculate the final conditional
+// along the path.
 //
-// These are hints for the solver.  Adding more elements (within
-// reason) doesn't slow us down, because we don't solve anything that
-// doesn't appear in the path.  On the other hand, not having enough
-// imports will limit what we can solve.
+// They are hints for the solver.  Adding more elements doesn't slow
+// us down, because we don't solve anything that doesn't appear in the
+// path.  On the other hand, not having enough imports will limit what
+// we can solve.
 
 void
-path_range_query::add_copies_to_imports ()
+path_range_query::compute_imports (bitmap imports, basic_block exit)
 {
-  auto_vec<tree> worklist (bitmap_count_bits (m_imports));
+  // Start with the imports from the exit block...
+  bitmap r_imports = m_ranger->gori ().imports (exit);
+  bitmap_copy (imports, r_imports);
+
+  auto_vec<tree> worklist (bitmap_count_bits (imports));
   bitmap_iterator bi;
   unsigned i;
-
-  EXECUTE_IF_SET_IN_BITMAP (m_imports, 0, i, bi)
+  EXECUTE_IF_SET_IN_BITMAP (imports, 0, i, bi)
     {
       tree name = ssa_name (i);
       worklist.quick_push (name);
     }
 
+  // ...and add any operands used to define these imports.
   while (!worklist.is_empty ())
     {
       tree name = worklist.pop ();
@@ -438,15 +546,12 @@ path_range_query::add_copies_to_imports ()
 
       if (is_gimple_assign (def_stmt))
 	{
-	  // ?? Adding assignment copies doesn't get us much.  At the
-	  // time of writing, we got 63 more threaded paths across the
-	  // .ii files from a bootstrap.
-	  add_to_imports (gimple_assign_rhs1 (def_stmt), m_imports);
+	  add_to_imports (gimple_assign_rhs1 (def_stmt), imports);
 	  tree rhs = gimple_assign_rhs2 (def_stmt);
-	  if (rhs && add_to_imports (rhs, m_imports))
+	  if (rhs && add_to_imports (rhs, imports))
 	    worklist.safe_push (rhs);
 	  rhs = gimple_assign_rhs3 (def_stmt);
-	  if (rhs && add_to_imports (rhs, m_imports))
+	  if (rhs && add_to_imports (rhs, imports))
 	    worklist.safe_push (rhs);
 	}
       else if (gphi *phi = dyn_cast <gphi *> (def_stmt))
@@ -457,8 +562,8 @@ path_range_query::add_copies_to_imports ()
 	      tree arg = gimple_phi_arg (phi, i)->def;
 
 	      if (TREE_CODE (arg) == SSA_NAME
-		  && m_path->contains (e->src)
-		  && bitmap_set_bit (m_imports, SSA_NAME_VERSION (arg)))
+		  && m_path.contains (e->src)
+		  && bitmap_set_bit (imports, SSA_NAME_VERSION (arg)))
 		worklist.safe_push (arg);
 	    }
 	}
@@ -468,35 +573,36 @@ path_range_query::add_copies_to_imports ()
 // Compute the ranges for IMPORTS along PATH.
 //
 // IMPORTS are the set of SSA names, any of which could potentially
-// change the value of the final conditional in PATH.
+// change the value of the final conditional in PATH.  Default to the
+// imports of the last block in the path if none is given.
 
 void
 path_range_query::compute_ranges (const vec<basic_block> &path,
 				  const bitmap_head *imports)
 {
   if (DEBUG_SOLVER)
-    fprintf (dump_file, "\n*********** path_range_query ******************\n");
+    fprintf (dump_file, "\n==============================================\n");
 
   set_path (path);
-  bitmap_copy (m_imports, imports);
   m_undefined_path = false;
 
+  if (imports)
+    bitmap_copy (m_imports, imports);
+  else
+    compute_imports (m_imports, exit_bb ());
+
   if (m_resolve)
-    {
-      add_copies_to_imports ();
-      get_path_oracle ()->reset_path ();
-      compute_relations (path);
-    }
+    get_path_oracle ()->reset_path ();
 
   if (DEBUG_SOLVER)
     {
-      fprintf (dump_file, "\npath_range_query: compute_ranges for path: ");
+      fprintf (dump_file, "path_range_query: compute_ranges for path: ");
       for (unsigned i = path.length (); i > 0; --i)
 	{
 	  basic_block bb = path[i - 1];
-	  fprintf (dump_file, "BB %d", bb->index);
+	  fprintf (dump_file, "%d", bb->index);
 	  if (i > 1)
-	    fprintf (dump_file, ", ");
+	    fprintf (dump_file, "->");
 	}
       fprintf (dump_file, "\n");
     }
@@ -507,7 +613,7 @@ path_range_query::compute_ranges (const vec<basic_block> &path,
 
       if (m_resolve)
 	{
-	  gori_compute &gori = m_ranger.gori ();
+	  gori_compute &gori = m_ranger->gori ();
 	  tree name;
 
 	  // Exported booleans along the path, may help conditionals.
@@ -527,7 +633,22 @@ path_range_query::compute_ranges (const vec<basic_block> &path,
     }
 
   if (DEBUG_SOLVER)
-    dump (dump_file);
+    {
+      get_path_oracle ()->dump (dump_file);
+      dump (dump_file);
+    }
+}
+
+// Convenience function to compute ranges along a path consisting of
+// E->SRC and E->DEST.
+
+void
+path_range_query::compute_ranges (edge e)
+{
+  auto_vec<basic_block> bbs (2);
+  bbs.quick_push (e->dest);
+  bbs.quick_push (e->src);
+  compute_ranges (bbs);
 }
 
 // A folding aid used to register and query relations along a path.
@@ -607,66 +728,38 @@ path_range_query::range_of_stmt (irange &r, gimple *stmt, tree)
   if (!irange::supports_type_p (type))
     return false;
 
-  // If resolving unknowns, fold the statement as it would have
-  // appeared at the end of the path.
+  // If resolving unknowns, fold the statement making use of any
+  // relations along the path.
   if (m_resolve)
     {
       fold_using_range f;
-      jt_fur_source src (stmt, this, &m_ranger.gori (), *m_path);
+      jt_fur_source src (stmt, this, &m_ranger->gori (), m_path);
       if (!f.fold_stmt (r, stmt, src))
 	r.set_varying (type);
     }
-  // Otherwise, use the global ranger to fold it as it would appear in
-  // the original IL.  This is more conservative, but faster.
+  // Otherwise, fold without relations.
   else if (!fold_range (r, stmt, this))
     r.set_varying (type);
 
   return true;
 }
 
-// Compute relations on a path.  This involves two parts: relations
-// along the conditionals joining a path, and relations determined by
-// examining PHIs.
-
 void
-path_range_query::compute_relations (const vec<basic_block> &path)
+path_range_query::maybe_register_phi_relation (gphi *phi, tree arg)
 {
-  if (!dom_info_available_p (CDI_DOMINATORS))
-    return;
+  basic_block bb = gimple_bb (phi);
+  tree result = gimple_phi_result (phi);
 
-  jt_fur_source src (NULL, this, &m_ranger.gori (), path);
-  basic_block prev = NULL;
-  for (unsigned i = path.length (); i > 0; --i)
-    {
-      basic_block bb = path[i - 1];
-      gimple *stmt = last_stmt (bb);
+  // Avoid recording the equivalence if the arg is defined in this
+  // block, as that could create an ordering problem.
+  if (ssa_defined_in_bb (arg, bb))
+    return;
 
-      compute_phi_relations (bb, prev);
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    fprintf (dump_file, "  from bb%d:", bb->index);
 
-      // Compute relations in outgoing edges along the path.  Skip the
-      // final conditional which we don't know yet.
-      if (i > 1
-	  && stmt
-	  && gimple_code (stmt) == GIMPLE_COND
-	  && irange::supports_type_p (TREE_TYPE (gimple_cond_lhs (stmt))))
-	{
-	  basic_block next = path[i - 2];
-	  int_range<2> r;
-	  gcond *cond = as_a<gcond *> (stmt);
-	  edge e0 = EDGE_SUCC (bb, 0);
-	  edge e1 = EDGE_SUCC (bb, 1);
-
-	  if (e0->dest == next)
-	    gcond_edge_range (r, e0);
-	  else if (e1->dest == next)
-	    gcond_edge_range (r, e1);
-	  else
-	    gcc_unreachable ();
-
-	  src.register_outgoing_edges (cond, r, e0, e1);
-	}
-      prev = bb;
-    }
+  get_path_oracle ()->killing_def (result);
+  m_oracle->register_relation (entry_bb (), EQ_EXPR, arg, result);
 }
 
 // Compute relations for each PHI in BB.  For example:
@@ -681,9 +774,7 @@ path_range_query::compute_phi_relations (basic_block bb, basic_block prev)
   if (prev == NULL)
     return;
 
-  basic_block entry = entry_bb ();
   edge e_in = find_edge (prev, bb);
-  gcc_checking_assert (e_in);
 
   for (gphi_iterator iter = gsi_start_phis (bb); !gsi_end_p (iter);
        gsi_next (&iter))
@@ -692,15 +783,46 @@ path_range_query::compute_phi_relations (basic_block bb, basic_block prev)
       tree result = gimple_phi_result (phi);
       unsigned nargs = gimple_phi_num_args (phi);
 
+      if (!import_p (result))
+	continue;
+
       for (size_t i = 0; i < nargs; ++i)
 	if (e_in == gimple_phi_arg_edge (phi, i))
 	  {
 	    tree arg = gimple_phi_arg_def (phi, i);
 
 	    if (gimple_range_ssa_p (arg))
-	      m_oracle->register_relation (entry, EQ_EXPR, arg, result);
-
+	      maybe_register_phi_relation (phi, arg);
 	    break;
 	  }
     }
 }
+
+// Compute outgoing relations from BB to NEXT.
+
+void
+path_range_query::compute_outgoing_relations (basic_block bb, basic_block next)
+{
+  gimple *stmt = last_stmt (bb);
+
+  if (stmt
+      && gimple_code (stmt) == GIMPLE_COND
+      && (import_p (gimple_cond_lhs (stmt))
+	  || import_p (gimple_cond_rhs (stmt))))
+    {
+      int_range<2> r;
+      gcond *cond = as_a<gcond *> (stmt);
+      edge e0 = EDGE_SUCC (bb, 0);
+      edge e1 = EDGE_SUCC (bb, 1);
+
+      if (e0->dest == next)
+	gcond_edge_range (r, e0);
+      else if (e1->dest == next)
+	gcond_edge_range (r, e1);
+      else
+	gcc_unreachable ();
+
+      jt_fur_source src (NULL, this, &m_ranger->gori (), m_path);
+      src.register_outgoing_edges (cond, r, e0, e1);
+    }
+}
diff --git a/gcc/gimple-range-path.h b/gcc/gimple-range-path.h
index 5f4e73a..57a9ae9 100644
--- a/gcc/gimple-range-path.h
+++ b/gcc/gimple-range-path.h
@@ -32,9 +32,12 @@ along with GCC; see the file COPYING3.  If not see
 class path_range_query : public range_query
 {
 public:
-  path_range_query (class gimple_ranger &ranger, bool resolve);
+  path_range_query (bool resolve = true, class gimple_ranger *ranger = NULL);
   virtual ~path_range_query ();
-  void compute_ranges (const vec<basic_block> &, const bitmap_head *imports);
+  void compute_ranges (const vec<basic_block> &,
+		       const bitmap_head *imports = NULL);
+  void compute_ranges (edge e);
+  void compute_imports (bitmap imports, basic_block exit);
   bool range_of_expr (irange &r, tree name, gimple * = NULL) override;
   bool range_of_stmt (irange &r, gimple *, tree name = NULL) override;
   bool unreachable_path_p ();
@@ -55,35 +58,40 @@ private:
   // Methods to compute ranges for the given path.
   bool range_defined_in_block (irange &, tree name, basic_block bb);
   void compute_ranges_in_block (basic_block bb);
+  void compute_ranges_in_phis (basic_block bb);
   void adjust_for_non_null_uses (basic_block bb);
   void ssa_range_in_phi (irange &r, gphi *phi);
-  void compute_relations (const vec<basic_block> &);
+  void compute_outgoing_relations (basic_block bb, basic_block next);
   void compute_phi_relations (basic_block bb, basic_block prev);
-  void add_copies_to_imports ();
+  void maybe_register_phi_relation (gphi *, tree arg);
   bool add_to_imports (tree name, bitmap imports);
+  bool import_p (tree name);
+  bool ssa_defined_in_bb (tree name, basic_block bb);
 
   // Path navigation.
   void set_path (const vec<basic_block> &);
-  basic_block entry_bb () { return (*m_path)[m_path->length () - 1]; }
-  basic_block exit_bb ()  { return (*m_path)[0]; }
-  basic_block curr_bb ()  { return (*m_path)[m_pos]; }
-  basic_block prev_bb ()  { return (*m_path)[m_pos + 1]; }
-  basic_block next_bb ()  { return (*m_path)[m_pos - 1]; }
-  bool at_entry ()	  { return m_pos == m_path->length () - 1; }
+  basic_block entry_bb () { return m_path[m_path.length () - 1]; }
+  basic_block exit_bb ()  { return m_path[0]; }
+  basic_block curr_bb ()  { return m_path[m_pos]; }
+  basic_block prev_bb ()  { return m_path[m_pos + 1]; }
+  basic_block next_bb ()  { return m_path[m_pos - 1]; }
+  bool at_entry ()	  { return m_pos == m_path.length () - 1; }
   bool at_exit ()	  { return m_pos == 0; }
   void move_next ()	  { --m_pos; }
 
   // Range cache for SSA names.
   ssa_global_cache *m_cache;
 
+  ssa_global_cache m_tmp_phi_cache;
+
   // Set for each SSA that has an active entry in the cache.
   bitmap m_has_cache_entry;
 
   // Path being analyzed.
-  const vec<basic_block> *m_path;
+  auto_vec<basic_block> m_path;
 
   auto_bitmap m_imports;
-  gimple_ranger &m_ranger;
+  gimple_ranger *m_ranger;
   non_null_ref m_non_null;
 
   // Current path position.
@@ -94,6 +102,10 @@ private:
 
   // Set if there were any undefined expressions while pre-calculating path.
   bool m_undefined_path;
+
+  // True if m_ranger was allocated in this class and must be freed at
+  // destruction.
+  bool m_alloced_ranger;
 };
 
 #endif // GCC_TREE_SSA_THREADSOLVER_H
diff --git a/gcc/gimple-range.cc b/gcc/gimple-range.cc
index 93d6da6..a2b68b2 100644
--- a/gcc/gimple-range.cc
+++ b/gcc/gimple-range.cc
@@ -34,15 +34,17 @@ along with GCC; see the file COPYING3.  If not see
 #include "cfgloop.h"
 #include "tree-scalar-evolution.h"
 #include "gimple-range.h"
+#include "gimple-fold.h"
 
 gimple_ranger::gimple_ranger () :
 	non_executable_edge_flag (cfun),
 	m_cache (non_executable_edge_flag),
-	tracer ("")
+	tracer (""),
+	current_bb (NULL)
 {
   // If the cache has a relation oracle, use it.
   m_oracle = m_cache.oracle ();
-  if (dump_file && (param_evrp_mode & EVRP_MODE_TRACE))
+  if (dump_file && (param_ranger_debug & RANGER_DEBUG_TRACE))
     tracer.enable_trace ();
 
   // Ensure the not_executable flag is clear everywhere.
@@ -82,8 +84,19 @@ gimple_ranger::range_of_expr (irange &r, tree expr, gimple *stmt)
   // If there is no statement, just get the global value.
   if (!stmt)
     {
+      int_range_max tmp;
       if (!m_cache.get_global_range (r, expr))
         r = gimple_range_global (expr);
+      // Pick up implied context information from the on-entry cache
+      // if current_bb is set.  Do not attempt any new calculations.
+      if (current_bb && m_cache.block_range (tmp, current_bb, expr, false))
+	{
+	  r.intersect (tmp);
+	  char str[80];
+	  sprintf (str, "picked up range from bb %d\n",current_bb->index);
+	  if (idx)
+	    tracer.print (idx, str);
+	}
     }
   // For a debug stmt, pick the best value currently available, do not
   // trigger new value calculations.  PR 100781.
@@ -256,7 +269,17 @@ gimple_ranger::range_of_stmt (irange &r, gimple *s, tree name)
 
   // If no name, simply call the base routine.
   if (!name)
-    res = fold_range_internal (r, s, NULL_TREE);
+    {
+      res = fold_range_internal (r, s, NULL_TREE);
+      if (res && is_a <gcond *> (s))
+	{
+	  // Update any exports in the cache if this is a gimple cond statement.
+	  tree exp;
+	  basic_block bb = gimple_bb (s);
+	  FOR_EACH_GORI_EXPORT_NAME (m_cache.m_gori, bb, exp)
+	    m_cache.propagate_updated_value (exp, bb);
+	}
+    }
   else if (!gimple_range_ssa_p (name))
     res = get_tree_range (r, name, NULL);
   // Check if the stmt has already been processed, and is not stale.
@@ -285,6 +308,19 @@ gimple_ranger::range_of_stmt (irange &r, gimple *s, tree name)
   return res;
 }
 
+// This routine will invoke the gimple fold_stmt routine, providing context to
+// range_of_expr calls via an private interal API.
+
+bool
+gimple_ranger::fold_stmt (gimple_stmt_iterator *gsi, tree (*valueize) (tree))
+{
+  gimple *stmt = gsi_stmt (*gsi);
+  current_bb = gimple_bb (stmt);
+  bool ret = ::fold_stmt (gsi, valueize);
+  current_bb = NULL;
+  return ret;
+}
+
 // This routine will export whatever global ranges are known to GCC
 // SSA_RANGE_NAME_INFO and SSA_NAME_PTR_INFO fields.
 
@@ -303,7 +339,7 @@ gimple_ranger::export_global_ranges ()
 	  && !r.varying_p())
 	{
 	  bool updated = update_global_range (r, name);
-	  if (!updated || !dump_file || !(dump_flags & TDF_DETAILS))
+	  if (!updated || !dump_file)
 	    continue;
 
 	  if (print_header)
@@ -431,6 +467,7 @@ enable_ranger (struct function *fun)
 {
   gimple_ranger *r;
 
+  gcc_checking_assert (!fun->x_range_query);
   r = new gimple_ranger;
   fun->x_range_query = r;
 
@@ -443,7 +480,7 @@ enable_ranger (struct function *fun)
 void
 disable_ranger (struct function *fun)
 {
+  gcc_checking_assert (fun->x_range_query);
   delete fun->x_range_query;
-
-  fun->x_range_query = &global_ranges;
+  fun->x_range_query = NULL;
 }
diff --git a/gcc/gimple-range.h b/gcc/gimple-range.h
index 0713af4..615496e 100644
--- a/gcc/gimple-range.h
+++ b/gcc/gimple-range.h
@@ -58,10 +58,12 @@ public:
   void debug ();
   void dump_bb (FILE *f, basic_block bb);
   auto_edge_flag non_executable_edge_flag;
+  bool fold_stmt (gimple_stmt_iterator *gsi, tree (*) (tree));
 protected:
   bool fold_range_internal (irange &r, gimple *s, tree name);
   ranger_cache m_cache;
   range_tracer tracer;
+  basic_block current_bb;
 };
 
 /* Create a new ranger instance and associate it with a function.
diff --git a/gcc/gimple-ssa-evrp.c b/gcc/gimple-ssa-evrp.c
index 7f20555..c3f8405 100644
--- a/gcc/gimple-ssa-evrp.c
+++ b/gcc/gimple-ssa-evrp.c
@@ -44,6 +44,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-range.h"
 #include "fold-const.h"
 #include "value-pointer-equiv.h"
+#include "tree-vrp.h"
 
 // This is the classic EVRP folder which uses a dominator walk and pushes
 // ranges into the next block if it is a single predecessor block.
@@ -110,88 +111,6 @@ protected:
   simplify_using_ranges simplifier;
 };
 
-// This is a ranger based folder which continues to use the dominator
-// walk to access the substitute and fold machinery.  Ranges are calculated
-// on demand.
-
-class rvrp_folder : public substitute_and_fold_engine
-{
-public:
-
-  rvrp_folder () : substitute_and_fold_engine (), m_simplifier ()
-  {
-    m_ranger = enable_ranger (cfun);
-    m_simplifier.set_range_query (m_ranger, m_ranger->non_executable_edge_flag);
-    m_pta = new pointer_equiv_analyzer (m_ranger);
-  }
-      
-  ~rvrp_folder ()
-  {
-    if (dump_file && (dump_flags & TDF_DETAILS))
-      m_ranger->dump (dump_file);
-
-    m_ranger->export_global_ranges ();
-    disable_ranger (cfun);
-    delete m_pta;
-  }
-
-  tree value_of_expr (tree name, gimple *s = NULL) OVERRIDE
-  {
-    // Shortcircuit subst_and_fold callbacks for abnormal ssa_names.
-    if (TREE_CODE (name) == SSA_NAME && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name))
-      return NULL;
-    tree ret = m_ranger->value_of_expr (name, s);
-    if (!ret && supported_pointer_equiv_p (name))
-      ret = m_pta->get_equiv (name);
-    return ret;
-  }
-
-  tree value_on_edge (edge e, tree name) OVERRIDE
-  {
-    // Shortcircuit subst_and_fold callbacks for abnormal ssa_names.
-    if (TREE_CODE (name) == SSA_NAME && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name))
-      return NULL;
-    tree ret = m_ranger->value_on_edge (e, name);
-    if (!ret && supported_pointer_equiv_p (name))
-      ret = m_pta->get_equiv (name);
-    return ret;
-  }
-
-  tree value_of_stmt (gimple *s, tree name = NULL) OVERRIDE
-  {
-    // Shortcircuit subst_and_fold callbacks for abnormal ssa_names.
-    if (TREE_CODE (name) == SSA_NAME && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name))
-      return NULL;
-    return m_ranger->value_of_stmt (s, name);
-  }
-
-  void pre_fold_bb (basic_block bb) OVERRIDE
-  {
-    m_pta->enter (bb);
-  }
-
-  void post_fold_bb (basic_block bb) OVERRIDE
-  {
-    m_pta->leave (bb);
-  }
-
-  void pre_fold_stmt (gimple *stmt) OVERRIDE
-  {
-    m_pta->visit_stmt (stmt);
-  }
-
-  bool fold_stmt (gimple_stmt_iterator *gsi) OVERRIDE
-  {
-    return m_simplifier.simplify (gsi);
-  }
-
-private:
-  DISABLE_COPY_AND_ASSIGN (rvrp_folder);
-  gimple_ranger *m_ranger;
-  simplify_using_ranges m_simplifier;
-  pointer_equiv_analyzer *m_pta;
-};
-
 // In a hybrid folder, start with an EVRP folder, and add the required
 // fold_stmt bits to either try the ranger first or second.
 //
@@ -381,7 +300,7 @@ hybrid_folder::choose_value (tree evrp_val, tree ranger_val)
     return evrp_val;
 
   // If values are different, return the first calculated value.
-  if ((param_evrp_mode & EVRP_MODE_RVRP_FIRST) == EVRP_MODE_RVRP_FIRST)
+  if (param_evrp_mode == EVRP_MODE_RVRP_FIRST)
     return ranger_val;
   return evrp_val;
 }
@@ -393,6 +312,9 @@ hybrid_folder::choose_value (tree evrp_val, tree ranger_val)
 static unsigned int
 execute_early_vrp ()
 {
+  if (param_evrp_mode == EVRP_MODE_RVRP_ONLY)
+    return execute_ranger_vrp (cfun, false);
+
   /* Ideally this setup code would move into the ctor for the folder
      However, this setup can change the number of blocks which
      invalidates the internal arrays that are set up by the dominator
@@ -403,7 +325,7 @@ execute_early_vrp ()
   calculate_dominance_info (CDI_DOMINATORS);
 
   // Only the last 2 bits matter for choosing the folder.
-  switch (param_evrp_mode & EVRP_MODE_RVRP_FIRST)
+  switch (param_evrp_mode)
     {
     case EVRP_MODE_EVRP_ONLY:
       {
@@ -411,12 +333,6 @@ execute_early_vrp ()
 	folder.substitute_and_fold ();
 	break;
       }
-    case EVRP_MODE_RVRP_ONLY:
-      {
-	rvrp_folder folder;
-	folder.substitute_and_fold ();
-	break;
-      }
     case EVRP_MODE_EVRP_FIRST:
       {
 	hybrid_folder folder (true);
diff --git a/gcc/gimple-ssa-sprintf.c b/gcc/gimple-ssa-sprintf.c
index 8e90b7c..ec78090 100644
--- a/gcc/gimple-ssa-sprintf.c
+++ b/gcc/gimple-ssa-sprintf.c
@@ -2024,8 +2024,8 @@ format_floating (const directive &dir, tree arg, range_query *)
    Used by the format_string function below.  */
 
 static fmtresult
-get_string_length (tree str, gimple *stmt, unsigned eltsize,
-		   range_query *query)
+get_string_length (tree str, gimple *stmt, unsigned HOST_WIDE_INT max_size,
+		   unsigned eltsize, range_query *query)
 {
   if (!str)
     return fmtresult ();
@@ -2065,6 +2065,20 @@ get_string_length (tree str, gimple *stmt, unsigned eltsize,
       && (!lendata.maxbound || lenmax <= tree_to_uhwi (lendata.maxbound))
       && lenmax <= tree_to_uhwi (lendata.maxlen))
     {
+      if (max_size > 0 && max_size < HOST_WIDE_INT_MAX)
+	{
+	  /* Adjust the conservative unknown/unbounded result if MAX_SIZE
+	     is valid.  Set UNLIKELY to maximum in case MAX_SIZE refers
+	     to a subobject.
+	     TODO: This is overly conservative.  Set UNLIKELY to the size
+	     of the outermost enclosing declared object.  */
+	  fmtresult res (0, max_size - 1);
+	  res.nonstr = lendata.decl;
+	  res.range.likely = res.range.max;
+	  res.range.unlikely = HOST_WIDE_INT_MAX;
+	  return res;
+	}
+
       fmtresult res;
       res.nonstr = lendata.decl;
       return res;
@@ -2203,110 +2217,80 @@ format_character (const directive &dir, tree arg, range_query *query)
   return res.adjust_for_width_or_precision (dir.width);
 }
 
-/* Determine the offset *INDEX of the first byte of an array element of
-   TYPE (possibly recursively) into which the byte offset OFF points.
-   On success set *INDEX to the offset of the first byte and return type.
-   Otherwise, if no such element can be found, return null.  */
+/* If TYPE is an array or struct or union, increment *FLDOFF by the starting
+   offset of the member that *OFF point into and set *FLDSIZE to its size
+   in bytes and decrement *OFF by the same.  Otherwise do nothing.  */
 
-static tree
-array_elt_at_offset (tree type, HOST_WIDE_INT off, HOST_WIDE_INT *index)
+static void
+set_aggregate_size_and_offset (tree type, HOST_WIDE_INT *fldoff,
+			       HOST_WIDE_INT *fldsize, HOST_WIDE_INT *off)
 {
-  gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
-
-  tree eltype = type;
-  while (TREE_CODE (TREE_TYPE (eltype)) == ARRAY_TYPE)
-    eltype = TREE_TYPE (eltype);
-
-  if (TYPE_MODE (TREE_TYPE (eltype)) != TYPE_MODE (char_type_node))
-    eltype = TREE_TYPE (eltype);
-
-  if (eltype == type)
-    {
-      *index = 0;
-      return type;
-    }
-
-  HOST_WIDE_INT typsz = int_size_in_bytes (type);
-  HOST_WIDE_INT eltsz = int_size_in_bytes (eltype);
-  if (off < typsz * eltsz)
+  /* The byte offset of the most basic struct member the byte
+     offset *OFF corresponds to, or for a (multidimensional)
+     array member, the byte offset of the array element.  */
+  if (TREE_CODE (type) == ARRAY_TYPE
+      && TREE_CODE (TREE_TYPE (type)) == ARRAY_TYPE)
     {
-      *index = (off / eltsz) * eltsz;
-      return TREE_CODE (eltype) == ARRAY_TYPE ? TREE_TYPE (eltype) : eltype;
+      HOST_WIDE_INT index = 0, arrsize = 0;
+      if (array_elt_at_offset (type, *off, &index, &arrsize))
+	{
+	  *fldoff += index;
+	  *off -= index;
+	  *fldsize = arrsize;
+	}
     }
-
-  return NULL_TREE;
-}
-
-/* Determine the offset *INDEX of the first byte of a struct member of TYPE
-   (possibly recursively) into which the byte offset OFF points.  On success
-   set *INDEX to the offset of the first byte and return true.  Otherwise,
-   if no such member can be found, return false.  */
-
-static bool
-field_at_offset (tree type, HOST_WIDE_INT off, HOST_WIDE_INT *index)
-{
-  gcc_assert (RECORD_OR_UNION_TYPE_P (type));
-
-  for (tree fld = TYPE_FIELDS (type); fld; fld = TREE_CHAIN (fld))
+  else if (RECORD_OR_UNION_TYPE_P (type))
     {
-      if (TREE_CODE (fld) != FIELD_DECL || DECL_ARTIFICIAL (fld))
-	continue;
-
-      tree fldtype = TREE_TYPE (fld);
-      HOST_WIDE_INT fldoff = int_byte_position (fld);
-
-      /* If the size is not available the field is a flexible array
-	 member.  Treat this case as success.  */
-      tree typesize = TYPE_SIZE_UNIT (fldtype);
-      HOST_WIDE_INT fldsize = (tree_fits_uhwi_p (typesize)
-			       ? tree_to_uhwi (typesize)
-			       : off);
-
-      if (fldoff + fldsize < off)
-	continue;
-
-      if (TREE_CODE (fldtype) == ARRAY_TYPE)
+      HOST_WIDE_INT index = 0;
+      tree sub = field_at_offset (type, NULL_TREE, *off, &index);
+      if (sub)
 	{
-	  HOST_WIDE_INT idx = 0;
-	  if (tree ft = array_elt_at_offset (fldtype, off, &idx))
-	    fldtype = ft;
+	  tree subsize = DECL_SIZE_UNIT (sub);
+	  if (*fldsize < HOST_WIDE_INT_MAX
+	      && subsize
+	      && tree_fits_uhwi_p (subsize))
+	    *fldsize = tree_to_uhwi (subsize);
 	  else
-	    break;
-
-	  *index += idx;
-	  fldoff -= idx;
-	  off -= idx;
-	}
-
-      if (RECORD_OR_UNION_TYPE_P (fldtype))
-	{
-	  *index += fldoff;
-	  return field_at_offset (fldtype, off - fldoff, index);
+	    *fldsize = HOST_WIDE_INT_MAX;
+	  *fldoff += index;
+	  *off -= index;
 	}
-
-      *index += fldoff;
-      return true;
     }
-
-  return false;
 }
 
 /* For an expression X of pointer type, recursively try to find the same
-   origin (object or pointer) as Y it references and return such an X.
-   When X refers to a struct member, set *FLDOFF to the offset of the
-   member from the beginning of the "most derived" object.  */
+   origin (object or pointer) as Y it references and return such a Y.
+   When X refers to an array element or struct member, set *FLDOFF to
+   the offset of the element or member from the beginning of the "most
+   derived" object and *FLDSIZE to its size.  When nonnull, set *OFF to
+   the overall offset from the beginning of the object so that
+   *FLDOFF <= *OFF.  */
 
 static tree
-get_origin_and_offset (tree x, HOST_WIDE_INT *fldoff, HOST_WIDE_INT *off)
+get_origin_and_offset_r (tree x, HOST_WIDE_INT *fldoff, HOST_WIDE_INT *fldsize,
+			 HOST_WIDE_INT *off)
 {
   if (!x)
     return NULL_TREE;
 
+  HOST_WIDE_INT sizebuf = -1;
+  if (!fldsize)
+    fldsize = &sizebuf;
+
+  if (DECL_P (x))
+    {
+      /* Set the size if it hasn't been set yet.  */
+      if (tree size = DECL_SIZE_UNIT (x))
+	if (*fldsize < 0 && tree_fits_shwi_p (size))
+	  *fldsize = tree_to_shwi (size);
+      return x;
+    }
+
   switch (TREE_CODE (x))
     {
     case ADDR_EXPR:
       x = TREE_OPERAND (x, 0);
-      return get_origin_and_offset (x, fldoff, off);
+      return get_origin_and_offset_r (x, fldoff, fldsize, off);
 
     case ARRAY_REF:
       {
@@ -2326,7 +2310,7 @@ get_origin_and_offset (tree x, HOST_WIDE_INT *fldoff, HOST_WIDE_INT *off)
 	  *fldoff = idx;
 
 	x = TREE_OPERAND (x, 0);
-	return get_origin_and_offset (x, fldoff, NULL);
+	return get_origin_and_offset_r (x, fldoff, fldsize, nullptr);
       }
 
     case MEM_REF:
@@ -2345,32 +2329,19 @@ get_origin_and_offset (tree x, HOST_WIDE_INT *fldoff, HOST_WIDE_INT *off)
 	    = (TREE_CODE (x) == ADDR_EXPR
 	       ? TREE_TYPE (TREE_OPERAND (x, 0)) : TREE_TYPE (TREE_TYPE (x)));
 
-	  /* The byte offset of the most basic struct member the byte
-	     offset *OFF corresponds to, or for a (multidimensional)
-	     array member, the byte offset of the array element.  */
-	  HOST_WIDE_INT index = 0;
-
-	  if ((RECORD_OR_UNION_TYPE_P (xtype)
-	       && field_at_offset (xtype, *off, &index))
-	      || (TREE_CODE (xtype) == ARRAY_TYPE
-		  && TREE_CODE (TREE_TYPE (xtype)) == ARRAY_TYPE
-		  && array_elt_at_offset (xtype, *off, &index)))
-	    {
-	      *fldoff += index;
-	      *off -= index;
-	    }
+	  set_aggregate_size_and_offset (xtype, fldoff, fldsize, off);
 	}
 
-      return get_origin_and_offset (x, fldoff, NULL);
+      return get_origin_and_offset_r (x, fldoff, fldsize, nullptr);
 
     case COMPONENT_REF:
       {
 	tree fld = TREE_OPERAND (x, 1);
 	*fldoff += int_byte_position (fld);
 
-	get_origin_and_offset (fld, fldoff, off);
+	get_origin_and_offset_r (fld, fldoff, fldsize, off);
 	x = TREE_OPERAND (x, 0);
-	return get_origin_and_offset (x, fldoff, off);
+	return get_origin_and_offset_r (x, fldoff, nullptr, off);
       }
 
     case SSA_NAME:
@@ -2382,27 +2353,41 @@ get_origin_and_offset (tree x, HOST_WIDE_INT *fldoff, HOST_WIDE_INT *off)
 	    if (code == ADDR_EXPR)
 	      {
 		x = gimple_assign_rhs1 (def);
-		return get_origin_and_offset (x, fldoff, off);
+		return get_origin_and_offset_r (x, fldoff, fldsize, off);
 	      }
 
 	    if (code == POINTER_PLUS_EXPR)
 	      {
 		tree offset = gimple_assign_rhs2 (def);
-		if (off)
-		  *off = (tree_fits_uhwi_p (offset)
-			  ? tree_to_uhwi (offset) : HOST_WIDE_INT_MAX);
+		if (off && tree_fits_uhwi_p (offset))
+		  *off = tree_to_uhwi (offset);
 
 		x = gimple_assign_rhs1 (def);
-		return get_origin_and_offset (x, fldoff, NULL);
+		x = get_origin_and_offset_r (x, fldoff, fldsize, off);
+		if (off && !tree_fits_uhwi_p (offset))
+		  *off = HOST_WIDE_INT_MAX;
+		if (off)
+		  {
+		    tree xtype = TREE_TYPE (x);
+		    set_aggregate_size_and_offset (xtype, fldoff, fldsize, off);
+		  }
+		return x;
 	      }
 	    else if (code == VAR_DECL)
 	      {
 		x = gimple_assign_rhs1 (def);
-		return get_origin_and_offset (x, fldoff, off);
+		return get_origin_and_offset_r (x, fldoff, fldsize, off);
 	      }
 	  }
 	else if (gimple_nop_p (def) && SSA_NAME_VAR (x))
 	  x = SSA_NAME_VAR (x);
+
+	tree xtype = TREE_TYPE (x);
+	if (POINTER_TYPE_P (xtype))
+	  xtype = TREE_TYPE (xtype);
+
+	if (off)
+	  set_aggregate_size_and_offset (xtype, fldoff, fldsize, off);
       }
 
     default:
@@ -2412,13 +2397,41 @@ get_origin_and_offset (tree x, HOST_WIDE_INT *fldoff, HOST_WIDE_INT *off)
   return x;
 }
 
+/* Nonrecursive version of the above.  */
+
+static tree
+get_origin_and_offset (tree x, HOST_WIDE_INT *fldoff, HOST_WIDE_INT *off,
+		       HOST_WIDE_INT *fldsize = nullptr)
+{
+  HOST_WIDE_INT sizebuf;
+  if (!fldsize)
+    fldsize = &sizebuf;
+
+  *fldsize = -1;
+
+  *fldoff = *off = *fldsize = 0;
+  tree orig = get_origin_and_offset_r (x, fldoff, fldsize, off);
+  if (!orig)
+    return NULL_TREE;
+
+  if (!*fldoff && *off == *fldsize)
+    {
+      *fldoff = *off;
+      *off = 0;
+    }
+
+  return orig;
+}
+
 /* If ARG refers to the same (sub)object or array element as described
    by DST and DST_FLD, return the byte offset into the struct member or
-   array element referenced by ARG.  Otherwise return HOST_WIDE_INT_MIN
-   to indicate that ARG and DST do not refer to the same object.  */
+   array element referenced by ARG and set *ARG_SIZE to the size of
+   the (sub)object.  Otherwise return HOST_WIDE_INT_MIN to indicate
+   that ARG and DST do not refer to the same object.  */
 
 static HOST_WIDE_INT
-alias_offset (tree arg, tree dst, HOST_WIDE_INT dst_fld)
+alias_offset (tree arg, HOST_WIDE_INT *arg_size,
+	      tree dst, HOST_WIDE_INT dst_fld)
 {
   /* See if the argument refers to the same base object as the destination
      of the formatted function call, and if so, try to determine if they
@@ -2430,7 +2443,7 @@ alias_offset (tree arg, tree dst, HOST_WIDE_INT dst_fld)
      to a struct member, see if the members are one and the same.  */
   HOST_WIDE_INT arg_off = 0, arg_fld = 0;
 
-  tree arg_orig = get_origin_and_offset (arg, &arg_fld, &arg_off);
+  tree arg_orig = get_origin_and_offset (arg, &arg_fld, &arg_off, arg_size);
 
   if (arg_orig == dst && arg_fld == dst_fld)
     return arg_off;
@@ -2448,6 +2461,10 @@ format_string (const directive &dir, tree arg, range_query *query)
 {
   fmtresult res;
 
+  /* The size of the (sub)object ARG refers to.  Used to adjust
+     the conservative get_string_length() result.  */
+  HOST_WIDE_INT arg_size = 0;
+
   if (warn_restrict)
     {
       /* See if ARG might alias the destination of the call with
@@ -2455,8 +2472,12 @@ format_string (const directive &dir, tree arg, range_query *query)
 	 so that the overlap can be determined for certain later,
 	 when the amount of output of the call (including subsequent
 	 directives) has been computed.  Otherwise, store HWI_MIN.  */
-      res.dst_offset = alias_offset (arg, dir.info->dst_origin,
+      res.dst_offset = alias_offset (arg, &arg_size, dir.info->dst_origin,
 				     dir.info->dst_field);
+      if (res.dst_offset >= 0 && res.dst_offset <= arg_size)
+	arg_size -= res.dst_offset;
+      else
+	arg_size = 0;
     }
 
   /* Compute the range the argument's length can be in.  */
@@ -2473,7 +2494,8 @@ format_string (const directive &dir, tree arg, range_query *query)
       gcc_checking_assert (count_by == 2 || count_by == 4);
     }
 
-  fmtresult slen = get_string_length (arg, dir.info->callstmt, count_by, query);
+  fmtresult slen =
+    get_string_length (arg, dir.info->callstmt, arg_size, count_by, query);
   if (slen.range.min == slen.range.max
       && slen.range.min < HOST_WIDE_INT_MAX)
     {
@@ -4030,11 +4052,11 @@ compute_format_length (call_info &info, format_result *res, range_query *query)
   return success;
 }
 
-/* Return the size of the object referenced by the expression DEST if
-   available, or the maximum possible size otherwise.  */
+/* Return the size of the object referenced by the expression DEST in
+   statement STMT, if available, or the maximum possible size otherwise.  */
 
 static unsigned HOST_WIDE_INT
-get_destination_size (tree dest, pointer_query &ptr_qry)
+get_destination_size (tree dest, gimple *stmt, pointer_query &ptr_qry)
 {
   /* When there is no destination return the maximum.  */
   if (!dest)
@@ -4042,7 +4064,7 @@ get_destination_size (tree dest, pointer_query &ptr_qry)
 
   /* Use compute_objsize to determine the size of the destination object.  */
   access_ref aref;
-  if (!ptr_qry.get_ref (dest, &aref))
+  if (!ptr_qry.get_ref (dest, stmt, &aref))
     return HOST_WIDE_INT_MAX;
 
   offset_int remsize = aref.size_remaining ();
@@ -4516,7 +4538,7 @@ handle_printf_call (gimple_stmt_iterator *gsi, pointer_query &ptr_qry)
       /* For non-bounded functions like sprintf, determine the size
 	 of the destination from the object or pointer passed to it
 	 as the first argument.  */
-      dstsize = get_destination_size (dstptr, ptr_qry);
+      dstsize = get_destination_size (dstptr, info.callstmt, ptr_qry);
     }
   else if (tree size = gimple_call_arg (info.callstmt, idx_dstsize))
     {
diff --git a/gcc/gimple-ssa-warn-access.cc b/gcc/gimple-ssa-warn-access.cc
index 00c3ea0..22c791d 100644
--- a/gcc/gimple-ssa-warn-access.cc
+++ b/gcc/gimple-ssa-warn-access.cc
@@ -1190,11 +1190,11 @@ warn_for_access (location_t loc, tree func, tree expr, int opt,
    by BNDRNG if nonnull and valid.  */
 
 static void
-get_size_range (range_query *query, tree bound, tree range[2],
+get_size_range (range_query *query, tree bound, gimple *stmt, tree range[2],
 		const offset_int bndrng[2])
 {
   if (bound)
-    get_size_range (query, bound, NULL, range);
+    get_size_range (query, bound, stmt, range);
 
   if (!bndrng || (bndrng[0] == 0 && bndrng[1] == HOST_WIDE_INT_M1U))
     return;
@@ -1251,7 +1251,8 @@ template <class GimpleOrTree>
 static bool
 check_access (GimpleOrTree exp, tree dstwrite,
 	      tree maxread, tree srcstr, tree dstsize,
-	      access_mode mode, const access_data *pad /* = NULL */)
+	      access_mode mode, const access_data *pad,
+	      range_query *rvals)
 {
   /* The size of the largest object is half the address space, or
      PTRDIFF_MAX.  (This is way too permissive.)  */
@@ -1338,7 +1339,8 @@ check_access (GimpleOrTree exp, tree dstwrite,
 
   /* Set RANGE to that of DSTWRITE if non-null, bounded by PAD->DST.BNDRNG
      if valid.  */
-  get_size_range (NULL, dstwrite, range, pad ? pad->dst.bndrng : NULL);
+  gimple *stmt = pad ? pad->stmt : nullptr;
+  get_size_range (rvals, dstwrite, stmt, range, pad ? pad->dst.bndrng : NULL);
 
   tree func = get_callee_fndecl (exp);
   /* Read vs write access by built-ins can be determined from the const
@@ -1432,7 +1434,7 @@ check_access (GimpleOrTree exp, tree dstwrite,
     {
       /* Set RANGE to that of MAXREAD, bounded by PAD->SRC.BNDRNG if
 	 PAD is nonnull and BNDRNG is valid.  */
-      get_size_range (NULL, maxread, range, pad ? pad->src.bndrng : NULL);
+      get_size_range (rvals, maxread, stmt, range, pad ? pad->src.bndrng : NULL);
 
       location_t loc = get_location (exp);
       tree size = dstsize;
@@ -1479,7 +1481,7 @@ check_access (GimpleOrTree exp, tree dstwrite,
     {
       /* Set RANGE to that of MAXREAD, bounded by PAD->SRC.BNDRNG if
 	 PAD is nonnull and BNDRNG is valid.  */
-      get_size_range (NULL, maxread, range, pad ? pad->src.bndrng : NULL);
+      get_size_range (rvals, maxread, stmt, range, pad ? pad->src.bndrng : NULL);
       /* Set OVERREAD for reads starting just past the end of an object.  */
       overread = pad->src.sizrng[1] - pad->src.offrng[0] < pad->src.bndrng[0];
       range[0] = wide_int_to_tree (sizetype, pad->src.bndrng[0]);
@@ -1512,13 +1514,14 @@ check_access (GimpleOrTree exp, tree dstwrite,
   return true;
 }
 
-bool
+static bool
 check_access (gimple *stmt, tree dstwrite,
 	      tree maxread, tree srcstr, tree dstsize,
-	      access_mode mode, const access_data *pad /* = NULL */)
+	      access_mode mode, const access_data *pad,
+	      range_query *rvals)
 {
-  return check_access<gimple *>(stmt, dstwrite, maxread, srcstr, dstsize,
-				mode, pad);
+  return check_access<gimple *> (stmt, dstwrite, maxread, srcstr, dstsize,
+				 mode, pad, rvals);
 }
 
 bool
@@ -1526,45 +1529,8 @@ check_access (tree expr, tree dstwrite,
 	      tree maxread, tree srcstr, tree dstsize,
 	      access_mode mode, const access_data *pad /* = NULL */)
 {
-  return check_access<tree>(expr, dstwrite, maxread, srcstr, dstsize,
-			    mode, pad);
-}
-
-/* A convenience wrapper for check_access above to check access
-   by a read-only function like puts.  */
-
-template <class GimpleOrTree>
-static bool
-check_read_access (GimpleOrTree expr, tree src, tree bound, int ost)
-{
-  if (!warn_stringop_overread)
-    return true;
-
-  if (bound && !useless_type_conversion_p (size_type_node, TREE_TYPE (bound)))
-    bound = fold_convert (size_type_node, bound);
-
-  tree fndecl = get_callee_fndecl (expr);
-  maybe_warn_nonstring_arg (fndecl, expr);
-
-  access_data data (expr, access_read_only, NULL_TREE, false, bound, true);
-  compute_objsize (src, ost, &data.src);
-  return check_access (expr, /*dstwrite=*/ NULL_TREE, /*maxread=*/ bound,
-		       /*srcstr=*/ src, /*dstsize=*/ NULL_TREE, data.mode,
-		       &data);
-}
-
-bool
-check_read_access (gimple *stmt, tree src, tree bound /* = NULL_TREE */,
-		   int ost /* = 1 */)
-{
-  return check_read_access<gimple *>(stmt, src, bound, ost);
-}
-
-bool
-check_read_access (tree expr, tree src, tree bound /* = NULL_TREE */,
-		   int ost /* = 1 */)
-{
-  return check_read_access<tree>(expr, src, bound, ost);
+  return check_access<tree> (expr, dstwrite, maxread, srcstr, dstsize,
+			     mode, pad, nullptr);
 }
 
 /* Return true if STMT is a call to an allocation function.  Unless
@@ -2109,6 +2075,9 @@ private:
   pass_waccess (pass_waccess &) = delete;
   void operator= (pass_waccess &) = delete;
 
+  /* Check a call to an atomic built-in function.  */
+  bool check_atomic_builtin (gcall *);
+
   /* Check a call to a built-in function.  */
   bool check_builtin (gcall *);
 
@@ -2130,6 +2099,7 @@ private:
   void check_stxncpy (gcall *);
   void check_strncmp (gcall *);
   void check_memop_access (gimple *, tree, tree, tree);
+  void check_read_access (gimple *, tree, tree = NULL_TREE, int = 1);
 
   void maybe_check_dealloc_call (gcall *);
   void maybe_check_access_sizes (rdwr_map *, tree, tree, gimple *);
@@ -2365,10 +2335,6 @@ pass_waccess::check_alloca (gcall *stmt)
 void
 pass_waccess::check_alloc_size_call (gcall *stmt)
 {
-  if (gimple_call_num_args (stmt) < 1)
-    /* Avoid invalid calls to functions without a prototype.  */
-    return;
-
   tree fndecl = gimple_call_fndecl (stmt);
   if (fndecl && gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
     {
@@ -2397,13 +2363,19 @@ pass_waccess::check_alloc_size_call (gcall *stmt)
      the actual argument(s) at those indices in ALLOC_ARGS.  */
   int idx[2] = { -1, -1 };
   tree alloc_args[] = { NULL_TREE, NULL_TREE };
+  unsigned nargs = gimple_call_num_args (stmt);
 
   tree args = TREE_VALUE (alloc_size);
   idx[0] = TREE_INT_CST_LOW (TREE_VALUE (args)) - 1;
+  /* Avoid invalid calls to functions without a prototype.  */
+  if ((unsigned) idx[0] >= nargs)
+    return;
   alloc_args[0] = call_arg (stmt, idx[0]);
   if (TREE_CHAIN (args))
     {
       idx[1] = TREE_INT_CST_LOW (TREE_VALUE (TREE_CHAIN (args))) - 1;
+      if ((unsigned) idx[1] >= nargs)
+	return;
       alloc_args[1] = call_arg (stmt, idx[1]);
     }
 
@@ -2425,14 +2397,14 @@ pass_waccess::check_strcat (gcall *stmt)
      the destination to which the SRC string is being appended so
      just diagnose cases when the souce string is longer than
      the destination object.  */
-  access_data data (stmt, access_read_write, NULL_TREE, true,
-		    NULL_TREE, true);
+  access_data data (m_ptr_qry.rvals, stmt, access_read_write, NULL_TREE,
+		    true, NULL_TREE, true);
   const int ost = warn_stringop_overflow ? warn_stringop_overflow - 1 : 1;
-  compute_objsize (src, ost, &data.src, &m_ptr_qry);
-  tree destsize = compute_objsize (dest, ost, &data.dst, &m_ptr_qry);
+  compute_objsize (src, stmt, ost, &data.src, &m_ptr_qry);
+  tree destsize = compute_objsize (dest, stmt, ost, &data.dst, &m_ptr_qry);
 
   check_access (stmt, /*dstwrite=*/NULL_TREE, /*maxread=*/NULL_TREE,
-		src, destsize, data.mode, &data);
+		src, destsize, data.mode, &data, m_ptr_qry.rvals);
 }
 
 /* Check a call STMT to strcat() for overflow and warn if it does.  */
@@ -2466,12 +2438,12 @@ pass_waccess::check_strncat (gcall *stmt)
       maxlen = lendata.maxbound;
     }
 
-  access_data data (stmt, access_read_write);
+  access_data data (m_ptr_qry.rvals, stmt, access_read_write);
   /* Try to verify that the destination is big enough for the shortest
      string.  First try to determine the size of the destination object
      into which the source is being copied.  */
   const int ost = warn_stringop_overflow - 1;
-  tree destsize = compute_objsize (dest, ost, &data.dst, &m_ptr_qry);
+  tree destsize = compute_objsize (dest, stmt, ost, &data.dst, &m_ptr_qry);
 
   /* Add one for the terminating nul.  */
   tree srclen = (maxlen
@@ -2500,7 +2472,7 @@ pass_waccess::check_strncat (gcall *stmt)
     srclen = maxread;
 
   check_access (stmt, /*dstwrite=*/NULL_TREE, maxread, srclen,
-		destsize, data.mode, &data);
+		destsize, data.mode, &data, m_ptr_qry.rvals);
 }
 
 /* Check a call STMT to stpcpy() or strcpy() for overflow and warn
@@ -2524,14 +2496,14 @@ pass_waccess::check_stxcpy (gcall *stmt)
 
   if (warn_stringop_overflow)
     {
-      access_data data (stmt, access_read_write, NULL_TREE, true,
-			NULL_TREE, true);
+      access_data data (m_ptr_qry.rvals, stmt, access_read_write, NULL_TREE,
+			true, NULL_TREE, true);
       const int ost = warn_stringop_overflow ? warn_stringop_overflow - 1 : 1;
-      compute_objsize (src, ost, &data.src, &m_ptr_qry);
-      tree dstsize = compute_objsize (dst, ost, &data.dst, &m_ptr_qry);
+      compute_objsize (src, stmt, ost, &data.src, &m_ptr_qry);
+      tree dstsize = compute_objsize (dst, stmt, ost, &data.dst, &m_ptr_qry);
       check_access (stmt, /*dstwrite=*/ NULL_TREE,
 		    /*maxread=*/ NULL_TREE, /*srcstr=*/ src,
-		    dstsize, data.mode, &data);
+		    dstsize, data.mode, &data, m_ptr_qry.rvals);
     }
 
   /* Check to see if the argument was declared attribute nonstring
@@ -2555,13 +2527,14 @@ pass_waccess::check_stxncpy (gcall *stmt)
   /* The number of bytes to write (not the maximum).  */
   tree len = call_arg (stmt, 2);
 
-  access_data data (stmt, access_read_write, len, true, len, true);
+  access_data data (m_ptr_qry.rvals, stmt, access_read_write, len, true, len,
+		    true);
   const int ost = warn_stringop_overflow ? warn_stringop_overflow - 1 : 1;
-  compute_objsize (src, ost, &data.src, &m_ptr_qry);
-  tree dstsize = compute_objsize (dst, ost, &data.dst, &m_ptr_qry);
+  compute_objsize (src, stmt, ost, &data.src, &m_ptr_qry);
+  tree dstsize = compute_objsize (dst, stmt, ost, &data.dst, &m_ptr_qry);
 
-  check_access (stmt, /*dstwrite=*/len,
-		/*maxread=*/len, src, dstsize, data.mode, &data);
+  check_access (stmt, /*dstwrite=*/len, /*maxread=*/len, src, dstsize,
+		data.mode, &data, m_ptr_qry.rvals);
 }
 
 /* Check a call STMT to stpncpy() or strncpy() for overflow and warn
@@ -2594,6 +2567,11 @@ pass_waccess::check_strncmp (gcall *stmt)
   tree len1 = c_strlen (arg1, 1, &lendata1);
   tree len2 = c_strlen (arg2, 1, &lendata2);
 
+  if (len1 && TREE_CODE (len1) != INTEGER_CST)
+    len1 = NULL_TREE;
+  if (len2 && TREE_CODE (len2) != INTEGER_CST)
+    len2 = NULL_TREE;
+
   if (len1 && len2)
     /* If the length of both arguments was computed they must both be
        nul-terminated and no further checking is necessary regardless
@@ -2606,13 +2584,15 @@ pass_waccess::check_strncmp (gcall *stmt)
   if (maybe_warn_nonstring_arg (get_callee_fndecl (stmt), stmt))
     return;
 
-  access_data adata1 (stmt, access_read_only, NULL_TREE, false, bound, true);
-  access_data adata2 (stmt, access_read_only, NULL_TREE, false, bound, true);
+  access_data adata1 (m_ptr_qry.rvals, stmt, access_read_only, NULL_TREE, false,
+		      bound, true);
+  access_data adata2 (m_ptr_qry.rvals, stmt, access_read_only, NULL_TREE, false,
+		      bound, true);
 
   /* Determine the range of the bound first and bail if it fails; it's
      cheaper than computing the size of the objects.  */
   tree bndrng[2] = { NULL_TREE, NULL_TREE };
-  get_size_range (m_ptr_qry.rvals, bound, bndrng, adata1.src.bndrng);
+  get_size_range (m_ptr_qry.rvals, bound, stmt, bndrng, adata1.src.bndrng);
   if (!bndrng[0] || integer_zerop (bndrng[0]))
     return;
 
@@ -2623,8 +2603,8 @@ pass_waccess::check_strncmp (gcall *stmt)
 
   /* compute_objsize almost never fails (and ultimately should never
      fail).  Don't bother to handle the rare case when it does.  */
-  if (!compute_objsize (arg1, 1, &adata1.src, &m_ptr_qry)
-      || !compute_objsize (arg2, 1, &adata2.src, &m_ptr_qry))
+  if (!compute_objsize (arg1, stmt, 1, &adata1.src, &m_ptr_qry)
+      || !compute_objsize (arg2, stmt, 1, &adata2.src, &m_ptr_qry))
     return;
 
   /* Compute the size of the remaining space in each array after
@@ -2672,13 +2652,120 @@ pass_waccess::check_memop_access (gimple *stmt, tree dest, tree src, tree size)
      try to determine the size of the largest source and destination
      object using type-0 Object Size regardless of the object size
      type specified by the option.  */
-  access_data data (stmt, access_read_write);
+  access_data data (m_ptr_qry.rvals, stmt, access_read_write);
   tree srcsize
-    = src ? compute_objsize (src, 0, &data.src, &m_ptr_qry) : NULL_TREE;
-  tree dstsize = compute_objsize (dest, 0, &data.dst, &m_ptr_qry);
+    = src ? compute_objsize (src, stmt, 0, &data.src, &m_ptr_qry) : NULL_TREE;
+  tree dstsize = compute_objsize (dest, stmt, 0, &data.dst, &m_ptr_qry);
+
+  check_access (stmt, size, /*maxread=*/NULL_TREE, srcsize, dstsize,
+		data.mode, &data, m_ptr_qry.rvals);
+}
+
+/* A convenience wrapper for check_access to check access by a read-only
+   function like puts or strcmp.  */
+
+void
+pass_waccess::check_read_access (gimple *stmt, tree src,
+				 tree bound /* = NULL_TREE */,
+				 int ost /* = 1 */)
+{
+  if (!warn_stringop_overread)
+    return;
+
+  if (bound && !useless_type_conversion_p (size_type_node, TREE_TYPE (bound)))
+    bound = fold_convert (size_type_node, bound);
+
+  tree fndecl = get_callee_fndecl (stmt);
+  maybe_warn_nonstring_arg (fndecl, stmt);
 
-  check_access (stmt, size, /*maxread=*/NULL_TREE,
-		srcsize, dstsize, data.mode, &data);
+  access_data data (m_ptr_qry.rvals, stmt, access_read_only, NULL_TREE,
+		   false, bound, true);
+  compute_objsize (src, stmt, ost, &data.src, &m_ptr_qry);
+  check_access (stmt, /*dstwrite=*/ NULL_TREE, /*maxread=*/ bound,
+		/*srcstr=*/ src, /*dstsize=*/ NULL_TREE, data.mode,
+		&data, m_ptr_qry.rvals);
+}
+
+
+/* Check a call STMT to an atomic or sync built-in.  */
+
+bool
+pass_waccess::check_atomic_builtin (gcall *stmt)
+{
+  tree callee = gimple_call_fndecl (stmt);
+  if (!callee)
+    return false;
+
+  /* The size in bytes of the access by the function, and the number
+     of the second argument to check (if any).  */
+  unsigned bytes = 0, arg2 = UINT_MAX;
+
+  switch (DECL_FUNCTION_CODE (callee))
+    {
+#define BUILTIN_ACCESS_SIZE_FNSPEC(N)			\
+      BUILT_IN_ATOMIC_LOAD_ ## N:			\
+    case BUILT_IN_SYNC_FETCH_AND_ADD_ ## N:		\
+    case BUILT_IN_SYNC_FETCH_AND_SUB_ ## N:		\
+    case BUILT_IN_SYNC_FETCH_AND_OR_ ## N:		\
+    case BUILT_IN_SYNC_FETCH_AND_AND_ ## N:		\
+    case BUILT_IN_SYNC_FETCH_AND_XOR_ ## N:		\
+    case BUILT_IN_SYNC_FETCH_AND_NAND_ ## N:		\
+    case BUILT_IN_SYNC_ADD_AND_FETCH_ ## N:		\
+    case BUILT_IN_SYNC_SUB_AND_FETCH_ ## N:		\
+    case BUILT_IN_SYNC_OR_AND_FETCH_ ## N:		\
+    case BUILT_IN_SYNC_AND_AND_FETCH_ ## N:		\
+    case BUILT_IN_SYNC_XOR_AND_FETCH_ ## N:		\
+    case BUILT_IN_SYNC_NAND_AND_FETCH_ ## N:		\
+    case BUILT_IN_SYNC_LOCK_TEST_AND_SET_ ## N:		\
+    case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_ ## N:	\
+    case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_ ## N:	\
+    case BUILT_IN_SYNC_LOCK_RELEASE_ ## N:		\
+    case BUILT_IN_ATOMIC_EXCHANGE_ ## N:		\
+    case BUILT_IN_ATOMIC_STORE_ ## N:			\
+    case BUILT_IN_ATOMIC_ADD_FETCH_ ## N:		\
+    case BUILT_IN_ATOMIC_SUB_FETCH_ ## N:		\
+    case BUILT_IN_ATOMIC_AND_FETCH_ ## N:		\
+    case BUILT_IN_ATOMIC_NAND_FETCH_ ## N:		\
+    case BUILT_IN_ATOMIC_XOR_FETCH_ ## N:		\
+    case BUILT_IN_ATOMIC_OR_FETCH_ ## N:		\
+    case BUILT_IN_ATOMIC_FETCH_ADD_ ## N:		\
+    case BUILT_IN_ATOMIC_FETCH_SUB_ ## N:		\
+    case BUILT_IN_ATOMIC_FETCH_AND_ ## N:		\
+    case BUILT_IN_ATOMIC_FETCH_NAND_ ## N:		\
+    case BUILT_IN_ATOMIC_FETCH_OR_ ## N:		\
+    case BUILT_IN_ATOMIC_FETCH_XOR_ ## N:		\
+	bytes = N;					\
+	break;						\
+    case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_ ## N:	\
+	bytes = N;					\
+	arg2 = 1
+
+    case BUILTIN_ACCESS_SIZE_FNSPEC (1);
+      break;
+    case BUILTIN_ACCESS_SIZE_FNSPEC (2);
+      break;
+    case BUILTIN_ACCESS_SIZE_FNSPEC (4);
+      break;
+    case BUILTIN_ACCESS_SIZE_FNSPEC (8);
+      break;
+    case BUILTIN_ACCESS_SIZE_FNSPEC (16);
+      break;
+
+    default:
+      return false;
+    }
+
+  tree size = build_int_cstu (sizetype, bytes);
+  tree dst = gimple_call_arg (stmt, 0);
+  check_memop_access (stmt, dst, NULL_TREE, size);
+
+  if (arg2 != UINT_MAX)
+    {
+      tree dst = gimple_call_arg (stmt, arg2);
+      check_memop_access (stmt, dst, NULL_TREE, size);
+    }
+
+  return true;
 }
 
 /* Check call STMT to a built-in function for invalid accesses.  Return
@@ -2699,6 +2786,15 @@ pass_waccess::check_builtin (gcall *stmt)
       check_alloca (stmt);
       return true;
 
+    case BUILT_IN_EXECL:
+    case BUILT_IN_EXECLE:
+    case BUILT_IN_EXECLP:
+    case BUILT_IN_EXECV:
+    case BUILT_IN_EXECVE:
+    case BUILT_IN_EXECVP:
+      check_read_access (stmt, call_arg (stmt, 0));
+      return true;
+
     case BUILT_IN_GETTEXT:
     case BUILT_IN_PUTS:
     case BUILT_IN_PUTS_UNLOCKED:
@@ -2721,8 +2817,12 @@ pass_waccess::check_builtin (gcall *stmt)
 
     case BUILT_IN_STRNDUP:
     case BUILT_IN_STRNLEN:
-      check_read_access (stmt, call_arg (stmt, 0), call_arg (stmt, 1));
-      return true;
+      {
+	tree str = call_arg (stmt, 0);
+	tree len = call_arg (stmt, 1);
+	check_read_access (stmt, str, len);
+	return true;
+      }
 
     case BUILT_IN_STRCAT:
       check_strcat (stmt);
@@ -2795,10 +2895,11 @@ pass_waccess::check_builtin (gcall *stmt)
       }
 	
     default:
-      return false;
+      if (check_atomic_builtin (stmt))
+	return true;
+      break;
     }
-
-  return true;
+  return false;
 }
 
 /* Returns the type of the argument ARGNO to function with type FNTYPE
@@ -2900,7 +3001,7 @@ pass_waccess::maybe_check_access_sizes (rdwr_map *rwm, tree fndecl, tree fntype,
       /* Format the value or range to avoid an explosion of messages.  */
       char sizstr[80];
       tree sizrng[2] = { size_zero_node, build_all_ones_cst (sizetype) };
-      if (get_size_range (m_ptr_qry.rvals, access_size, NULL, sizrng, 1))
+      if (get_size_range (m_ptr_qry.rvals, access_size, stmt, sizrng, 1))
 	{
 	  char *s0 = print_generic_expr_to_str (sizrng[0]);
 	  if (tree_int_cst_equal (sizrng[0], sizrng[1]))
@@ -2913,7 +3014,7 @@ pass_waccess::maybe_check_access_sizes (rdwr_map *rwm, tree fndecl, tree fntype,
 	      char *s1 = print_generic_expr_to_str (sizrng[1]);
 	      gcc_checking_assert (strlen (s0) + strlen (s1)
 				   < sizeof sizstr - 4);
-	      sprintf (sizstr, "[%s, %s]", s0, s1);
+	      sprintf (sizstr, "[%.37s, %.37s]", s0, s1);
 	      free (s1);
 	    }
 	  free (s0);
@@ -3028,11 +3129,11 @@ pass_waccess::maybe_check_access_sizes (rdwr_map *rwm, tree fndecl, tree fntype,
 	    }
 	}
 
-      access_data data (ptr, access.second.mode, NULL_TREE, false,
-			NULL_TREE, false);
+      access_data data (m_ptr_qry.rvals, stmt, access.second.mode,
+			NULL_TREE, false, NULL_TREE, false);
       access_ref* const pobj = (access.second.mode == access_write_only
 				? &data.dst : &data.src);
-      tree objsize = compute_objsize (ptr, 1, pobj, &m_ptr_qry);
+      tree objsize = compute_objsize (ptr, stmt, 1, pobj, &m_ptr_qry);
 
       /* The size of the destination or source object.  */
       tree dstsize = NULL_TREE, srcsize = NULL_TREE;
@@ -3064,7 +3165,7 @@ pass_waccess::maybe_check_access_sizes (rdwr_map *rwm, tree fndecl, tree fntype,
       if (mode == access_deferred)
 	mode = TYPE_READONLY (argtype) ? access_read_only : access_read_write;
       check_access (stmt, access_size, /*maxread=*/ NULL_TREE, srcsize,
-		    dstsize, mode, &data);
+		    dstsize, mode, &data, m_ptr_qry.rvals);
 
       if (warning_suppressed_p (stmt, OPT_Wstringop_overflow_))
 	opt_warned = OPT_Wstringop_overflow_;
@@ -3187,7 +3288,7 @@ pass_waccess::maybe_check_dealloc_call (gcall *call)
     return;
 
   access_ref aref;
-  if (!compute_objsize (ptr, 0, &aref, &m_ptr_qry))
+  if (!compute_objsize (ptr, call, 0, &aref, &m_ptr_qry))
     return;
 
   tree ref = aref.ref;
diff --git a/gcc/gimple-ssa-warn-access.h b/gcc/gimple-ssa-warn-access.h
index 00f5bb1..124f3f5 100644
--- a/gcc/gimple-ssa-warn-access.h
+++ b/gcc/gimple-ssa-warn-access.h
@@ -45,7 +45,4 @@ class access_data;
 extern bool check_access (tree, tree, tree, tree, tree, access_mode,
 			  const access_data * = NULL);
 
-extern bool check_read_access (gimple *, tree, tree = NULL_TREE, int ost = 1);
-extern bool check_read_access (tree, tree, tree = NULL_TREE, int = 1);
-
 #endif   // GCC_GIMPLE_SSA_WARN_ACCESS_H
diff --git a/gcc/gimple.c b/gcc/gimple.c
index cc7a88e..037c6e4 100644
--- a/gcc/gimple.c
+++ b/gcc/gimple.c
@@ -1567,21 +1567,35 @@ gimple_call_arg_flags (const gcall *stmt, unsigned arg)
   int flags = 0;
 
   if (fnspec.known_p ())
+    flags = fnspec.arg_eaf_flags (arg);
+  tree callee = gimple_call_fndecl (stmt);
+  if (callee)
     {
-      if (!fnspec.arg_specified_p (arg))
-	;
-      else if (!fnspec.arg_used_p (arg))
-	flags = EAF_UNUSED;
-      else
+      cgraph_node *node = cgraph_node::get (callee);
+      modref_summary *summary = node ? get_modref_function_summary (node)
+				: NULL;
+
+      if (summary && summary->arg_flags.length () > arg)
 	{
-	  if (fnspec.arg_direct_p (arg))
-	    flags |= EAF_DIRECT;
-	  if (fnspec.arg_noescape_p (arg))
-	    flags |= EAF_NOESCAPE | EAF_NODIRECTESCAPE;
-	  if (fnspec.arg_readonly_p (arg))
-	    flags |= EAF_NOCLOBBER;
+	  int modref_flags = summary->arg_flags[arg];
+
+	  /* We have possibly optimized out load.  Be conservative here.  */
+	  if (!node->binds_to_current_def_p ())
+	    modref_flags = interposable_eaf_flags (modref_flags, flags);
+	  if (dbg_cnt (ipa_mod_ref_pta))
+	    flags |= modref_flags;
 	}
     }
+  return flags;
+}
+
+/* Detects argument flags for return slot on call STMT.  */
+
+int
+gimple_call_retslot_flags (const gcall *stmt)
+{
+  int flags = implicit_retslot_eaf_flags;
+
   tree callee = gimple_call_fndecl (stmt);
   if (callee)
     {
@@ -1589,18 +1603,41 @@ gimple_call_arg_flags (const gcall *stmt, unsigned arg)
       modref_summary *summary = node ? get_modref_function_summary (node)
 				: NULL;
 
-      if (summary && summary->arg_flags.length () > arg)
+      if (summary)
 	{
-	  int modref_flags = summary->arg_flags[arg];
+	  int modref_flags = summary->retslot_flags;
 
 	  /* We have possibly optimized out load.  Be conservative here.  */
 	  if (!node->binds_to_current_def_p ())
-	    {
-	      if ((modref_flags & EAF_UNUSED) && !(flags & EAF_UNUSED))
-		modref_flags &= ~EAF_UNUSED;
-	      if ((modref_flags & EAF_DIRECT) && !(flags & EAF_DIRECT))
-		modref_flags &= ~EAF_DIRECT;
-	    }
+	    modref_flags = interposable_eaf_flags (modref_flags, flags);
+	  if (dbg_cnt (ipa_mod_ref_pta))
+	    flags |= modref_flags;
+	}
+    }
+  return flags;
+}
+
+/* Detects argument flags for static chain on call STMT.  */
+
+int
+gimple_call_static_chain_flags (const gcall *stmt)
+{
+  int flags = 0;
+
+  tree callee = gimple_call_fndecl (stmt);
+  if (callee)
+    {
+      cgraph_node *node = cgraph_node::get (callee);
+      modref_summary *summary = node ? get_modref_function_summary (node)
+				: NULL;
+
+      /* Nested functions should always bind to current def since
+	 there is no public ABI for them.  */
+      gcc_checking_assert (node->binds_to_current_def_p ());
+      if (summary)
+	{
+	  int modref_flags = summary->static_chain_flags;
+
 	  if (dbg_cnt (ipa_mod_ref_pta))
 	    flags |= modref_flags;
 	}
diff --git a/gcc/gimple.h b/gcc/gimple.h
index 303623b..3cde3cd 100644
--- a/gcc/gimple.h
+++ b/gcc/gimple.h
@@ -1589,6 +1589,8 @@ gimple_seq gimple_seq_copy (gimple_seq);
 bool gimple_call_same_target_p (const gimple *, const gimple *);
 int gimple_call_flags (const gimple *);
 int gimple_call_arg_flags (const gcall *, unsigned);
+int gimple_call_retslot_flags (const gcall *);
+int gimple_call_static_chain_flags (const gcall *);
 int gimple_call_return_flags (const gcall *);
 bool gimple_call_nonnull_result_p (gcall *);
 tree gimple_call_nonnull_arg (gcall *);
diff --git a/gcc/gimplify.c b/gcc/gimplify.c
index d8e4b13..1602a62 100644
--- a/gcc/gimplify.c
+++ b/gcc/gimplify.c
@@ -1784,8 +1784,8 @@ gimple_add_init_for_auto_var (tree decl,
    that padding is initialized to zero. So, we always initialize paddings
    to zeroes regardless INIT_TYPE.
    To do the padding initialization, we insert a call to
-   __BUILTIN_CLEAR_PADDING (&decl, 0, for_auto_init = true).
-   Note, we add an additional dummy argument for __BUILTIN_CLEAR_PADDING,
+   __builtin_clear_padding (&decl, 0, for_auto_init = true).
+   Note, we add an additional dummy argument for __builtin_clear_padding,
    'for_auto_init' to distinguish whether this call is for automatic
    variable initialization or not.
    */
@@ -1954,8 +1954,14 @@ gimplify_decl_expr (tree *stmt_p, gimple_seq *seq_p)
 	     pattern initialization.
 	     In order to make the paddings as zeroes for pattern init, We
 	     should add a call to __builtin_clear_padding to clear the
-	     paddings to zero in compatiple with CLANG.  */
-	  if (flag_auto_var_init == AUTO_INIT_PATTERN)
+	     paddings to zero in compatiple with CLANG.
+	     We cannot insert this call if the variable is a gimple register
+	     since __builtin_clear_padding will take the address of the
+	     variable.  As a result, if a long double/_Complex long double
+	     variable will spilled into stack later, its padding is 0XFE.  */
+	  if (flag_auto_var_init == AUTO_INIT_PATTERN
+	      && !is_gimple_reg (decl)
+	      && clear_padding_type_may_have_padding_p (TREE_TYPE (decl)))
 	    gimple_add_padding_init_for_auto_var (decl, is_vla, seq_p);
 	}
     }
@@ -5384,12 +5390,19 @@ gimplify_init_constructor (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
 
   /* If the user requests to initialize automatic variables, we
      should initialize paddings inside the variable.  Add a call to
-     __BUILTIN_CLEAR_PADDING (&object, 0, for_auto_init = true) to
+     __builtin_clear_pading (&object, 0, for_auto_init = true) to
      initialize paddings of object always to zero regardless of
      INIT_TYPE.  Note, we will not insert this call if the aggregate
      variable has be completely cleared already or it's initialized
-     with an empty constructor.  */
+     with an empty constructor.  We cannot insert this call if the
+     variable is a gimple register since __builtin_clear_padding will take
+     the address of the variable.  As a result, if a long double/_Complex long
+     double variable will be spilled into stack later, its padding cannot
+     be cleared with __builtin_clear_padding.  We should clear its padding
+     when it is spilled into memory.  */
   if (is_init_expr
+      && !is_gimple_reg (object)
+      && clear_padding_type_may_have_padding_p (type)
       && ((AGGREGATE_TYPE_P (type) && !cleared && !is_empty_ctor)
 	  || !AGGREGATE_TYPE_P (type))
       && is_var_need_auto_init (object))
@@ -10260,9 +10273,24 @@ gimplify_scan_omp_clauses (tree *list_p, gimple_seq *pre_p,
 	    = gimple_boolify (OMP_CLAUSE_OPERAND (c, 0));
 	  /* Fall through.  */
 
+	case OMP_CLAUSE_NUM_TEAMS:
+	  if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS
+	      && OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c)
+	      && !is_gimple_min_invariant (OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c)))
+	    {
+	      if (error_operand_p (OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c)))
+		{
+		  remove = true;
+		  break;
+		}
+	      OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c)
+		= get_initialized_tmp_var (OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c),
+					   pre_p, NULL, true);
+	    }
+	  /* Fall through.  */
+
 	case OMP_CLAUSE_SCHEDULE:
 	case OMP_CLAUSE_NUM_THREADS:
-	case OMP_CLAUSE_NUM_TEAMS:
 	case OMP_CLAUSE_THREAD_LIMIT:
 	case OMP_CLAUSE_DIST_SCHEDULE:
 	case OMP_CLAUSE_DEVICE:
@@ -10861,6 +10889,10 @@ gimplify_adjust_omp_clauses_1 (splay_tree_node n, void *data)
 	  gcc_unreachable ();
 	}
       OMP_CLAUSE_SET_MAP_KIND (clause, kind);
+      /* Setting of the implicit flag for the runtime is currently disabled for
+	 OpenACC.  */
+      if ((gimplify_omp_ctxp->region_type & ORT_ACC) == 0)
+	OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (clause) = 1;
       if (DECL_SIZE (decl)
 	  && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
 	{
@@ -11476,9 +11508,15 @@ gimplify_adjust_omp_clauses (gimple_seq *pre_p, gimple_seq body, tree *list_p,
 	list_p = &OMP_CLAUSE_CHAIN (c);
     }
 
-  /* Add in any implicit data sharing.  */
+  /* Add in any implicit data sharing.  Implicit clauses are added at the start
+     of the clause list, but after any non-map clauses.  */
   struct gimplify_adjust_omp_clauses_data data;
-  data.list_p = list_p;
+  tree *implicit_add_list_p = orig_list_p;
+  while (*implicit_add_list_p
+	 && OMP_CLAUSE_CODE (*implicit_add_list_p) != OMP_CLAUSE_MAP)
+    implicit_add_list_p = &OMP_CLAUSE_CHAIN (*implicit_add_list_p);
+
+  data.list_p = implicit_add_list_p;
   data.pre_p = pre_p;
   splay_tree_foreach (ctx->variables, gimplify_adjust_omp_clauses_1, &data);
 
@@ -12298,6 +12336,24 @@ gimplify_omp_for (tree *expr_p, gimple_seq *pre_p)
 	  gimplify_omp_ctxp->loop_iter_var.quick_push (decl);
 	}
 
+      if (for_stmt == orig_for_stmt)
+	{
+	  tree orig_decl = decl;
+	  if (OMP_FOR_ORIG_DECLS (for_stmt))
+	    {
+	      tree orig_decl = TREE_VEC_ELT (OMP_FOR_ORIG_DECLS (for_stmt), i);
+	      if (TREE_CODE (orig_decl) == TREE_LIST)
+		{
+		  orig_decl = TREE_PURPOSE (orig_decl);
+		  if (!orig_decl)
+		    orig_decl = decl;
+		}
+	    }
+	  if (is_global_var (orig_decl) && DECL_THREAD_LOCAL_P (orig_decl))
+	    error_at (EXPR_LOCATION (for_stmt),
+		      "threadprivate iteration variable %qD", orig_decl);
+	}
+
       /* Make sure the iteration variable is private.  */
       tree c = NULL_TREE;
       tree c2 = NULL_TREE;
@@ -13504,7 +13560,8 @@ optimize_target_teams (tree target, gimple_seq *pre_p)
 {
   tree body = OMP_BODY (target);
   tree teams = walk_tree (&body, find_omp_teams, NULL, NULL);
-  tree num_teams = integer_zero_node;
+  tree num_teams_lower = NULL_TREE;
+  tree num_teams_upper = integer_zero_node;
   tree thread_limit = integer_zero_node;
   location_t num_teams_loc = EXPR_LOCATION (target);
   location_t thread_limit_loc = EXPR_LOCATION (target);
@@ -13512,14 +13569,42 @@ optimize_target_teams (tree target, gimple_seq *pre_p)
   struct gimplify_omp_ctx *target_ctx = gimplify_omp_ctxp;
 
   if (teams == NULL_TREE)
-    num_teams = integer_one_node;
+    num_teams_upper = integer_one_node;
   else
     for (c = OMP_TEAMS_CLAUSES (teams); c; c = OMP_CLAUSE_CHAIN (c))
       {
 	if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_TEAMS)
 	  {
-	    p = &num_teams;
+	    p = &num_teams_upper;
 	    num_teams_loc = OMP_CLAUSE_LOCATION (c);
+	    if (OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c))
+	      {
+		expr = OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c);
+		if (TREE_CODE (expr) == INTEGER_CST)
+		  num_teams_lower = expr;
+		else if (walk_tree (&expr, computable_teams_clause,
+				    NULL, NULL))
+		  num_teams_lower = integer_minus_one_node;
+		else
+		  {
+		    num_teams_lower = expr;
+		    gimplify_omp_ctxp = gimplify_omp_ctxp->outer_context;
+		    if (gimplify_expr (&num_teams_lower, pre_p, NULL,
+				       is_gimple_val, fb_rvalue, false)
+			== GS_ERROR)
+		      {
+			gimplify_omp_ctxp = target_ctx;
+			num_teams_lower = integer_minus_one_node;
+		      }
+		    else
+		      {
+			gimplify_omp_ctxp = target_ctx;
+			if (!DECL_P (expr) && TREE_CODE (expr) != TARGET_EXPR)
+			  OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c)
+			    = num_teams_lower;
+		      }
+		  }
+	      }
 	  }
 	else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_THREAD_LIMIT)
 	  {
@@ -13552,12 +13637,16 @@ optimize_target_teams (tree target, gimple_seq *pre_p)
 	if (!DECL_P (expr) && TREE_CODE (expr) != TARGET_EXPR)
 	  OMP_CLAUSE_OPERAND (c, 0) = *p;
       }
-  c = build_omp_clause (thread_limit_loc, OMP_CLAUSE_THREAD_LIMIT);
-  OMP_CLAUSE_THREAD_LIMIT_EXPR (c) = thread_limit;
-  OMP_CLAUSE_CHAIN (c) = OMP_TARGET_CLAUSES (target);
-  OMP_TARGET_CLAUSES (target) = c;
+  if (!omp_find_clause (OMP_TARGET_CLAUSES (target), OMP_CLAUSE_THREAD_LIMIT))
+    {
+      c = build_omp_clause (thread_limit_loc, OMP_CLAUSE_THREAD_LIMIT);
+      OMP_CLAUSE_THREAD_LIMIT_EXPR (c) = thread_limit;
+      OMP_CLAUSE_CHAIN (c) = OMP_TARGET_CLAUSES (target);
+      OMP_TARGET_CLAUSES (target) = c;
+    }
   c = build_omp_clause (num_teams_loc, OMP_CLAUSE_NUM_TEAMS);
-  OMP_CLAUSE_NUM_TEAMS_EXPR (c) = num_teams;
+  OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (c) = num_teams_upper;
+  OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (c) = num_teams_lower;
   OMP_CLAUSE_CHAIN (c) = OMP_TARGET_CLAUSES (target);
   OMP_TARGET_CLAUSES (target) = c;
 }
diff --git a/gcc/go/gofrontend/MERGE b/gcc/go/gofrontend/MERGE
index affba73..05e47ec 100644
--- a/gcc/go/gofrontend/MERGE
+++ b/gcc/go/gofrontend/MERGE
@@ -1,4 +1,4 @@
-925ace70ac7426c3f8b5c0bfb75aa9601f071de4
+3e9f4ee16683883ccfb8661d99318c74bb7a4bef
 
 The first line of this file holds the git revision number of the last
 merge done from the gofrontend repository.
diff --git a/gcc/go/gofrontend/expressions.cc b/gcc/go/gofrontend/expressions.cc
index ddb1d91..7970282 100644
--- a/gcc/go/gofrontend/expressions.cc
+++ b/gcc/go/gofrontend/expressions.cc
@@ -7981,7 +7981,7 @@ Bound_method_expression::do_check_types(Gogo*)
 Bound_method_expression::Method_value_thunks
   Bound_method_expression::method_value_thunks;
 
-// Find or create the thunk for METHOD.
+// Find or create the thunk for FN.
 
 Named_object*
 Bound_method_expression::create_thunk(Gogo* gogo, const Method* method,
@@ -8078,14 +8078,28 @@ Bound_method_expression::create_thunk(Gogo* gogo, const Method* method,
   gogo->add_statement(s);
   Block* b = gogo->finish_block(loc);
   gogo->add_block(b, loc);
+
+  // This is called after lowering but before determine_types.
   gogo->lower_block(new_no, b);
-  gogo->flatten_block(new_no, b);
+
   gogo->finish_function(loc);
 
   ins.first->second = new_no;
   return new_no;
 }
 
+// Look up a thunk for FN.
+
+Named_object*
+Bound_method_expression::lookup_thunk(Named_object* fn)
+{
+  Method_value_thunks::const_iterator p =
+    Bound_method_expression::method_value_thunks.find(fn);
+  if (p == Bound_method_expression::method_value_thunks.end())
+    return NULL;
+  return p->second;
+}
+
 // Return an expression to check *REF for nil while dereferencing
 // according to FIELD_INDEXES.  Update *REF to build up the field
 // reference.  This is a static function so that we don't have to
@@ -8129,10 +8143,11 @@ Bound_method_expression::do_flatten(Gogo* gogo, Named_object*,
 {
   Location loc = this->location();
 
-  Named_object* thunk = Bound_method_expression::create_thunk(gogo,
-							      this->method_,
-							      this->function_);
-  if (thunk->is_erroneous())
+  Named_object* thunk = Bound_method_expression::lookup_thunk(this->function_);
+
+  // The thunk should have been created during the
+  // create_function_descriptors pass.
+  if (thunk == NULL || thunk->is_erroneous())
     {
       go_assert(saw_errors());
       return Expression::make_error(loc);
@@ -14757,14 +14772,34 @@ Interface_field_reference_expression::create_thunk(Gogo* gogo,
   gogo->add_statement(s);
   Block* b = gogo->finish_block(loc);
   gogo->add_block(b, loc);
+
+  // This is called after lowering but before determine_types.
   gogo->lower_block(new_no, b);
-  gogo->flatten_block(new_no, b);
+
   gogo->finish_function(loc);
 
   ins.first->second->push_back(std::make_pair(name, new_no));
   return new_no;
 }
 
+// Lookup a thunk to call method NAME on TYPE.
+
+Named_object*
+Interface_field_reference_expression::lookup_thunk(Interface_type* type,
+						   const std::string& name)
+{
+  Interface_method_thunks::const_iterator p =
+    Interface_field_reference_expression::interface_method_thunks.find(type);
+  if (p == Interface_field_reference_expression::interface_method_thunks.end())
+    return NULL;
+  for (Method_thunks::const_iterator pm = p->second->begin();
+       pm != p->second->end();
+       ++pm)
+    if (pm->first == name)
+      return pm->second;
+  return NULL;
+}
+
 // Get the backend representation for a method value.
 
 Bexpression*
@@ -14778,9 +14813,11 @@ Interface_field_reference_expression::do_get_backend(Translate_context* context)
     }
 
   Named_object* thunk =
-    Interface_field_reference_expression::create_thunk(context->gogo(),
-						       type, this->name_);
-  if (thunk->is_erroneous())
+    Interface_field_reference_expression::lookup_thunk(type, this->name_);
+
+  // The thunk should have been created during the
+  // create_function_descriptors pass.
+  if (thunk == NULL || thunk->is_erroneous())
     {
       go_assert(saw_errors());
       return context->backend()->error_expression();
diff --git a/gcc/go/gofrontend/expressions.h b/gcc/go/gofrontend/expressions.h
index 9348354..92e8d8d 100644
--- a/gcc/go/gofrontend/expressions.h
+++ b/gcc/go/gofrontend/expressions.h
@@ -3405,6 +3405,10 @@ class Bound_method_expression : public Expression
   static Named_object*
   create_thunk(Gogo*, const Method* method, Named_object* function);
 
+  // Look up a thunk.
+  static Named_object*
+  lookup_thunk(Named_object* function);
+
  protected:
   int
   do_traverse(Traverse*);
@@ -3578,6 +3582,10 @@ class Interface_field_reference_expression : public Expression
   static Named_object*
   create_thunk(Gogo*, Interface_type* type, const std::string& name);
 
+  // Look up a thunk.
+  static Named_object*
+  lookup_thunk(Interface_type* type, const std::string& name);
+
   // Return an expression for the pointer to the function to call.
   Expression*
   get_function();
diff --git a/gcc/go/gofrontend/gogo.cc b/gcc/go/gofrontend/gogo.cc
index 95b76bd..290d294 100644
--- a/gcc/go/gofrontend/gogo.cc
+++ b/gcc/go/gofrontend/gogo.cc
@@ -3430,6 +3430,11 @@ Create_function_descriptors::expression(Expression** pexpr)
 	      if (args->traverse(this) == TRAVERSE_EXIT)
 		return TRAVERSE_EXIT;
 	    }
+
+	  // Traverse the subexpressions of the function, if any.
+	  if (fn->traverse_subexpressions(this) == TRAVERSE_EXIT)
+	    return TRAVERSE_EXIT;
+
 	  return TRAVERSE_SKIP_COMPONENTS;
 	}
     }
diff --git a/gcc/go/gofrontend/lex.cc b/gcc/go/gofrontend/lex.cc
index dd66c02..d6909b8 100644
--- a/gcc/go/gofrontend/lex.cc
+++ b/gcc/go/gofrontend/lex.cc
@@ -1909,14 +1909,13 @@ Lex::skip_cpp_comment()
   if (saw_error)
     return;
 
-  // Recognize various magic comments at the start of a line.
+  // Recognize various magic comments at the start of a line, preceded
+  // only by spaces or tabs.
 
-  if (lineoff != 2)
-    {
-      // Not at the start of the line.  (lineoff == 2 because of the
-      // two characters in "//").
+  // "- 2" for the "//" at the start of the comment.
+  for (const char* psp = this->linebuf_; psp < p - 2; psp++)
+    if (*psp != ' ' && *psp != '\t')
       return;
-    }
 
   while (pend > p
 	 && (pend[-1] == ' ' || pend[-1] == '\t'
diff --git a/gcc/go/gofrontend/lex.h b/gcc/go/gofrontend/lex.h
index 75c8429..1c4cc5b 100644
--- a/gcc/go/gofrontend/lex.h
+++ b/gcc/go/gofrontend/lex.h
@@ -420,6 +420,12 @@ class Lex
     std::swap(*embeds, this->embeds_);
   }
 
+  // Clear any go:embed patterns seen so far.  This is used for
+  // erroneous cases.
+  void
+  clear_embeds()
+  { this->embeds_.clear(); }
+
   // Return whether the identifier NAME should be exported.  NAME is a
   // mangled name which includes only ASCII characters.
   static bool
diff --git a/gcc/go/gofrontend/parse.cc b/gcc/go/gofrontend/parse.cc
index e43b5f2..cc197e5 100644
--- a/gcc/go/gofrontend/parse.cc
+++ b/gcc/go/gofrontend/parse.cc
@@ -1307,46 +1307,14 @@ void
 Parse::declaration()
 {
   const Token* token = this->peek_token();
-
-  unsigned int pragmas = this->lex_->get_and_clear_pragmas();
-  if (pragmas != 0
-      && !token->is_keyword(KEYWORD_FUNC)
-      && !token->is_keyword(KEYWORD_TYPE))
-    go_warning_at(token->location(), 0,
-		  "ignoring magic comment before non-function");
-
-  std::vector<std::string>* embeds = NULL;
-  if (this->lex_->has_embeds())
-    {
-      embeds = new(std::vector<std::string>);
-      this->lex_->get_and_clear_embeds(embeds);
-
-      if (!this->gogo_->current_file_imported_embed())
-	{
-	  go_error_at(token->location(),
-		      "invalid go:embed: missing import %<embed%>");
-	  delete embeds;
-	  embeds = NULL;
-	}
-      if (!token->is_keyword(KEYWORD_VAR))
-	{
-	  go_error_at(token->location(), "misplaced go:embed directive");
-	  if (embeds != NULL)
-	    {
-	      delete embeds;
-	      embeds = NULL;
-	    }
-	}
-    }
-
   if (token->is_keyword(KEYWORD_CONST))
     this->const_decl();
   else if (token->is_keyword(KEYWORD_TYPE))
-    this->type_decl(pragmas);
+    this->type_decl();
   else if (token->is_keyword(KEYWORD_VAR))
-    this->var_decl(embeds);
+    this->var_decl();
   else if (token->is_keyword(KEYWORD_FUNC))
-    this->function_decl(pragmas);
+    this->function_decl();
   else
     {
       go_error_at(this->location(), "expected declaration");
@@ -1367,8 +1335,7 @@ Parse::declaration_may_start_here()
 // Decl<P> = P | "(" [ List<P> ] ")" .
 
 void
-Parse::decl(void (Parse::*pfn)(unsigned int, std::vector<std::string>*),
-	    unsigned int pragmas, std::vector<std::string>* embeds)
+Parse::decl(void (Parse::*pfn)())
 {
   if (this->peek_token()->is_eof())
     {
@@ -1378,15 +1345,18 @@ Parse::decl(void (Parse::*pfn)(unsigned int, std::vector<std::string>*),
     }
 
   if (!this->peek_token()->is_op(OPERATOR_LPAREN))
-    (this->*pfn)(pragmas, embeds);
+    (this->*pfn)();
   else
     {
-      if (pragmas != 0)
-	go_warning_at(this->location(), 0,
-		      "ignoring magic %<//go:...%> comment before group");
-      if (embeds != NULL)
+      if (this->lex_->get_and_clear_pragmas() != 0)
 	go_error_at(this->location(),
-		    "ignoring %<//go:embed%> comment before group");
+		    "ignoring compiler directive before group");
+      if (this->lex_->has_embeds())
+	{
+	  this->lex_->clear_embeds();
+	  go_error_at(this->location(),
+		      "ignoring %<//go:embed%> comment before group");
+	}
       if (!this->advance_token()->is_op(OPERATOR_RPAREN))
 	{
 	  this->list(pfn, true);
@@ -1410,10 +1380,9 @@ Parse::decl(void (Parse::*pfn)(unsigned int, std::vector<std::string>*),
 // might follow.  This is either a '}' or a ')'.
 
 void
-Parse::list(void (Parse::*pfn)(unsigned int, std::vector<std::string>*),
-	    bool follow_is_paren)
+Parse::list(void (Parse::*pfn)(), bool follow_is_paren)
 {
-  (this->*pfn)(0, NULL);
+  (this->*pfn)();
   Operator follow = follow_is_paren ? OPERATOR_RPAREN : OPERATOR_RCURLY;
   while (this->peek_token()->is_op(OPERATOR_SEMICOLON)
 	 || this->peek_token()->is_op(OPERATOR_COMMA))
@@ -1422,7 +1391,7 @@ Parse::list(void (Parse::*pfn)(unsigned int, std::vector<std::string>*),
 	go_error_at(this->location(), "unexpected comma");
       if (this->advance_token()->is_op(follow))
 	break;
-      (this->*pfn)(0, NULL);
+      (this->*pfn)();
     }
 }
 
@@ -1469,6 +1438,8 @@ Parse::const_decl()
 void
 Parse::const_spec(int iota, Type** last_type, Expression_list** last_expr_list)
 {
+  this->check_directives();
+
   Location loc = this->location();
   Typed_identifier_list til;
   this->identifier_list(&til);
@@ -1545,18 +1516,21 @@ Parse::const_spec(int iota, Type** last_type, Expression_list** last_expr_list)
 // TypeDecl = "type" Decl<TypeSpec> .
 
 void
-Parse::type_decl(unsigned int pragmas)
+Parse::type_decl()
 {
   go_assert(this->peek_token()->is_keyword(KEYWORD_TYPE));
   this->advance_token();
-  this->decl(&Parse::type_spec, pragmas, NULL);
+  this->decl(&Parse::type_spec);
 }
 
 // TypeSpec = identifier ["="] Type .
 
 void
-Parse::type_spec(unsigned int pragmas, std::vector<std::string>*)
+Parse::type_spec()
 {
+  unsigned int pragmas = this->lex_->get_and_clear_pragmas();
+  this->check_directives();
+
   const Token* token = this->peek_token();
   if (!token->is_identifier())
     {
@@ -1649,23 +1623,34 @@ Parse::type_spec(unsigned int pragmas, std::vector<std::string>*)
 // VarDecl = "var" Decl<VarSpec> .
 
 void
-Parse::var_decl(std::vector<std::string>* embeds)
+Parse::var_decl()
 {
   go_assert(this->peek_token()->is_keyword(KEYWORD_VAR));
   this->advance_token();
-  this->decl(&Parse::var_spec, 0, embeds);
+  this->decl(&Parse::var_spec);
 }
 
 // VarSpec = IdentifierList
 //             ( CompleteType [ "=" ExpressionList ] | "=" ExpressionList ) .
 
 void
-Parse::var_spec(unsigned int pragmas, std::vector<std::string>* embeds)
+Parse::var_spec()
 {
   Location loc = this->location();
 
-  if (pragmas != 0)
-    go_warning_at(loc, 0, "ignoring magic %<//go:...%> comment before var");
+  std::vector<std::string>* embeds = NULL;
+  if (this->lex_->has_embeds())
+    {
+      if (!this->gogo_->current_file_imported_embed())
+	go_error_at(loc, "invalid go:embed: missing import %<embed%>");
+      else
+	{
+	  embeds = new(std::vector<std::string>);
+	  this->lex_->get_and_clear_embeds(embeds);
+	}
+    }
+
+  this->check_directives();
 
   // Get the variable names.
   Typed_identifier_list til;
@@ -2339,9 +2324,13 @@ Parse::simple_var_decl_or_assignment(const std::string& name,
 // PRAGMAS is a bitset of magic comments.
 
 void
-Parse::function_decl(unsigned int pragmas)
+Parse::function_decl()
 {
   go_assert(this->peek_token()->is_keyword(KEYWORD_FUNC));
+
+  unsigned int pragmas = this->lex_->get_and_clear_pragmas();
+  this->check_directives();
+
   Location location = this->location();
   std::string extern_name = this->lex_->extern_name();
   const Token* token = this->advance_token();
@@ -5370,7 +5359,7 @@ Parse::for_stat(Label* label)
 	{
 	  go_error_at(this->location(),
                       "var declaration not allowed in for initializer");
-	  this->var_decl(NULL);
+	  this->var_decl();
 	}
 
       if (token->is_op(OPERATOR_SEMICOLON))
@@ -5815,17 +5804,15 @@ Parse::import_decl()
 {
   go_assert(this->peek_token()->is_keyword(KEYWORD_IMPORT));
   this->advance_token();
-  this->decl(&Parse::import_spec, 0, NULL);
+  this->decl(&Parse::import_spec);
 }
 
 // ImportSpec = [ "." | PackageName ] PackageFileName .
 
 void
-Parse::import_spec(unsigned int pragmas, std::vector<std::string>*)
+Parse::import_spec()
 {
-  if (pragmas != 0)
-    go_warning_at(this->location(), 0,
-		  "ignoring magic %<//go:...%> comment before import");
+  this->check_directives();
 
   const Token* token = this->peek_token();
   Location location = token->location();
@@ -5916,6 +5903,23 @@ Parse::program()
 	  this->skip_past_error(OPERATOR_INVALID);
 	}
     }
+
+  this->check_directives();
+}
+
+// If there are any pending compiler directives, clear them and give
+// an error.  This is called when directives are not permitted.
+
+void
+Parse::check_directives()
+{
+  if (this->lex_->get_and_clear_pragmas() != 0)
+    go_error_at(this->location(), "misplaced compiler directive");
+  if (this->lex_->has_embeds())
+    {
+      this->lex_->clear_embeds();
+      go_error_at(this->location(), "misplaced go:embed directive");
+    }
 }
 
 // Skip forward to a semicolon or OP.  OP will normally be
diff --git a/gcc/go/gofrontend/parse.h b/gcc/go/gofrontend/parse.h
index 2c3c505..6e300ef 100644
--- a/gcc/go/gofrontend/parse.h
+++ b/gcc/go/gofrontend/parse.h
@@ -181,15 +181,14 @@ class Parse
   void method_spec(Typed_identifier_list*);
   void declaration();
   bool declaration_may_start_here();
-  void decl(void (Parse::*)(unsigned int, std::vector<std::string>*),
-	    unsigned int pragmas, std::vector<std::string>* embeds);
-  void list(void (Parse::*)(unsigned int, std::vector<std::string>*), bool);
+  void decl(void (Parse::*)());
+  void list(void (Parse::*)(), bool);
   void const_decl();
   void const_spec(int, Type**, Expression_list**);
-  void type_decl(unsigned int pragmas);
-  void type_spec(unsigned int pragmas, std::vector<std::string>*);
-  void var_decl(std::vector<std::string>* embeds);
-  void var_spec(unsigned int pragmas, std::vector<std::string>*);
+  void type_decl();
+  void type_spec();
+  void var_decl();
+  void var_spec();
   void init_vars(const Typed_identifier_list*, Type*, Expression_list*,
 		 bool is_coloneq, std::vector<std::string>*, Location);
   bool init_vars_from_call(const Typed_identifier_list*, Type*, Expression*,
@@ -210,7 +209,7 @@ class Parse
   void simple_var_decl_or_assignment(const std::string&, Location,
 				     bool may_be_composite_lit,
 				     Range_clause*, Type_switch*);
-  void function_decl(unsigned int pragmas);
+  void function_decl();
   Typed_identifier* receiver();
   Expression* operand(bool may_be_sink, bool *is_parenthesized);
   Expression* enclosing_var_reference(Named_object*, Named_object*,
@@ -278,7 +277,10 @@ class Parse
   void goto_stat();
   void package_clause();
   void import_decl();
-  void import_spec(unsigned int pragmas, std::vector<std::string>*);
+  void import_spec();
+
+  // Check for unused compiler directives.
+  void check_directives();
 
   // Skip past an error looking for a semicolon or OP.  Return true if
   // all is well, false if we found EOF.
diff --git a/gcc/input.c b/gcc/input.c
index dd753de..4650547 100644
--- a/gcc/input.c
+++ b/gcc/input.c
@@ -1060,7 +1060,8 @@ make_location (location_t caret, source_range src_range)
    source line in order to calculate the display width.  If that cannot be done
    for any reason, then returns the byte column as a fallback.  */
 int
-location_compute_display_column (expanded_location exploc, int tabstop)
+location_compute_display_column (expanded_location exploc,
+				 const cpp_char_column_policy &policy)
 {
   if (!(exploc.file && *exploc.file && exploc.line && exploc.column))
     return exploc.column;
@@ -1068,7 +1069,7 @@ location_compute_display_column (expanded_location exploc, int tabstop)
   /* If line is NULL, this function returns exploc.column which is the
      desired fallback.  */
   return cpp_byte_column_to_display_column (line.get_buffer (), line.length (),
-					    exploc.column, tabstop);
+					    exploc.column, policy);
 }
 
 /* Dump statistics to stderr about the memory usage of the line_table
@@ -3767,43 +3768,50 @@ test_line_offset_overflow ()
 void test_cpp_utf8 ()
 {
   const int def_tabstop = 8;
+  cpp_char_column_policy policy (def_tabstop, cpp_wcwidth);
+
   /* Verify that wcwidth of invalid UTF-8 or control bytes is 1.  */
   {
-    int w_bad = cpp_display_width ("\xf0!\x9f!\x98!\x82!", 8, def_tabstop);
+    int w_bad = cpp_display_width ("\xf0!\x9f!\x98!\x82!", 8, policy);
     ASSERT_EQ (8, w_bad);
-    int w_ctrl = cpp_display_width ("\r\n\v\0\1", 5, def_tabstop);
+    int w_ctrl = cpp_display_width ("\r\n\v\0\1", 5, policy);
     ASSERT_EQ (5, w_ctrl);
   }
 
   /* Verify that wcwidth of valid UTF-8 is as expected.  */
   {
-    const int w_pi = cpp_display_width ("\xcf\x80", 2, def_tabstop);
+    const int w_pi = cpp_display_width ("\xcf\x80", 2, policy);
     ASSERT_EQ (1, w_pi);
-    const int w_emoji = cpp_display_width ("\xf0\x9f\x98\x82", 4, def_tabstop);
+    const int w_emoji = cpp_display_width ("\xf0\x9f\x98\x82", 4, policy);
     ASSERT_EQ (2, w_emoji);
     const int w_umlaut_precomposed = cpp_display_width ("\xc3\xbf", 2,
-							def_tabstop);
+							policy);
     ASSERT_EQ (1, w_umlaut_precomposed);
     const int w_umlaut_combining = cpp_display_width ("y\xcc\x88", 3,
-						      def_tabstop);
+						      policy);
     ASSERT_EQ (1, w_umlaut_combining);
-    const int w_han = cpp_display_width ("\xe4\xb8\xba", 3, def_tabstop);
+    const int w_han = cpp_display_width ("\xe4\xb8\xba", 3, policy);
     ASSERT_EQ (2, w_han);
-    const int w_ascii = cpp_display_width ("GCC", 3, def_tabstop);
+    const int w_ascii = cpp_display_width ("GCC", 3, policy);
     ASSERT_EQ (3, w_ascii);
     const int w_mixed = cpp_display_width ("\xcf\x80 = 3.14 \xf0\x9f\x98\x82"
 					   "\x9f! \xe4\xb8\xba y\xcc\x88",
-					   24, def_tabstop);
+					   24, policy);
     ASSERT_EQ (18, w_mixed);
   }
 
   /* Verify that display width properly expands tabs.  */
   {
     const char *tstr = "\tabc\td";
-    ASSERT_EQ (6, cpp_display_width (tstr, 6, 1));
-    ASSERT_EQ (10, cpp_display_width (tstr, 6, 3));
-    ASSERT_EQ (17, cpp_display_width (tstr, 6, 8));
-    ASSERT_EQ (1, cpp_display_column_to_byte_column (tstr, 6, 7, 8));
+    ASSERT_EQ (6, cpp_display_width (tstr, 6,
+				     cpp_char_column_policy (1, cpp_wcwidth)));
+    ASSERT_EQ (10, cpp_display_width (tstr, 6,
+				      cpp_char_column_policy (3, cpp_wcwidth)));
+    ASSERT_EQ (17, cpp_display_width (tstr, 6,
+				      cpp_char_column_policy (8, cpp_wcwidth)));
+    ASSERT_EQ (1,
+	       cpp_display_column_to_byte_column
+		 (tstr, 6, 7, cpp_char_column_policy (8, cpp_wcwidth)));
   }
 
   /* Verify that cpp_byte_column_to_display_column can go past the end,
@@ -3816,13 +3824,13 @@ void test_cpp_utf8 ()
       /* 111122223456
 	 Byte columns.  */
 
-    ASSERT_EQ (5, cpp_display_width (str, 6, def_tabstop));
+    ASSERT_EQ (5, cpp_display_width (str, 6, policy));
     ASSERT_EQ (105,
-	       cpp_byte_column_to_display_column (str, 6, 106, def_tabstop));
+	       cpp_byte_column_to_display_column (str, 6, 106, policy));
     ASSERT_EQ (10000,
-	       cpp_byte_column_to_display_column (NULL, 0, 10000, def_tabstop));
+	       cpp_byte_column_to_display_column (NULL, 0, 10000, policy));
     ASSERT_EQ (0,
-	       cpp_byte_column_to_display_column (NULL, 10000, 0, def_tabstop));
+	       cpp_byte_column_to_display_column (NULL, 10000, 0, policy));
   }
 
   /* Verify that cpp_display_column_to_byte_column can go past the end,
@@ -3836,25 +3844,25 @@ void test_cpp_utf8 ()
       /* 000000000000000000000000000000000111111
 	 111122223333444456666777788889999012345
 	 Byte columns.  */
-    ASSERT_EQ (4, cpp_display_column_to_byte_column (str, 15, 2, def_tabstop));
+    ASSERT_EQ (4, cpp_display_column_to_byte_column (str, 15, 2, policy));
     ASSERT_EQ (15,
-	       cpp_display_column_to_byte_column (str, 15, 11, def_tabstop));
+	       cpp_display_column_to_byte_column (str, 15, 11, policy));
     ASSERT_EQ (115,
-	       cpp_display_column_to_byte_column (str, 15, 111, def_tabstop));
+	       cpp_display_column_to_byte_column (str, 15, 111, policy));
     ASSERT_EQ (10000,
-	       cpp_display_column_to_byte_column (NULL, 0, 10000, def_tabstop));
+	       cpp_display_column_to_byte_column (NULL, 0, 10000, policy));
     ASSERT_EQ (0,
-	       cpp_display_column_to_byte_column (NULL, 10000, 0, def_tabstop));
+	       cpp_display_column_to_byte_column (NULL, 10000, 0, policy));
 
     /* Verify that we do not interrupt a UTF-8 sequence.  */
-    ASSERT_EQ (4, cpp_display_column_to_byte_column (str, 15, 1, def_tabstop));
+    ASSERT_EQ (4, cpp_display_column_to_byte_column (str, 15, 1, policy));
 
     for (int byte_col = 1; byte_col <= 15; ++byte_col)
       {
 	const int disp_col
-	  = cpp_byte_column_to_display_column (str, 15, byte_col, def_tabstop);
+	  = cpp_byte_column_to_display_column (str, 15, byte_col, policy);
 	const int byte_col2
-	  = cpp_display_column_to_byte_column (str, 15, disp_col, def_tabstop);
+	  = cpp_display_column_to_byte_column (str, 15, disp_col, policy);
 
 	/* If we ask for the display column in the middle of a UTF-8
 	   sequence, it will return the length of the partial sequence,
diff --git a/gcc/input.h b/gcc/input.h
index e688107..bc44ba2 100644
--- a/gcc/input.h
+++ b/gcc/input.h
@@ -36,11 +36,36 @@ extern GTY(()) class line_maps *saved_line_table;
    both UNKNOWN_LOCATION and BUILTINS_LOCATION fit into that.  */
 STATIC_ASSERT (BUILTINS_LOCATION < RESERVED_LOCATION_COUNT);
 
+/* Hasher for 'location_t' values satisfying '!RESERVED_LOCATION_P', thus able
+   to use 'UNKNOWN_LOCATION'/'BUILTINS_LOCATION' as spare values for
+   'Empty'/'Deleted'.  */
+/* Per PR103157 "'gengtype': 'typedef' causing infinite-recursion code to be
+   generated", don't use
+       typedef int_hash<location_t, UNKNOWN_LOCATION, BUILTINS_LOCATION>
+         location_hash;
+   here.
+
+   It works for a single-use case, but when using a 'struct'-based variant
+       struct location_hash
+         : int_hash<location_t, UNKNOWN_LOCATION, BUILTINS_LOCATION> {};
+   in more than one place, 'gengtype' generates duplicate functions (thus:
+   "error: redefinition of 'void gt_ggc_mx(location_hash&)'" etc.).
+   Attempting to mark that one up with GTY options, we run into a 'gengtype'
+   "parse error: expected '{', have '<'", which probably falls into category
+   "understanding of C++ is limited", as documented in 'gcc/doc/gty.texi'.
+
+   Thus, use a plain ol' '#define':
+*/
+#define location_hash int_hash<location_t, UNKNOWN_LOCATION, BUILTINS_LOCATION>
+
 extern bool is_location_from_builtin_token (location_t);
 extern expanded_location expand_location (location_t);
 
-extern int location_compute_display_column (expanded_location exploc,
-					    int tabstop);
+class cpp_char_column_policy;
+
+extern int
+location_compute_display_column (expanded_location exploc,
+				 const cpp_char_column_policy &policy);
 
 /* A class capturing the bounds of a buffer, to allow for run-time
    bounds-checking in a checked build.  */
@@ -230,8 +255,6 @@ public:
   location_t * GTY ((atomic)) m_locs;
 };
 
-struct location_hash : int_hash <location_t, UNKNOWN_LOCATION> { };
-
 class GTY(()) string_concat_db
 {
  public:
diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c
index 9e10da0..e8fd16b 100644
--- a/gcc/internal-fn.c
+++ b/gcc/internal-fn.c
@@ -3059,10 +3059,10 @@ expand_DEFERRED_INIT (internal_fn, gcall *stmt)
       mark_addressable (lhs);
       tree var_addr = build_fold_addr_expr (lhs);
 
-      tree value = (init_type == AUTO_INIT_PATTERN) ?
-		    build_int_cst (integer_type_node,
-				   INIT_PATTERN_VALUE) :
-		    integer_zero_node;
+      tree value = (init_type == AUTO_INIT_PATTERN)
+		    ? build_int_cst (integer_type_node,
+				     INIT_PATTERN_VALUE)
+		    : integer_zero_node;
       tree m_call = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET),
 				     3, var_addr, value, var_size);
       /* Expand this memset call.  */
@@ -3070,29 +3070,32 @@ expand_DEFERRED_INIT (internal_fn, gcall *stmt)
     }
   else
     {
-      /* If this variable is in a register use expand_assignment.  */
+      /* If this variable is in a register use expand_assignment.
+	 For boolean scalars force zero-init.  */
       tree init;
-      if (tree_fits_uhwi_p (var_size)
+      scalar_int_mode var_mode;
+      if (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE
+	  && tree_fits_uhwi_p (var_size)
 	  && (init_type == AUTO_INIT_PATTERN
 	      || !is_gimple_reg_type (var_type))
 	  && int_mode_for_size (tree_to_uhwi (var_size) * BITS_PER_UNIT,
-				0).exists ())
+				0).exists (&var_mode)
+	  && have_insn_for (SET, var_mode))
 	{
 	  unsigned HOST_WIDE_INT total_bytes = tree_to_uhwi (var_size);
-	  unsigned char *buf = (unsigned char *) xmalloc (total_bytes);
+	  unsigned char *buf = XALLOCAVEC (unsigned char, total_bytes);
 	  memset (buf, (init_type == AUTO_INIT_PATTERN
 			? INIT_PATTERN_VALUE : 0), total_bytes);
-	  if (can_native_interpret_type_p (var_type))
-	    init = native_interpret_expr (var_type, buf, total_bytes);
+	  tree itype = build_nonstandard_integer_type
+			 (total_bytes * BITS_PER_UNIT, 1);
+	  wide_int w = wi::from_buffer (buf, total_bytes);
+	  init = wide_int_to_tree (itype, w);
+	  /* Pun the LHS to make sure its type has constant size
+	     unless it is an SSA name where that's already known.  */
+	  if (TREE_CODE (lhs) != SSA_NAME)
+	    lhs = build1 (VIEW_CONVERT_EXPR, itype, lhs);
 	  else
-	    {
-	      tree itype = build_nonstandard_integer_type
-			     (total_bytes * BITS_PER_UNIT, 1);
-	      wide_int w = wi::from_buffer (buf, total_bytes);
-	      init = wide_int_to_tree (itype, w);
-	      /* Pun the LHS to make sure its type has constant size.  */
-	      lhs = build1 (VIEW_CONVERT_EXPR, itype, lhs);
-	    }
+	    init = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), init);
 	}
       else
 	/* Use zero-init also for variable-length sizes.  */
diff --git a/gcc/ipa-cp.c b/gcc/ipa-cp.c
index b987d97..703541d 100644
--- a/gcc/ipa-cp.c
+++ b/gcc/ipa-cp.c
@@ -400,9 +400,9 @@ object_allocator<ipcp_value_source<tree> > ipcp_sources_pool
 object_allocator<ipcp_agg_lattice> ipcp_agg_lattice_pool
   ("IPA_CP aggregate lattices");
 
-/* Maximal count found in program.  */
+/* Base count to use in heuristics when using profile feedback.  */
 
-static profile_count max_count;
+static profile_count base_count;
 
 /* Original overall size of the program.  */
 
@@ -701,20 +701,36 @@ ipcp_versionable_function_p (struct cgraph_node *node)
 
 struct caller_statistics
 {
+  /* If requested (see below), self-recursive call counts are summed into this
+     field.  */
+  profile_count rec_count_sum;
+  /* The sum of all ipa counts of all the other (non-recursive) calls.  */
   profile_count count_sum;
+  /* Sum of all frequencies for all calls.  */
   sreal freq_sum;
+  /* Number of calls and hot calls respectively.  */
   int n_calls, n_hot_calls;
+  /* If itself is set up, also count the number of non-self-recursive
+     calls.  */
+  int n_nonrec_calls;
+  /* If non-NULL, this is the node itself and calls from it should have their
+     counts included in rec_count_sum and not count_sum.  */
+  cgraph_node *itself;
 };
 
-/* Initialize fields of STAT to zeroes.  */
+/* Initialize fields of STAT to zeroes and optionally set it up so that edges
+   from IGNORED_CALLER are not counted.  */
 
 static inline void
-init_caller_stats (struct caller_statistics *stats)
+init_caller_stats (caller_statistics *stats, cgraph_node *itself = NULL)
 {
+  stats->rec_count_sum = profile_count::zero ();
   stats->count_sum = profile_count::zero ();
   stats->n_calls = 0;
   stats->n_hot_calls = 0;
+  stats->n_nonrec_calls = 0;
   stats->freq_sum = 0;
+  stats->itself = itself;
 }
 
 /* Worker callback of cgraph_for_node_and_aliases accumulating statistics of
@@ -729,10 +745,22 @@ gather_caller_stats (struct cgraph_node *node, void *data)
   for (cs = node->callers; cs; cs = cs->next_caller)
     if (!cs->caller->thunk)
       {
-        if (cs->count.ipa ().initialized_p ())
-	  stats->count_sum += cs->count.ipa ();
+	ipa_node_params *info = ipa_node_params_sum->get (cs->caller);
+	if (info && info->node_dead)
+	  continue;
+
+	if (cs->count.ipa ().initialized_p ())
+	  {
+	    if (stats->itself && stats->itself == cs->caller)
+	      stats->rec_count_sum += cs->count.ipa ();
+	    else
+	      stats->count_sum += cs->count.ipa ();
+	  }
 	stats->freq_sum += cs->sreal_frequency ();
 	stats->n_calls++;
+	if (stats->itself && stats->itself != cs->caller)
+	  stats->n_nonrec_calls++;
+
 	if (cs->maybe_hot_p ())
 	  stats->n_hot_calls ++;
       }
@@ -781,7 +809,8 @@ ipcp_cloning_candidate_p (struct cgraph_node *node)
   /* When profile is available and function is hot, propagate into it even if
      calls seems cold; constant propagation can improve function's speed
      significantly.  */
-  if (max_count > profile_count::zero ())
+  if (stats.count_sum > profile_count::zero ()
+      && node->count.ipa ().initialized_p ())
     {
       if (stats.count_sum > node->count.ipa ().apply_scale (90, 100))
 	{
@@ -3282,10 +3311,10 @@ good_cloning_opportunity_p (struct cgraph_node *node, sreal time_benefit,
 
   ipa_node_params *info = ipa_node_params_sum->get (node);
   int eval_threshold = opt_for_fn (node->decl, param_ipa_cp_eval_threshold);
-  if (max_count > profile_count::zero ())
+  if (count_sum > profile_count::zero ())
     {
-
-      sreal factor = count_sum.probability_in (max_count).to_sreal ();
+      gcc_assert (base_count > profile_count::zero ());
+      sreal factor = count_sum.probability_in (base_count).to_sreal ();
       sreal evaluation = (time_benefit * factor) / size_cost;
       evaluation = incorporate_penalties (node, info, evaluation);
       evaluation *= 1000;
@@ -3922,6 +3951,21 @@ value_topo_info<valtype>::propagate_effects ()
     }
 }
 
+/* Callback for qsort to sort counts of all edges.  */
+
+static int
+compare_edge_profile_counts (const void *a, const void *b)
+{
+  const profile_count *cnt1 = (const profile_count *) a;
+  const profile_count *cnt2 = (const profile_count *) b;
+
+  if (*cnt1 < *cnt2)
+    return 1;
+  if (*cnt1 > *cnt2)
+    return -1;
+  return 0;
+}
+
 
 /* Propagate constants, polymorphic contexts and their effects from the
    summaries interprocedurally.  */
@@ -3934,8 +3978,10 @@ ipcp_propagate_stage (class ipa_topo_info *topo)
   if (dump_file)
     fprintf (dump_file, "\n Propagating constants:\n\n");
 
-  max_count = profile_count::uninitialized ();
+  base_count = profile_count::uninitialized ();
 
+  bool compute_count_base = false;
+  unsigned base_count_pos_percent = 0;
   FOR_EACH_DEFINED_FUNCTION (node)
   {
     if (node->has_gimple_body_p ()
@@ -3953,9 +3999,57 @@ ipcp_propagate_stage (class ipa_topo_info *topo)
     ipa_size_summary *s = ipa_size_summaries->get (node);
     if (node->definition && !node->alias && s != NULL)
       overall_size += s->self_size;
-    max_count = max_count.max (node->count.ipa ());
+    if (node->count.ipa ().initialized_p ())
+      {
+	compute_count_base = true;
+	unsigned pos_percent = opt_for_fn (node->decl,
+					   param_ipa_cp_profile_count_base);
+	base_count_pos_percent = MAX (base_count_pos_percent, pos_percent);
+      }
   }
 
+  if (compute_count_base)
+    {
+      auto_vec<profile_count> all_edge_counts;
+      all_edge_counts.reserve_exact (symtab->edges_count);
+      FOR_EACH_DEFINED_FUNCTION (node)
+	for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
+	  {
+	    profile_count count = cs->count.ipa ();
+	    if (!(count > profile_count::zero ()))
+	      continue;
+
+	    enum availability avail;
+	    cgraph_node *tgt
+	      = cs->callee->function_or_virtual_thunk_symbol (&avail);
+	    ipa_node_params *info = ipa_node_params_sum->get (tgt);
+	    if (info && info->versionable)
+	      all_edge_counts.quick_push (count);
+	  }
+
+      if (!all_edge_counts.is_empty ())
+	{
+	  gcc_assert (base_count_pos_percent <= 100);
+	  all_edge_counts.qsort (compare_edge_profile_counts);
+
+	  unsigned base_count_pos
+	    = ((all_edge_counts.length () * (base_count_pos_percent)) / 100);
+	  base_count = all_edge_counts[base_count_pos];
+
+	  if (dump_file)
+	    {
+	      fprintf (dump_file, "\nSelected base_count from %u edges at "
+		       "position %u, arriving at: ", all_edge_counts.length (),
+		       base_count_pos);
+	      base_count.dump (dump_file);
+	      fprintf (dump_file, "\n");
+	    }
+	}
+      else if (dump_file)
+	fprintf (dump_file, "\nNo candidates with non-zero call count found, "
+		 "continuing as if without profile feedback.\n");
+    }
+
   orig_overall_size = overall_size;
 
   if (dump_file)
@@ -4202,19 +4296,22 @@ get_next_cgraph_edge_clone (struct cgraph_edge *cs)
 
 /* Given VAL that is intended for DEST, iterate over all its sources and if any
    of them is viable and hot, return true.  In that case, for those that still
-   hold, add their edge frequency and their number into *FREQUENCY and
-   *CALLER_COUNT respectively.  */
+   hold, add their edge frequency and their number and cumulative profile
+   counts of self-ecursive and other edges into *FREQUENCY, *CALLER_COUNT,
+   REC_COUNT_SUM and NONREC_COUNT_SUM respectively.  */
 
 template <typename valtype>
 static bool
 get_info_about_necessary_edges (ipcp_value<valtype> *val, cgraph_node *dest,
-				sreal *freq_sum, profile_count *count_sum,
-				int *caller_count)
+				sreal *freq_sum, int *caller_count,
+				profile_count *rec_count_sum,
+				profile_count *nonrec_count_sum)
 {
   ipcp_value_source<valtype> *src;
   sreal freq = 0;
   int count = 0;
-  profile_count cnt = profile_count::zero ();
+  profile_count rec_cnt = profile_count::zero ();
+  profile_count nonrec_cnt = profile_count::zero ();
   bool hot = false;
   bool non_self_recursive = false;
 
@@ -4227,11 +4324,15 @@ get_info_about_necessary_edges (ipcp_value<valtype> *val, cgraph_node *dest,
 	    {
 	      count++;
 	      freq += cs->sreal_frequency ();
-	      if (cs->count.ipa ().initialized_p ())
-	        cnt += cs->count.ipa ();
 	      hot |= cs->maybe_hot_p ();
 	      if (cs->caller != dest)
-		non_self_recursive = true;
+		{
+		  non_self_recursive = true;
+		  if (cs->count.ipa ().initialized_p ())
+		    rec_cnt += cs->count.ipa ();
+		}
+	      else if (cs->count.ipa ().initialized_p ())
+	        nonrec_cnt += cs->count.ipa ();
 	    }
 	  cs = get_next_cgraph_edge_clone (cs);
 	}
@@ -4243,8 +4344,9 @@ get_info_about_necessary_edges (ipcp_value<valtype> *val, cgraph_node *dest,
     return false;
 
   *freq_sum = freq;
-  *count_sum = cnt;
   *caller_count = count;
+  *rec_count_sum = rec_cnt;
+  *nonrec_count_sum = nonrec_cnt;
 
   if (!hot && ipa_node_params_sum->get (dest)->node_within_scc)
     {
@@ -4349,112 +4451,399 @@ get_replacement_map (class ipa_node_params *info, tree value, int parm_num,
   return replace_map;
 }
 
-/* Dump new profiling counts */
+/* Dump new profiling counts of NODE.  SPEC is true when NODE is a specialzied
+   one, otherwise it will be referred to as the original node.  */
 
 static void
-dump_profile_updates (struct cgraph_node *orig_node,
-		      struct cgraph_node *new_node)
+dump_profile_updates (cgraph_node *node, bool spec)
 {
-  struct cgraph_edge *cs;
+  if (spec)
+    fprintf (dump_file, "     setting count of the specialized node %s to ",
+	     node->dump_name ());
+  else
+    fprintf (dump_file, "     setting count of the original node %s to ",
+	     node->dump_name ());
 
-  fprintf (dump_file, "    setting count of the specialized node to ");
-  new_node->count.dump (dump_file);
+  node->count.dump (dump_file);
   fprintf (dump_file, "\n");
-  for (cs = new_node->callees; cs; cs = cs->next_callee)
+  for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
     {
-      fprintf (dump_file, "      edge to %s has count ",
+      fprintf (dump_file, "       edge to %s has count ",
 	       cs->callee->dump_name ());
       cs->count.dump (dump_file);
       fprintf (dump_file, "\n");
     }
+}
 
-  fprintf (dump_file, "    setting count of the original node to ");
-  orig_node->count.dump (dump_file);
-  fprintf (dump_file, "\n");
-  for (cs = orig_node->callees; cs; cs = cs->next_callee)
+/* With partial train run we do not want to assume that original's count is
+   zero whenever we redurect all executed edges to clone.  Simply drop profile
+   to local one in this case.  In eany case, return the new value.  ORIG_NODE
+   is the original node and its count has not been updaed yet.  */
+
+profile_count
+lenient_count_portion_handling (profile_count remainder, cgraph_node *orig_node)
+{
+  if (remainder.ipa_p () && !remainder.ipa ().nonzero_p ()
+      && orig_node->count.ipa_p () && orig_node->count.ipa ().nonzero_p ()
+      && opt_for_fn (orig_node->decl, flag_profile_partial_training))
+    remainder = remainder.guessed_local ();
+
+  return remainder;
+}
+
+/* Structure to sum counts coming from nodes other than the original node and
+   its clones.  */
+
+struct gather_other_count_struct
+{
+  cgraph_node *orig;
+  profile_count other_count;
+};
+
+/* Worker callback of call_for_symbol_thunks_and_aliases summing the number of
+   counts that come from non-self-recursive calls..  */
+
+static bool
+gather_count_of_non_rec_edges (cgraph_node *node, void *data)
+{
+  gather_other_count_struct *desc = (gather_other_count_struct *) data;
+  for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
+    if (cs->caller != desc->orig && cs->caller->clone_of != desc->orig)
+      desc->other_count += cs->count.ipa ();
+  return false;
+}
+
+/* Structure to help analyze if we need to boost counts of some clones of some
+   non-recursive edges to match the new callee count.  */
+
+struct desc_incoming_count_struct
+{
+  cgraph_node *orig;
+  hash_set <cgraph_edge *> *processed_edges;
+  profile_count count;
+  unsigned unproc_orig_rec_edges;
+};
+
+/* Go over edges calling NODE and its thunks and gather information about
+   incoming counts so that we know if we need to make any adjustments.  */
+
+static void
+analyze_clone_icoming_counts (cgraph_node *node,
+			      desc_incoming_count_struct *desc)
+{
+  for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
+    if (cs->caller->thunk)
+      {
+	analyze_clone_icoming_counts (cs->caller, desc);
+	continue;
+      }
+    else
+      {
+	if (cs->count.initialized_p ())
+	  desc->count += cs->count.ipa ();
+	if (!desc->processed_edges->contains (cs)
+	    && cs->caller->clone_of == desc->orig)
+	  desc->unproc_orig_rec_edges++;
+      }
+}
+
+/* If caller edge counts of a clone created for a self-recursive arithmetic
+   jump function must be adjusted because it is coming from a the "seed" clone
+   for the first value and so has been excessively scaled back as if it was not
+   a recursive call, adjust it so that the incoming counts of NODE match its
+   count. NODE is the node or its thunk.  */
+
+static void
+adjust_clone_incoming_counts (cgraph_node *node,
+			      desc_incoming_count_struct *desc)
+{
+  for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
+    if (cs->caller->thunk)
+      {
+	adjust_clone_incoming_counts (cs->caller, desc);
+	profile_count sum = profile_count::zero ();
+	for (cgraph_edge *e = cs->caller->callers; e; e = e->next_caller)
+	  if (e->count.initialized_p ())
+	    sum += e->count.ipa ();
+	cs->count = cs->count.combine_with_ipa_count (sum);
+      }
+    else if (!desc->processed_edges->contains (cs)
+	     && cs->caller->clone_of == desc->orig)
+      {
+	cs->count += desc->count;
+	if (dump_file)
+	  {
+	    fprintf (dump_file, "       Adjusted count of an incoming edge of "
+		     "a clone %s -> %s to ", cs->caller->dump_name (),
+		     cs->callee->dump_name ());
+	    cs->count.dump (dump_file);
+	    fprintf (dump_file, "\n");
+	  }
+      }
+}
+
+/* When ORIG_NODE has been cloned for values which have been generated fora
+   self-recursive call as a result of an arithmetic pass-through
+   jump-functions, adjust its count together with counts of all such clones in
+   SELF_GEN_CLONES which also at this point contains ORIG_NODE itself.
+
+   The function sums the counts of the original node and all its clones that
+   cannot be attributed to a specific clone because it comes from a
+   non-recursive edge.  This sum is then evenly divided between the clones and
+   on top of that each one gets all the counts which can be attributed directly
+   to it.  */
+
+static void
+update_counts_for_self_gen_clones (cgraph_node *orig_node,
+				   const vec<cgraph_node *> &self_gen_clones)
+{
+  profile_count redist_sum = orig_node->count.ipa ();
+  if (!(redist_sum > profile_count::zero ()))
+    return;
+
+  if (dump_file)
+    fprintf (dump_file, "     Updating profile of self recursive clone "
+	     "series\n");
+
+  gather_other_count_struct gocs;
+  gocs.orig = orig_node;
+  gocs.other_count = profile_count::zero ();
+
+  auto_vec <profile_count, 8> other_edges_count;
+  for (cgraph_node *n : self_gen_clones)
+    {
+      gocs.other_count = profile_count::zero ();
+      n->call_for_symbol_thunks_and_aliases (gather_count_of_non_rec_edges,
+					     &gocs, false);
+      other_edges_count.safe_push (gocs.other_count);
+      redist_sum -= gocs.other_count;
+    }
+
+  hash_set<cgraph_edge *> processed_edges;
+  unsigned i = 0;
+  for (cgraph_node *n : self_gen_clones)
+    {
+      profile_count orig_count = n->count;
+      profile_count new_count
+	= (redist_sum.apply_scale (1, self_gen_clones.length ())
+	   + other_edges_count[i]);
+      new_count = lenient_count_portion_handling (new_count, orig_node);
+      n->count = new_count;
+      profile_count::adjust_for_ipa_scaling (&new_count, &orig_count);
+      for (cgraph_edge *cs = n->callees; cs; cs = cs->next_callee)
+	{
+	  cs->count = cs->count.apply_scale (new_count, orig_count);
+	  processed_edges.add (cs);
+	}
+      for (cgraph_edge *cs = n->indirect_calls; cs; cs = cs->next_callee)
+	cs->count = cs->count.apply_scale (new_count, orig_count);
+
+      i++;
+    }
+
+  /* There are still going to be edges to ORIG_NODE that have one or more
+     clones coming from another node clone in SELF_GEN_CLONES and which we
+     scaled by the same amount, which means that the total incoming sum of
+     counts to ORIG_NODE will be too high, scale such edges back.  */
+  for (cgraph_edge *cs = orig_node->callees; cs; cs = cs->next_callee)
     {
-      fprintf (dump_file, "      edge to %s is left with ",
-	       cs->callee->dump_name ());
-      cs->count.dump (dump_file);
-      fprintf (dump_file, "\n");
+      if (cs->callee->ultimate_alias_target () == orig_node)
+	{
+	  unsigned den = 0;
+	  for (cgraph_edge *e = cs; e; e = get_next_cgraph_edge_clone (e))
+	    if (e->callee->ultimate_alias_target () == orig_node
+		&& processed_edges.contains (e))
+	      den++;
+	  if (den > 0)
+	    for (cgraph_edge *e = cs; e; e = get_next_cgraph_edge_clone (e))
+	      if (e->callee->ultimate_alias_target () == orig_node
+		  && processed_edges.contains (e))
+		e->count = e->count.apply_scale (1, den);
+	}
+    }
+
+  /* Edges from the seeds of the valus generated for arithmetic jump-functions
+     along self-recursive edges are likely to have fairly low count and so
+     edges from them to nodes in the self_gen_clones do not correspond to the
+     artificially distributed count of the nodes, the total sum of incoming
+     edges to some clones might be too low.  Detect this situation and correct
+     it.  */
+  for (cgraph_node *n : self_gen_clones)
+    {
+      if (!(n->count.ipa () > profile_count::zero ()))
+	continue;
+
+      desc_incoming_count_struct desc;
+      desc.orig = orig_node;
+      desc.processed_edges = &processed_edges;
+      desc.count = profile_count::zero ();
+      desc.unproc_orig_rec_edges = 0;
+      analyze_clone_icoming_counts (n, &desc);
+
+      if (n->count.differs_from_p (desc.count))
+	{
+	  if (n->count > desc.count
+	      && desc.unproc_orig_rec_edges > 0)
+	    {
+	      desc.count = n->count - desc.count;
+	      desc.count
+		= desc.count.apply_scale (1, desc.unproc_orig_rec_edges);
+	      adjust_clone_incoming_counts (n, &desc);
+	    }
+	  else if (dump_file)
+	    fprintf (dump_file,
+		     "       Unable to fix up incoming counts for %s.\n",
+		     n->dump_name ());
+	}
     }
+
+  if (dump_file)
+    for (cgraph_node *n : self_gen_clones)
+      dump_profile_updates (n, n != orig_node);
+  return;
 }
 
 /* After a specialized NEW_NODE version of ORIG_NODE has been created, update
-   their profile information to reflect this.  */
+   their profile information to reflect this.  This function should not be used
+   for clones generated for arithmetic pass-through jump functions on a
+   self-recursive call graph edge, that situation is handled by
+   update_counts_for_self_gen_clones.  */
 
 static void
 update_profiling_info (struct cgraph_node *orig_node,
 		       struct cgraph_node *new_node)
 {
-  struct cgraph_edge *cs;
   struct caller_statistics stats;
-  profile_count new_sum, orig_sum;
-  profile_count remainder, orig_node_count = orig_node->count;
-  profile_count orig_new_node_count = new_node->count;
+  profile_count new_sum;
+  profile_count remainder, orig_node_count = orig_node->count.ipa ();
 
-  if (!(orig_node_count.ipa () > profile_count::zero ()))
+  if (!(orig_node_count > profile_count::zero ()))
     return;
 
-  init_caller_stats (&stats);
-  orig_node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats,
-					       false);
-  orig_sum = stats.count_sum;
-  init_caller_stats (&stats);
+  if (dump_file)
+    {
+      fprintf (dump_file, "     Updating profile from original count: ");
+      orig_node_count.dump (dump_file);
+      fprintf (dump_file, "\n");
+    }
+
+  init_caller_stats (&stats, new_node);
   new_node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats,
 					      false);
   new_sum = stats.count_sum;
 
-  if (orig_node_count < orig_sum + new_sum)
+  if (new_sum > orig_node_count)
     {
-      if (dump_file)
+      /* TODO: Perhaps this should be gcc_unreachable ()?  */
+      remainder = profile_count::zero ().guessed_local ();
+    }
+  else if (stats.rec_count_sum.nonzero_p ())
+    {
+      int new_nonrec_calls = stats.n_nonrec_calls;
+      /* There are self-recursive edges which are likely to bring in the
+	 majority of calls but which we must divide in between the original and
+	 new node.  */
+      init_caller_stats (&stats, orig_node);
+      orig_node->call_for_symbol_thunks_and_aliases (gather_caller_stats,
+						     &stats, false);
+      int orig_nonrec_calls = stats.n_nonrec_calls;
+      profile_count orig_nonrec_call_count = stats.count_sum;
+
+      if (orig_node->local)
 	{
-	  fprintf (dump_file, "    Problem: node %s has too low count ",
-		   orig_node->dump_name ());
-	  orig_node_count.dump (dump_file);
-	  fprintf (dump_file, "while the sum of incoming count is ");
-	  (orig_sum + new_sum).dump (dump_file);
-	  fprintf (dump_file, "\n");
+	  if (!orig_nonrec_call_count.nonzero_p ())
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "       The original is local and the only "
+			 "incoming edges from non-dead callers with nonzero "
+			 "counts are self-recursive, assuming it is cold.\n");
+	      /* The NEW_NODE count and counts of all its outgoing edges
+		 are still unmodified copies of ORIG_NODE's.  Just clear
+		 the latter and bail out.  */
+	      profile_count zero;
+              if (opt_for_fn (orig_node->decl, flag_profile_partial_training))
+                zero = profile_count::zero ().guessed_local ();
+	      else
+		zero = profile_count::adjusted_zero ();
+	      orig_node->count = zero;
+	      for (cgraph_edge *cs = orig_node->callees;
+		   cs;
+		   cs = cs->next_callee)
+		cs->count = zero;
+	      for (cgraph_edge *cs = orig_node->indirect_calls;
+		   cs;
+		   cs = cs->next_callee)
+		cs->count = zero;
+	      return;
+	    }
+	}
+      else
+	{
+	  /* Let's behave as if there was another caller that accounts for all
+	     the calls that were either indirect or from other compilation
+	     units. */
+	  orig_nonrec_calls++;
+	  profile_count pretend_caller_count
+	    = (orig_node_count - new_sum - orig_nonrec_call_count
+	       - stats.rec_count_sum);
+	  orig_nonrec_call_count += pretend_caller_count;
 	}
 
-      orig_node_count = (orig_sum + new_sum).apply_scale (12, 10);
+      /* Divide all "unexplained" counts roughly proportionally to sums of
+	 counts of non-recursive calls.
+
+	 We put rather arbitrary limits on how many counts we claim because the
+	 number of non-self-recursive incoming count is only a rough guideline
+	 and there are cases (such as mcf) where using it blindly just takes
+	 too many.  And if lattices are considered in the opposite order we
+	 could also take too few.  */
+      profile_count unexp = orig_node_count - new_sum - orig_nonrec_call_count;
+
+      int limit_den = 2 * (orig_nonrec_calls + new_nonrec_calls);
+      profile_count new_part
+	= MAX(MIN (unexp.apply_scale (new_sum,
+				      new_sum + orig_nonrec_call_count),
+		   unexp.apply_scale (limit_den - 1, limit_den)),
+	      unexp.apply_scale (new_nonrec_calls, limit_den));
       if (dump_file)
 	{
-	  fprintf (dump_file, "      proceeding by pretending it was ");
-	  orig_node_count.dump (dump_file);
-	  fprintf (dump_file, "\n");
+	  fprintf (dump_file, "       Claiming ");
+	  new_part.dump (dump_file);
+	  fprintf (dump_file, " of unexplained ");
+	  unexp.dump (dump_file);
+	  fprintf (dump_file, " counts because of self-recursive "
+		   "calls\n");
 	}
+      new_sum += new_part;
+      remainder = lenient_count_portion_handling (orig_node_count - new_sum,
+						  orig_node);
     }
-
-  remainder = orig_node_count.combine_with_ipa_count (orig_node_count.ipa ()
-						      - new_sum.ipa ());
-
-  /* With partial train run we do not want to assume that original's
-     count is zero whenever we redurect all executed edges to clone.
-     Simply drop profile to local one in this case.  */
-  if (remainder.ipa_p () && !remainder.ipa ().nonzero_p ()
-      && orig_node->count.ipa_p () && orig_node->count.ipa ().nonzero_p ()
-      && flag_profile_partial_training)
-    remainder = remainder.guessed_local ();
+  else
+    remainder = lenient_count_portion_handling (orig_node_count - new_sum,
+						orig_node);
 
   new_sum = orig_node_count.combine_with_ipa_count (new_sum);
   new_node->count = new_sum;
   orig_node->count = remainder;
 
+  profile_count orig_new_node_count = orig_node_count;
   profile_count::adjust_for_ipa_scaling (&new_sum, &orig_new_node_count);
-  for (cs = new_node->callees; cs; cs = cs->next_callee)
+  for (cgraph_edge *cs = new_node->callees; cs; cs = cs->next_callee)
     cs->count = cs->count.apply_scale (new_sum, orig_new_node_count);
-  for (cs = new_node->indirect_calls; cs; cs = cs->next_callee)
+  for (cgraph_edge *cs = new_node->indirect_calls; cs; cs = cs->next_callee)
     cs->count = cs->count.apply_scale (new_sum, orig_new_node_count);
 
   profile_count::adjust_for_ipa_scaling (&remainder, &orig_node_count);
-  for (cs = orig_node->callees; cs; cs = cs->next_callee)
+  for (cgraph_edge *cs = orig_node->callees; cs; cs = cs->next_callee)
     cs->count = cs->count.apply_scale (remainder, orig_node_count);
-  for (cs = orig_node->indirect_calls; cs; cs = cs->next_callee)
+  for (cgraph_edge *cs = orig_node->indirect_calls; cs; cs = cs->next_callee)
     cs->count = cs->count.apply_scale (remainder, orig_node_count);
 
   if (dump_file)
-    dump_profile_updates (orig_node, new_node);
+    {
+      dump_profile_updates (new_node, true);
+      dump_profile_updates (orig_node, false);
+    }
 }
 
 /* Update the respective profile of specialized NEW_NODE and the original
@@ -4495,7 +4884,10 @@ update_specialized_profile (struct cgraph_node *new_node,
     }
 
   if (dump_file)
-    dump_profile_updates (orig_node, new_node);
+    {
+      dump_profile_updates (new_node, true);
+      dump_profile_updates (orig_node, false);
+    }
 }
 
 static void adjust_references_in_caller (cgraph_edge *cs,
@@ -4795,8 +5187,7 @@ create_specialized_node (struct cgraph_node *node,
       if (aggvals)
 	ipa_dump_agg_replacement_values (dump_file, aggvals);
     }
-  ipa_check_create_node_params ();
-  update_profiling_info (node, new_node);
+
   new_info = ipa_node_params_sum->get (new_node);
   new_info->ipcp_orig_node = node;
   new_node->ipcp_clone = true;
@@ -5621,17 +6012,20 @@ ipcp_val_agg_replacement_ok_p (ipa_agg_replacement_value *,
 /* Decide whether to create a special version of NODE for value VAL of
    parameter at the given INDEX.  If OFFSET is -1, the value is for the
    parameter itself, otherwise it is stored at the given OFFSET of the
-   parameter.  AVALS describes the other already known values.  */
+   parameter.  AVALS describes the other already known values.  SELF_GEN_CLONES
+   is a vector which contains clones created for self-recursive calls with an
+   arithmetic pass-through jump function.  */
 
 template <typename valtype>
 static bool
 decide_about_value (struct cgraph_node *node, int index, HOST_WIDE_INT offset,
-		    ipcp_value<valtype> *val, ipa_auto_call_arg_values *avals)
+		    ipcp_value<valtype> *val, ipa_auto_call_arg_values *avals,
+		    vec<cgraph_node *> *self_gen_clones)
 {
   struct ipa_agg_replacement_value *aggvals;
   int caller_count;
   sreal freq_sum;
-  profile_count count_sum;
+  profile_count count_sum, rec_count_sum;
   vec<cgraph_edge *> callers;
 
   if (val->spec_node)
@@ -5647,13 +6041,31 @@ decide_about_value (struct cgraph_node *node, int index, HOST_WIDE_INT offset,
 		 val->local_size_cost + overall_size);
       return false;
     }
-  else if (!get_info_about_necessary_edges (val, node, &freq_sum, &count_sum,
-					    &caller_count))
+  else if (!get_info_about_necessary_edges (val, node, &freq_sum, &caller_count,
+					    &rec_count_sum, &count_sum))
     return false;
 
   if (!dbg_cnt (ipa_cp_values))
     return false;
 
+  if (val->self_recursion_generated_p ())
+    {
+      /* The edge counts in this case might not have been adjusted yet.
+	 Nevertleless, even if they were it would be only a guesswork which we
+	 can do now.  The recursive part of the counts can be derived from the
+	 count of the original node anyway.  */
+      if (node->count.ipa ().nonzero_p ())
+	{
+	  unsigned dem = self_gen_clones->length () + 1;
+	  rec_count_sum = node->count.ipa ().apply_scale (1, dem);
+	}
+      else
+	rec_count_sum = profile_count::zero ();
+    }
+
+  /* get_info_about_necessary_edges only sums up ipa counts.  */
+  count_sum += rec_count_sum;
+
   if (dump_file && (dump_flags & TDF_DETAILS))
     {
       fprintf (dump_file, " - considering value ");
@@ -5694,6 +6106,12 @@ decide_about_value (struct cgraph_node *node, int index, HOST_WIDE_INT offset,
 						      offset, val->value));
   val->spec_node = create_specialized_node (node, known_csts, known_contexts,
 					    aggvals, callers);
+
+  if (val->self_recursion_generated_p ())
+    self_gen_clones->safe_push (val->spec_node);
+  else
+    update_profiling_info (node, val->spec_node);
+
   callers.release ();
   overall_size += val->local_size_cost;
   if (dump_file && (dump_flags & TDF_DETAILS))
@@ -5722,6 +6140,7 @@ decide_whether_version_node (struct cgraph_node *node)
     fprintf (dump_file, "\nEvaluating opportunities for %s.\n",
 	     node->dump_name ());
 
+  auto_vec <cgraph_node *, 9> self_gen_clones;
   ipa_auto_call_arg_values avals;
   gather_context_independent_values (info, &avals, false, NULL);
 
@@ -5736,7 +6155,8 @@ decide_whether_version_node (struct cgraph_node *node)
 	{
 	  ipcp_value<tree> *val;
 	  for (val = lat->values; val; val = val->next)
-	    ret |= decide_about_value (node, i, -1, val, &avals);
+	    ret |= decide_about_value (node, i, -1, val, &avals,
+				       &self_gen_clones);
 	}
 
       if (!plats->aggs_bottom)
@@ -5750,7 +6170,8 @@ decide_whether_version_node (struct cgraph_node *node)
 		&& (plats->aggs_contain_variable
 		    || !aglat->is_single_const ()))
 	      for (val = aglat->values; val; val = val->next)
-		ret |= decide_about_value (node, i, aglat->offset, val, &avals);
+		ret |= decide_about_value (node, i, aglat->offset, val, &avals,
+					   &self_gen_clones);
 	}
 
       if (!ctxlat->bottom
@@ -5758,10 +6179,17 @@ decide_whether_version_node (struct cgraph_node *node)
 	{
 	  ipcp_value<ipa_polymorphic_call_context> *val;
 	  for (val = ctxlat->values; val; val = val->next)
-	    ret |= decide_about_value (node, i, -1, val, &avals);
+	    ret |= decide_about_value (node, i, -1, val, &avals,
+				       &self_gen_clones);
 	}
     }
 
+  if (!self_gen_clones.is_empty ())
+    {
+      self_gen_clones.safe_push (node);
+      update_counts_for_self_gen_clones (node, self_gen_clones);
+    }
+
   if (info->do_clone_for_all_contexts)
     {
       if (!dbg_cnt (ipa_cp_values))
@@ -6205,7 +6633,7 @@ make_pass_ipa_cp (gcc::context *ctxt)
 void
 ipa_cp_c_finalize (void)
 {
-  max_count = profile_count::uninitialized ();
+  base_count = profile_count::uninitialized ();
   overall_size = 0;
   orig_overall_size = 0;
   ipcp_free_transformation_sum ();
diff --git a/gcc/ipa-fnsummary.c b/gcc/ipa-fnsummary.c
index 3119991..7e9201a 100644
--- a/gcc/ipa-fnsummary.c
+++ b/gcc/ipa-fnsummary.c
@@ -86,6 +86,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-into-ssa.h"
 #include "symtab-clones.h"
 #include "gimple-range.h"
+#include "tree-dfa.h"
 
 /* Summaries.  */
 fast_function_summary <ipa_fn_summary *, va_gc> *ipa_fn_summaries;
@@ -93,7 +94,7 @@ fast_function_summary <ipa_size_summary *, va_heap> *ipa_size_summaries;
 fast_call_summary <ipa_call_summary *, va_heap> *ipa_call_summaries;
 
 /* Edge predicates goes here.  */
-static object_allocator<predicate> edge_predicate_pool ("edge predicates");
+static object_allocator<ipa_predicate> edge_predicate_pool ("edge predicates");
 
 
 /* Dump IPA hints.  */
@@ -161,14 +162,14 @@ ipa_dump_hints (FILE *f, ipa_hints hints)
 
 void
 ipa_fn_summary::account_size_time (int size, sreal time,
-				   const predicate &exec_pred,
-				   const predicate &nonconst_pred_in,
+				   const ipa_predicate &exec_pred,
+				   const ipa_predicate &nonconst_pred_in,
 				   bool call)
 {
   size_time_entry *e;
   bool found = false;
   int i;
-  predicate nonconst_pred;
+  ipa_predicate nonconst_pred;
   vec<size_time_entry> *table = call ? &call_size_time_table : &size_time_table;
 
   if (exec_pred == false)
@@ -270,7 +271,7 @@ redirect_to_unreachable (struct cgraph_edge *e)
 /* Set predicate for edge E.  */
 
 static void
-edge_set_predicate (struct cgraph_edge *e, predicate *predicate)
+edge_set_predicate (struct cgraph_edge *e, ipa_predicate *predicate)
 {
   /* If the edge is determined to be never executed, redirect it
      to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
@@ -300,7 +301,7 @@ edge_set_predicate (struct cgraph_edge *e, predicate *predicate)
 /* Set predicate for hint *P.  */
 
 static void
-set_hint_predicate (predicate **p, predicate new_predicate)
+set_hint_predicate (ipa_predicate **p, ipa_predicate new_predicate)
 {
   if (new_predicate == false || new_predicate == true)
     {
@@ -323,7 +324,7 @@ set_hint_predicate (predicate **p, predicate new_predicate)
 
 static void
 add_freqcounting_predicate (vec<ipa_freqcounting_predicate, va_gc> **v,
-			    const predicate &new_predicate, sreal add_freq,
+			    const ipa_predicate &new_predicate, sreal add_freq,
 			    unsigned max_num_predicates)
 {
   if (new_predicate == false || new_predicate == true)
@@ -371,8 +372,8 @@ evaluate_conditions_for_known_args (struct cgraph_node *node,
 				    clause_t *ret_clause,
 				    clause_t *ret_nonspec_clause)
 {
-  clause_t clause = inline_p ? 0 : 1 << predicate::not_inlined_condition;
-  clause_t nonspec_clause = 1 << predicate::not_inlined_condition;
+  clause_t clause = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
+  clause_t nonspec_clause = 1 << ipa_predicate::not_inlined_condition;
   class ipa_fn_summary *info = ipa_fn_summaries->get (node);
   int i;
   struct condition *c;
@@ -395,7 +396,7 @@ evaluate_conditions_for_known_args (struct cgraph_node *node,
 
       if (c->agg_contents)
 	{
-	  if (c->code == predicate::changed
+	  if (c->code == ipa_predicate::changed
 	      && !c->by_ref
 	      && (avals->safe_sval_at(c->operand_num) == error_mark_node))
 	    continue;
@@ -412,27 +413,28 @@ evaluate_conditions_for_known_args (struct cgraph_node *node,
       else
 	{
 	  val = avals->safe_sval_at (c->operand_num);
-	  if (val && val == error_mark_node && c->code != predicate::changed)
+	  if (val && val == error_mark_node
+	      && c->code != ipa_predicate::changed)
 	    val = NULL_TREE;
 	}
 
       if (!val
-	  && (c->code == predicate::changed
-	      || c->code == predicate::is_not_constant))
+	  && (c->code == ipa_predicate::changed
+	      || c->code == ipa_predicate::is_not_constant))
 	{
-	  clause |= 1 << (i + predicate::first_dynamic_condition);
-	  nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
+	  clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
+	  nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
 	  continue;
 	}
-      if (c->code == predicate::changed)
+      if (c->code == ipa_predicate::changed)
 	{
-	  nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
+	  nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
 	  continue;
 	}
 
-      if (c->code == predicate::is_not_constant)
+      if (c->code == ipa_predicate::is_not_constant)
 	{
-	  nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
+	  nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
 	  continue;
 	}
 
@@ -471,8 +473,9 @@ evaluate_conditions_for_known_args (struct cgraph_node *node,
 	    continue;
 	  if (res && integer_onep (res))
 	    {
-	      clause |= 1 << (i + predicate::first_dynamic_condition);
-	      nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
+	      clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
+	      nonspec_clause
+		|= 1 << (i + ipa_predicate::first_dynamic_condition);
 	      continue;
 	    }
 	}
@@ -527,8 +530,8 @@ evaluate_conditions_for_known_args (struct cgraph_node *node,
 	    }
 	}
 
-      clause |= 1 << (i + predicate::first_dynamic_condition);
-      nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
+      clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
+      nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
     }
   *ret_clause = clause;
   if (ret_nonspec_clause)
@@ -586,7 +589,7 @@ evaluate_properties_for_edge (struct cgraph_edge *e, bool inline_p,
   class ipa_edge_args *args;
 
   if (clause_ptr)
-    *clause_ptr = inline_p ? 0 : 1 << predicate::not_inlined_condition;
+    *clause_ptr = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
 
   if (ipa_node_params_sum
       && !e->call_stmt_cannot_inline_p
@@ -784,7 +787,7 @@ remap_freqcounting_preds_after_dup (vec<ipa_freqcounting_predicate, va_gc> *v,
   int len = res->length();
   for (int i = len - 1; i >= 0; i--)
     {
-      predicate new_predicate
+      ipa_predicate new_predicate
 	= (*res)[i].predicate->remap_after_duplication (possible_truths);
       /* We do not want to free previous predicate; it is used by node
 	 origin.  */
@@ -822,7 +825,7 @@ ipa_fn_summary_t::duplicate (cgraph_node *src,
       int count = ipa_get_param_count (parms_info);
       int i, j;
       clause_t possible_truths;
-      predicate true_pred = true;
+      ipa_predicate true_pred = true;
       size_time_entry *e;
       int optimized_out_size = 0;
       bool inlined_to_p = false;
@@ -859,8 +862,8 @@ ipa_fn_summary_t::duplicate (cgraph_node *src,
          to be true.  */
       for (i = 0; src_info->size_time_table.iterate (i, &e); i++)
 	{
-	  predicate new_exec_pred;
-	  predicate new_nonconst_pred;
+	  ipa_predicate new_exec_pred;
+	  ipa_predicate new_nonconst_pred;
 	  new_exec_pred = e->exec_predicate.remap_after_duplication
 				 (possible_truths);
 	  new_nonconst_pred = e->nonconst_predicate.remap_after_duplication
@@ -876,7 +879,7 @@ ipa_fn_summary_t::duplicate (cgraph_node *src,
          Also copy constantness arrays.   */
       for (edge = dst->callees; edge; edge = next)
 	{
-	  predicate new_predicate;
+	  ipa_predicate new_predicate;
 	  class ipa_call_summary *es = ipa_call_summaries->get (edge);
 	  next = edge->next_callee;
 
@@ -895,7 +898,7 @@ ipa_fn_summary_t::duplicate (cgraph_node *src,
          Also copy constantness arrays.   */
       for (edge = dst->indirect_calls; edge; edge = next)
 	{
-	  predicate new_predicate;
+	  ipa_predicate new_predicate;
 	  class ipa_call_summary *es = ipa_call_summaries->get (edge);
 	  next = edge->next_callee;
 
@@ -943,13 +946,13 @@ ipa_fn_summary_t::duplicate (cgraph_node *src,
       ipa_freqcounting_predicate *f;
       for (int i = 0; vec_safe_iterate (info->loop_iterations, i, &f); i++)
 	{
-	  predicate p = *f->predicate;
+	  ipa_predicate p = *f->predicate;
 	  f->predicate = NULL;
 	  set_hint_predicate (&f->predicate, p);
 	}
       for (int i = 0; vec_safe_iterate (info->loop_strides, i, &f); i++)
 	{
-	  predicate p = *f->predicate;
+	  ipa_predicate p = *f->predicate;
 	  f->predicate = NULL;
 	  set_hint_predicate (&f->predicate, p);
 	}
@@ -1608,12 +1611,12 @@ set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
 	  if (this_code != ERROR_MARK
 	      && !dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
 	    {
-	      predicate p
+	      ipa_predicate p
 		= add_condition (summary, params_summary, index,
 			       	 param_type, &aggpos,
 				 this_code, gimple_cond_rhs (last), param_ops);
 	      e->aux = edge_predicate_pool.allocate ();
-	      *(predicate *) e->aux = p;
+	      *(ipa_predicate *) e->aux = p;
 	    }
 	}
       vec_free (param_ops);
@@ -1645,11 +1648,11 @@ set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
     add_builtin_constant_p_parm (summary, index);
   FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE)
     {
-      predicate p = add_condition (summary, params_summary, index,
+      ipa_predicate p = add_condition (summary, params_summary, index,
 		     		   param_type, &aggpos,
-				   predicate::is_not_constant, NULL_TREE);
+				   ipa_predicate::is_not_constant, NULL_TREE);
       e->aux = edge_predicate_pool.allocate ();
-      *(predicate *) e->aux = p;
+      *(ipa_predicate *) e->aux = p;
     }
 }
 
@@ -1700,7 +1703,7 @@ set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
   FOR_EACH_EDGE (e, ei, bb->succs)
     {
       e->aux = edge_predicate_pool.allocate ();
-      *(predicate *) e->aux = false;
+      *(ipa_predicate *) e->aux = false;
     }
 
   e = gimple_switch_edge (cfun, last, 0);
@@ -1717,7 +1720,7 @@ set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
       tree cl = gimple_switch_label (last, case_idx);
       tree min = CASE_LOW (cl);
       tree max = CASE_HIGH (cl);
-      predicate p;
+      ipa_predicate p;
 
       e = gimple_switch_edge (cfun, last, case_idx);
 
@@ -1741,15 +1744,15 @@ set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
 		           &aggpos, EQ_EXPR, min, param_ops);
       else
 	{
-	  predicate p1, p2;
+	  ipa_predicate p1, p2;
 	  p1 = add_condition (summary, params_summary, index, param_type,
 			      &aggpos, GE_EXPR, min, param_ops);
 	  p2 = add_condition (summary,  params_summary,index, param_type,
 			      &aggpos, LE_EXPR, max, param_ops);
 	  p = p1 & p2;
 	}
-      *(class predicate *) e->aux
-	= p.or_with (summary->conds, *(class predicate *) e->aux);
+      *(ipa_predicate *) e->aux
+	= p.or_with (summary->conds, *(ipa_predicate *) e->aux);
 
       /* If there are too many disjoint case ranges, predicate for default
 	 case might become too complicated.  So add a limit here.  */
@@ -1795,13 +1798,13 @@ set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
   e = gimple_switch_edge (cfun, last, 0);
   if (bound_count > bound_limit)
     {
-      *(class predicate *) e->aux = true;
+      *(ipa_predicate *) e->aux = true;
       vec_free (param_ops);
       return;
     }
 
-  predicate p_seg = true;
-  predicate p_all = false;
+  ipa_predicate p_seg = true;
+  ipa_predicate p_all = false;
 
   if (vr_type != VR_RANGE)
     {
@@ -1856,8 +1859,8 @@ set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
     }
 
   p_all = p_all.or_with (summary->conds, p_seg);
-  *(class predicate *) e->aux
-    = p_all.or_with (summary->conds, *(class predicate *) e->aux);
+  *(ipa_predicate *) e->aux
+    = p_all.or_with (summary->conds, *(ipa_predicate *) e->aux);
 
   vec_free (param_ops);
 }
@@ -1885,7 +1888,7 @@ compute_bb_predicates (struct ipa_func_body_info *fbi,
   /* Entry block is always executable.  */
   ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
     = edge_predicate_pool.allocate ();
-  *(predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux = true;
+  *(ipa_predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux = true;
 
   /* A simple dataflow propagation of predicates forward in the CFG.
      TODO: work in reverse postorder.  */
@@ -1894,17 +1897,17 @@ compute_bb_predicates (struct ipa_func_body_info *fbi,
       done = true;
       FOR_EACH_BB_FN (bb, my_function)
 	{
-	  predicate p = false;
+	  ipa_predicate p = false;
 	  edge e;
 	  edge_iterator ei;
 	  FOR_EACH_EDGE (e, ei, bb->preds)
 	    {
 	      if (e->src->aux)
 		{
-		  predicate this_bb_predicate
-		    = *(predicate *) e->src->aux;
+		  ipa_predicate this_bb_predicate
+		    = *(ipa_predicate *) e->src->aux;
 		  if (e->aux)
-		    this_bb_predicate &= (*(class predicate *) e->aux);
+		    this_bb_predicate &= (*(ipa_predicate *) e->aux);
 		  p = p.or_with (summary->conds, this_bb_predicate);
 		  if (p == true)
 		    break;
@@ -1918,18 +1921,18 @@ compute_bb_predicates (struct ipa_func_body_info *fbi,
 		{
 		  done = false;
 		  bb->aux = edge_predicate_pool.allocate ();
-		  *((predicate *) bb->aux) = p;
+		  *((ipa_predicate *) bb->aux) = p;
 		}
-	      else if (p != *(predicate *) bb->aux)
+	      else if (p != *(ipa_predicate *) bb->aux)
 		{
 		  /* This OR operation is needed to ensure monotonous data flow
 		     in the case we hit the limit on number of clauses and the
 		     and/or operations above give approximate answers.  */
-		  p = p.or_with (summary->conds, *(predicate *)bb->aux);
-	          if (p != *(predicate *) bb->aux)
+		  p = p.or_with (summary->conds, *(ipa_predicate *)bb->aux);
+		  if (p != *(ipa_predicate *)bb->aux)
 		    {
 		      done = false;
-		      *((predicate *) bb->aux) = p;
+		      *((ipa_predicate *)bb->aux) = p;
 		    }
 		}
 
@@ -1949,15 +1952,16 @@ compute_bb_predicates (struct ipa_func_body_info *fbi,
 		{
 		  done = false;
 		  pdom_bb->aux = edge_predicate_pool.allocate ();
-		  *((predicate *) pdom_bb->aux) = p;
+		  *((ipa_predicate *)pdom_bb->aux) = p;
 		}
-	      else if (p != *(predicate *) pdom_bb->aux)
+	      else if (p != *(ipa_predicate *)pdom_bb->aux)
 		{
-		  p = p.or_with (summary->conds, *(predicate *)pdom_bb->aux);
-		  if (p != *(predicate *) pdom_bb->aux)
+		  p = p.or_with (summary->conds,
+				 *(ipa_predicate *)pdom_bb->aux);
+		  if (p != *(ipa_predicate *)pdom_bb->aux)
 		    {
 		      done = false;
-		      *((predicate *) pdom_bb->aux) = p;
+		      *((ipa_predicate *)pdom_bb->aux) = p;
 		    }
 		}
 	    }
@@ -1969,12 +1973,12 @@ compute_bb_predicates (struct ipa_func_body_info *fbi,
 /* Return predicate specifying when the STMT might have result that is not
    a compile time constant.  */
 
-static predicate
+static ipa_predicate
 will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
 				    class ipa_fn_summary *summary,
 				    class ipa_node_params *params_summary,
 				    tree expr,
-				    vec<predicate> nonconstant_names)
+				    vec<ipa_predicate> nonconstant_names)
 {
   tree parm;
   int index;
@@ -1985,14 +1989,14 @@ will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
   parm = unmodified_parm (fbi, NULL, expr, NULL);
   if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
     return add_condition (summary, params_summary, index, TREE_TYPE (parm), NULL,
-			  predicate::changed, NULL_TREE);
+			  ipa_predicate::changed, NULL_TREE);
   if (is_gimple_min_invariant (expr))
     return false;
   if (TREE_CODE (expr) == SSA_NAME)
     return nonconstant_names[SSA_NAME_VERSION (expr)];
   if (BINARY_CLASS_P (expr) || COMPARISON_CLASS_P (expr))
     {
-      predicate p1
+      ipa_predicate p1
 	= will_be_nonconstant_expr_predicate (fbi, summary,
 					      params_summary,
 					      TREE_OPERAND (expr, 0),
@@ -2000,7 +2004,7 @@ will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
       if (p1 == true)
 	return p1;
 
-      predicate p2
+      ipa_predicate p2
 	= will_be_nonconstant_expr_predicate (fbi, summary,
 					      params_summary,
 					      TREE_OPERAND (expr, 1),
@@ -2009,7 +2013,7 @@ will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
     }
   else if (TREE_CODE (expr) == COND_EXPR)
     {
-      predicate p1
+      ipa_predicate p1
 	= will_be_nonconstant_expr_predicate (fbi, summary,
 					      params_summary,
 					      TREE_OPERAND (expr, 0),
@@ -2017,7 +2021,7 @@ will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
       if (p1 == true)
 	return p1;
 
-      predicate p2
+      ipa_predicate p2
 	= will_be_nonconstant_expr_predicate (fbi, summary,
 					      params_summary,
 					      TREE_OPERAND (expr, 1),
@@ -2045,18 +2049,18 @@ will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
 /* Return predicate specifying when the STMT might have result that is not
    a compile time constant.  */
 
-static predicate
+static ipa_predicate
 will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
 			       class ipa_fn_summary *summary,
 			       class ipa_node_params *params_summary,
 			       gimple *stmt,
-			       vec<predicate> nonconstant_names)
+			       vec<ipa_predicate> nonconstant_names)
 {
-  predicate p = true;
+  ipa_predicate p = true;
   ssa_op_iter iter;
   tree use;
   tree param_type = NULL_TREE;
-  predicate op_non_const;
+  ipa_predicate op_non_const;
   bool is_load;
   int base_index;
   struct agg_position_info aggpos;
@@ -2108,7 +2112,7 @@ will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
     op_non_const =
       add_condition (summary, params_summary,
 		     base_index, param_type, &aggpos,
-		     predicate::changed, NULL_TREE);
+		     ipa_predicate::changed, NULL_TREE);
   else
     op_non_const = false;
   FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
@@ -2121,7 +2125,7 @@ will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
 	  if (index != base_index)
 	    p = add_condition (summary, params_summary, index,
 			       TREE_TYPE (parm), NULL,
-			       predicate::changed, NULL_TREE);
+			       ipa_predicate::changed, NULL_TREE);
 	  else
 	    continue;
 	}
@@ -2314,8 +2318,8 @@ phi_result_unknown_predicate (ipa_func_body_info *fbi,
 			      ipa_fn_summary *summary,
 			      class ipa_node_params *params_summary,
 			      basic_block bb,
-			      predicate *p,
-			      vec<predicate> nonconstant_names)
+			      ipa_predicate *p,
+			      vec<ipa_predicate> nonconstant_names)
 {
   edge e;
   edge_iterator ei;
@@ -2373,8 +2377,8 @@ phi_result_unknown_predicate (ipa_func_body_info *fbi,
 
 static void
 predicate_for_phi_result (class ipa_fn_summary *summary, gphi *phi,
-			  predicate *p,
-			  vec<predicate> nonconstant_names)
+			  ipa_predicate *p,
+			  vec<ipa_predicate> nonconstant_names)
 {
   unsigned i;
 
@@ -2569,7 +2573,20 @@ points_to_local_or_readonly_memory_p (tree t)
   if (integer_zerop (t))
     return flag_delete_null_pointer_checks;
   if (TREE_CODE (t) == SSA_NAME)
-    return !ptr_deref_may_alias_global_p (t);
+    {
+      /* For IPA passes we can consinder accesses to return slot local
+	 even if it is not local in the sense that memory is dead by
+	 the end of founction.
+	 The outer function will see a store in the call assignment
+	 and thus this will do right thing for all uses of this
+	 function in the current IPA passes (modref, pure/const discovery
+	 and inlining heuristics).  */
+      if (DECL_RESULT (current_function_decl)
+	  && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
+	  && t == ssa_default_def (cfun, DECL_RESULT (current_function_decl)))
+	return true;
+      return !ptr_deref_may_alias_global_p (t);
+    }
   if (TREE_CODE (t) == ADDR_EXPR)
     return refs_local_or_readonly_memory_p (TREE_OPERAND (t, 0));
   return false;
@@ -2593,9 +2610,9 @@ analyze_function_body (struct cgraph_node *node, bool early)
   class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
   ipa_node_params *params_summary
     = early ? NULL : ipa_node_params_sum->get (node);
-  predicate bb_predicate;
+  ipa_predicate bb_predicate;
   struct ipa_func_body_info fbi;
-  vec<predicate> nonconstant_names = vNULL;
+  vec<ipa_predicate> nonconstant_names = vNULL;
   int nblocks, n;
   int *order;
   gimple *fix_builtin_expect_stmt;
@@ -2650,7 +2667,7 @@ analyze_function_body (struct cgraph_node *node, bool early)
   bb_predicate = true;
   info->account_size_time (0, 0, bb_predicate, bb_predicate);
 
-  bb_predicate = predicate::not_inlined ();
+  bb_predicate = ipa_predicate::not_inlined ();
   info->account_size_time (opt_for_fn (node->decl,
 				param_uninlined_function_insns)
 			   * ipa_fn_summary::size_scale,
@@ -2681,7 +2698,7 @@ analyze_function_body (struct cgraph_node *node, bool early)
       if (fbi.info)
 	{
 	  if (bb->aux)
-	    bb_predicate = *(predicate *) bb->aux;
+	    bb_predicate = *(ipa_predicate *)bb->aux;
 	  else
 	    bb_predicate = false;
 	}
@@ -2696,7 +2713,7 @@ analyze_function_body (struct cgraph_node *node, bool early)
 
       if (fbi.info && nonconstant_names.exists ())
 	{
-	  predicate phi_predicate;
+	  ipa_predicate phi_predicate;
 	  bool first_phi = true;
 
 	  for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
@@ -2729,7 +2746,7 @@ analyze_function_body (struct cgraph_node *node, bool early)
 	  int this_size = estimate_num_insns (stmt, &eni_size_weights);
 	  int this_time = estimate_num_insns (stmt, &eni_time_weights);
 	  int prob;
-	  predicate will_be_nonconstant;
+	  ipa_predicate will_be_nonconstant;
 
           /* This relation stmt should be folded after we remove
              __builtin_expect call. Adjust the cost here.  */
@@ -2762,7 +2779,7 @@ analyze_function_body (struct cgraph_node *node, bool early)
 		  && gimple_call_lhs (stmt)
 		  && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
 		{
-		  predicate false_p = false;
+		  ipa_predicate false_p = false;
 		  nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))]
 		    = false_p;
 		}
@@ -2839,7 +2856,7 @@ analyze_function_body (struct cgraph_node *node, bool early)
 	      if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
 		fprintf (dump_file, "\t\tWill be eliminated by inlining\n");
 
-	      class predicate p = bb_predicate & will_be_nonconstant;
+	      ipa_predicate p = bb_predicate & will_be_nonconstant;
 
 	      /* We can ignore statement when we proved it is never going
 		 to happen, but we cannot do that for call statements
@@ -2858,7 +2875,8 @@ analyze_function_body (struct cgraph_node *node, bool early)
 		{
 		  if (prob)
 		    {
-		      predicate ip = bb_predicate & predicate::not_inlined ();
+		      ipa_predicate ip
+			= bb_predicate & ipa_predicate::not_inlined ();
 		      info->account_size_time (this_size * prob,
 					       (final_time * prob) / 2, ip,
 					       p);
@@ -2888,7 +2906,7 @@ analyze_function_body (struct cgraph_node *node, bool early)
 		     || TREE_CODE (op) == ARRAY_RANGE_REF)
 		    && TREE_CODE (TREE_OPERAND (op, 1)) == SSA_NAME)
 		  {
-		    predicate p = bb_predicate;
+		    ipa_predicate p = bb_predicate;
 		    if (fbi.info)
 		      p = p & will_be_nonconstant_expr_predicate
 				 (&fbi, info, params_summary,
@@ -2925,7 +2943,7 @@ analyze_function_body (struct cgraph_node *node, bool early)
       scev_initialize ();
       for (auto loop : loops_list (cfun, 0))
 	{
-	  predicate loop_iterations = true;
+	  ipa_predicate loop_iterations = true;
 	  sreal header_freq;
 	  edge ex;
 	  unsigned int j;
@@ -2936,13 +2954,13 @@ analyze_function_body (struct cgraph_node *node, bool early)
 	  profile_count phdr_count = loop_preheader_edge (loop)->count ();
 	  sreal phdr_freq = phdr_count.to_sreal_scale (entry_count);
 
-	  bb_predicate = *(predicate *) loop->header->aux;
+	  bb_predicate = *(ipa_predicate *)loop->header->aux;
 	  auto_vec<edge> exits = get_loop_exit_edges (loop);
 	  FOR_EACH_VEC_ELT (exits, j, ex)
 	    if (number_of_iterations_exit (loop, ex, &niter_desc, false)
 		&& !is_gimple_min_invariant (niter_desc.niter))
 	    {
-	      predicate will_be_nonconstant
+	      ipa_predicate will_be_nonconstant
 		= will_be_nonconstant_expr_predicate (&fbi, info,
 						      params_summary,
 						      niter_desc.niter,
@@ -2963,7 +2981,7 @@ analyze_function_body (struct cgraph_node *node, bool early)
       for (loop = loops_for_fn (cfun)->tree_root->inner;
 	   loop != NULL; loop = loop->next)
 	{
-	  predicate loop_stride = true;
+	  ipa_predicate loop_stride = true;
 	  basic_block *body = get_loop_body (loop);
 	  profile_count phdr_count = loop_preheader_edge (loop)->count ();
 	  sreal phdr_freq = phdr_count.to_sreal_scale (entry_count);
@@ -2973,7 +2991,7 @@ analyze_function_body (struct cgraph_node *node, bool early)
 	      if (!body[i]->aux)
 		continue;
 
-	      bb_predicate = *(predicate *) body[i]->aux;
+	      bb_predicate = *(ipa_predicate *)body[i]->aux;
 	      for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi);
 		   gsi_next (&gsi))
 		{
@@ -2993,7 +3011,7 @@ analyze_function_body (struct cgraph_node *node, bool early)
 		      || is_gimple_min_invariant (iv.step))
 		    continue;
 
-		  predicate will_be_nonconstant
+		  ipa_predicate will_be_nonconstant
 		    = will_be_nonconstant_expr_predicate (&fbi, info,
 				    			  params_summary,
 				   			  iv.step,
@@ -3017,12 +3035,12 @@ analyze_function_body (struct cgraph_node *node, bool early)
       edge_iterator ei;
 
       if (bb->aux)
-	edge_predicate_pool.remove ((predicate *)bb->aux);
+	edge_predicate_pool.remove ((ipa_predicate *)bb->aux);
       bb->aux = NULL;
       FOR_EACH_EDGE (e, ei, bb->succs)
 	{
 	  if (e->aux)
-	    edge_predicate_pool.remove ((predicate *) e->aux);
+	    edge_predicate_pool.remove ((ipa_predicate *)e->aux);
 	  e->aux = NULL;
 	}
     }
@@ -3078,7 +3096,7 @@ compute_fn_summary (struct cgraph_node *node, bool early)
   if (node->thunk)
     {
       ipa_call_summary *es = ipa_call_summaries->get_create (node->callees);
-      predicate t = true;
+      ipa_predicate t = true;
 
       node->can_change_signature = false;
       es->call_stmt_size = eni_size_weights.call_cost;
@@ -3088,7 +3106,7 @@ compute_fn_summary (struct cgraph_node *node, bool early)
 				 param_uninlined_function_thunk_insns),
 			       opt_for_fn (node->decl,
 				 param_uninlined_function_thunk_time), t, t);
-      t = predicate::not_inlined ();
+      t = ipa_predicate::not_inlined ();
       info->account_size_time (2 * ipa_fn_summary::size_scale, 0, t, t);
       ipa_update_overall_fn_summary (node);
       size_info->self_size = size_info->size;
@@ -3117,10 +3135,38 @@ compute_fn_summary (struct cgraph_node *node, bool early)
        else
 	 info->inlinable = tree_inlinable_function_p (node->decl);
 
-       /* Type attributes can use parameter indices to describe them.  */
-       if (TYPE_ATTRIBUTES (TREE_TYPE (node->decl))
-	   /* Likewise for #pragma omp declare simd functions or functions
-	      with simd attribute.  */
+       bool no_signature = false;
+       /* Type attributes can use parameter indices to describe them.
+	  Special case fn spec since we can safely preserve them in
+	  modref summaries.  */
+       for (tree list = TYPE_ATTRIBUTES (TREE_TYPE (node->decl));
+	    list && !no_signature; list = TREE_CHAIN (list))
+	if (!ipa_param_adjustments::type_attribute_allowed_p
+			(get_attribute_name (list)))
+	   {
+	     if (dump_file)
+		{
+		  fprintf (dump_file, "No signature change:"
+			   " function type has unhandled attribute %s.\n",
+			   IDENTIFIER_POINTER (get_attribute_name (list)));
+		}
+	     no_signature = true;
+	   }
+       for (tree parm = DECL_ARGUMENTS (node->decl);
+	    parm && !no_signature; parm = DECL_CHAIN (parm))
+	 if (variably_modified_type_p (TREE_TYPE (parm), node->decl))
+	   {
+	     if (dump_file)
+		{
+		  fprintf (dump_file, "No signature change:"
+			   " has parameter with variably modified type.\n");
+		}
+	     no_signature = true;
+	   }
+
+       /* Likewise for #pragma omp declare simd functions or functions
+	  with simd attribute.  */
+       if (no_signature
 	   || lookup_attribute ("omp declare simd",
 				DECL_ATTRIBUTES (node->decl)))
 	 node->can_change_signature = false;
@@ -3321,7 +3367,7 @@ summarize_calls_size_and_time (struct cgraph_node *node,
 
       estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
 
-      struct predicate pred = true;
+      ipa_predicate pred = true;
       class ipa_call_summary *es = ipa_call_summaries->get (e);
 
       if (es->predicate)
@@ -3334,7 +3380,7 @@ summarize_calls_size_and_time (struct cgraph_node *node,
       sreal time = 0;
 
       estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
-      struct predicate pred = true;
+      ipa_predicate pred = true;
       class ipa_call_summary *es = ipa_call_summaries->get (e);
 
       if (es->predicate)
@@ -3400,7 +3446,7 @@ estimate_calls_size_and_time (struct cgraph_node *node, int *size,
       /* Build summary if it is absent.  */
       if (!sum->call_size_time_table.length ())
 	{
-	  predicate true_pred = true;
+	  ipa_predicate true_pred = true;
 	  sum->account_size_time (0, 0, true_pred, true_pred, true);
 	  summarize_calls_size_and_time (node, sum);
 	}
@@ -3698,8 +3744,8 @@ ipa_call_context::estimate_size_and_time (ipa_call_estimates *estimates,
       fprintf (dump_file, "   Estimating body: %s\n"
 	       "   Known to be false: ", m_node->dump_name ());
 
-      for (i = predicate::not_inlined_condition;
-	   i < (predicate::first_dynamic_condition
+      for (i = ipa_predicate::not_inlined_condition;
+	   i < (ipa_predicate::first_dynamic_condition
 		+ (int) vec_safe_length (info->conds)); i++)
 	if (!(m_possible_truths & (1 << i)))
 	  {
@@ -3970,12 +4016,12 @@ remap_edge_summaries (struct cgraph_edge *inlined_edge,
 		      const vec<int> &operand_map,
 		      const vec<HOST_WIDE_INT> &offset_map,
 		      clause_t possible_truths,
-		      predicate *toplev_predicate)
+		      ipa_predicate *toplev_predicate)
 {
   struct cgraph_edge *e, *next;
   for (e = node->callees; e; e = next)
     {
-      predicate p;
+      ipa_predicate p;
       next = e->next_callee;
 
       if (e->inline_failed)
@@ -4004,7 +4050,7 @@ remap_edge_summaries (struct cgraph_edge *inlined_edge,
   for (e = node->indirect_calls; e; e = next)
     {
       class ipa_call_summary *es = ipa_call_summaries->get (e);
-      predicate p;
+      ipa_predicate p;
       next = e->next_callee;
 
       remap_edge_params (inlined_edge, e);
@@ -4031,13 +4077,13 @@ remap_freqcounting_predicate (class ipa_fn_summary *info,
 			      const vec<int> &operand_map,
 			      const vec<HOST_WIDE_INT> &offset_map,
 			      clause_t possible_truths,
-			      predicate *toplev_predicate)
+			      ipa_predicate *toplev_predicate)
 
 {
   ipa_freqcounting_predicate *fcp;
   for (int i = 0; vec_safe_iterate (v, i, &fcp); i++)
     {
-      predicate p
+      ipa_predicate p
 	= fcp->predicate->remap_after_inlining (info, params_summary,
 						callee_info, operand_map,
 						offset_map, possible_truths,
@@ -4061,7 +4107,7 @@ ipa_merge_fn_summary_after_inlining (struct cgraph_edge *edge)
   auto_vec<int, 8> operand_map;
   auto_vec<HOST_WIDE_INT, 8> offset_map;
   int i;
-  predicate toplev_predicate;
+  ipa_predicate toplev_predicate;
   class ipa_call_summary *es = ipa_call_summaries->get (edge);
   ipa_node_params *params_summary = (ipa_node_params_sum
 				     ? ipa_node_params_sum->get (to) : NULL);
@@ -4125,13 +4171,13 @@ ipa_merge_fn_summary_after_inlining (struct cgraph_edge *edge)
   sreal freq = edge->sreal_frequency ();
   for (i = 0; callee_info->size_time_table.iterate (i, &e); i++)
     {
-      predicate p;
+      ipa_predicate p;
       p = e->exec_predicate.remap_after_inlining
 			     (info, params_summary,
 			      callee_info, operand_map,
 			      offset_map, clause,
 			      toplev_predicate);
-      predicate nonconstp;
+      ipa_predicate nonconstp;
       nonconstp = e->nonconst_predicate.remap_after_inlining
 				     (info, params_summary,
 				      callee_info, operand_map,
@@ -4216,7 +4262,7 @@ ipa_update_overall_fn_summary (struct cgraph_node *node, bool reset)
   if (node->callees || node->indirect_calls)
     estimate_calls_size_and_time (node, &size_info->size, &info->min_size,
 				  &info->time, NULL,
-				  ~(clause_t) (1 << predicate::false_condition),
+				  ~(clause_t) (1 << ipa_predicate::false_condition),
 				  NULL);
   size_info->size = RDIV (size_info->size, ipa_fn_summary::size_scale);
   info->min_size = RDIV (info->min_size, ipa_fn_summary::size_scale);
@@ -4304,7 +4350,7 @@ read_ipa_call_summary (class lto_input_block *ib, struct cgraph_edge *e,
 {
   class ipa_call_summary *es = prevails
 				? ipa_call_summaries->get_create (e) : NULL;
-  predicate p;
+  ipa_predicate p;
   int length, i;
 
   int size = streamer_read_uhwi (ib);
@@ -4385,7 +4431,7 @@ inline_read_section (struct lto_file_decl_data *file_data, const char *data,
       lto_symtab_encoder_t encoder;
       struct bitpack_d bp;
       struct cgraph_edge *e;
-      predicate p;
+      ipa_predicate p;
 
       index = streamer_read_uhwi (&ib);
       encoder = file_data->symtab_node_encoder;
diff --git a/gcc/ipa-fnsummary.h b/gcc/ipa-fnsummary.h
index 78399b0..fdfbd66 100644
--- a/gcc/ipa-fnsummary.h
+++ b/gcc/ipa-fnsummary.h
@@ -76,11 +76,11 @@ class size_time_entry
 {
 public:
   /* Predicate for code to be executed.  */
-  predicate exec_predicate;
+  ipa_predicate exec_predicate;
   /* Predicate for value to be constant and optimized out in a specialized copy.
      When deciding on specialization this makes it possible to see how much
      the executed code paths will simplify.  */
-  predicate nonconst_predicate;
+  ipa_predicate nonconst_predicate;
   int size;
   sreal time;
 };
@@ -114,7 +114,7 @@ struct GTY(()) ipa_freqcounting_predicate
   /* The described event happens with this frequency... */
   sreal freq;
   /* ...when this predicate evaluates to false. */
-  class predicate * GTY((skip)) predicate;
+  ipa_predicate * GTY((skip)) predicate;
 };
 
 /* Function inlining information.  */
@@ -203,7 +203,8 @@ public:
   int scc_no;
 
   /* Record time and size under given predicates.  */
-  void account_size_time (int, sreal, const predicate &, const predicate &,
+  void account_size_time (int, sreal, const ipa_predicate &,
+			  const ipa_predicate &,
 		  	  bool call = false);
 
   /* We keep values scaled up, so fractional sizes can be accounted.  */
@@ -286,7 +287,7 @@ public:
   /* Default destructor.  */
   ~ipa_call_summary ();
 
-  class predicate *predicate;
+  ipa_predicate *predicate;
   /* Vector indexed by parameters.  */
   vec<inline_param_summary> param;
   /* Estimated size and time of the call statement.  */
diff --git a/gcc/ipa-modref-tree.c b/gcc/ipa-modref-tree.c
index 8d147a1..bbe23a5 100644
--- a/gcc/ipa-modref-tree.c
+++ b/gcc/ipa-modref-tree.c
@@ -25,6 +25,797 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree.h"
 #include "ipa-modref-tree.h"
 #include "selftest.h"
+#include "tree-ssa-alias.h"
+#include "gimple.h"
+
+/* Return true if both accesses are the same.  */
+bool
+modref_access_node::operator == (modref_access_node &a) const
+{
+  if (parm_index != a.parm_index)
+    return false;
+  if (parm_index != MODREF_UNKNOWN_PARM)
+    {
+      if (parm_offset_known != a.parm_offset_known)
+	return false;
+      if (parm_offset_known
+	  && !known_eq (parm_offset, a.parm_offset))
+	return false;
+    }
+  if (range_info_useful_p () != a.range_info_useful_p ())
+    return false;
+  if (range_info_useful_p ()
+      && (!known_eq (a.offset, offset)
+	  || !known_eq (a.size, size)
+	  || !known_eq (a.max_size, max_size)))
+    return false;
+  return true;
+}
+
+/* Return true A is a subaccess.  */
+bool
+modref_access_node::contains (const modref_access_node &a) const
+{
+  poly_int64 aoffset_adj = 0;
+  if (parm_index != MODREF_UNKNOWN_PARM)
+    {
+      if (parm_index != a.parm_index)
+	return false;
+      if (parm_offset_known)
+	{
+	   if (!a.parm_offset_known)
+	     return false;
+	   /* Accesses are never below parm_offset, so look
+	      for smaller offset.
+	      If access ranges are known still allow merging
+	      when bit offsets comparsion passes.  */
+	   if (!known_le (parm_offset, a.parm_offset)
+	       && !range_info_useful_p ())
+	     return false;
+	   /* We allow negative aoffset_adj here in case
+	      there is an useful range.  This is because adding
+	      a.offset may result in non-ngative offset again.
+	      Ubsan fails on val << LOG_BITS_PER_UNIT where val
+	      is negative.  */
+	   aoffset_adj = (a.parm_offset - parm_offset)
+			 * BITS_PER_UNIT;
+	}
+    }
+  if (range_info_useful_p ())
+    {
+      if (!a.range_info_useful_p ())
+	return false;
+      /* Sizes of stores are used to check that object is big enough
+	 to fit the store, so smaller or unknown sotre is more general
+	 than large store.  */
+      if (known_size_p (size)
+	  && (!known_size_p (a.size)
+	      || !known_le (size, a.size)))
+	return false;
+      if (known_size_p (max_size))
+	return known_subrange_p (a.offset + aoffset_adj,
+				 a.max_size, offset, max_size);
+      else
+	return known_le (offset, a.offset + aoffset_adj);
+    }
+  return true;
+}
+
+/* Update access range to new parameters.
+   If RECORD_ADJUSTMENTS is true, record number of changes in the access
+   and if threshold is exceeded start dropping precision
+   so only constantly many updates are possible.  This makes dataflow
+   to converge.  */
+void
+modref_access_node::update (poly_int64 parm_offset1,
+			    poly_int64 offset1, poly_int64 size1,
+			    poly_int64 max_size1, bool record_adjustments)
+{
+  if (known_eq (parm_offset, parm_offset1)
+      && known_eq (offset, offset1)
+      && known_eq (size, size1)
+      && known_eq (max_size, max_size1))
+    return;
+  if (!record_adjustments
+      || (++adjustments) < param_modref_max_adjustments)
+    {
+      parm_offset = parm_offset1;
+      offset = offset1;
+      size = size1;
+      max_size = max_size1;
+    }
+  else
+    {
+      if (dump_file)
+	fprintf (dump_file,
+		 "--param param=modref-max-adjustments limit reached:");
+      if (!known_eq (parm_offset, parm_offset1))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, " parm_offset cleared");
+	  parm_offset_known = false;
+	}
+      if (!known_eq (size, size1))
+	{
+	  size = -1;
+	  if (dump_file)
+	    fprintf (dump_file, " size cleared");
+	}
+      if (!known_eq (max_size, max_size1))
+	{
+	  max_size = -1;
+	  if (dump_file)
+	    fprintf (dump_file, " max_size cleared");
+	}
+      if (!known_eq (offset, offset1))
+	{
+	  offset = 0;
+	  if (dump_file)
+	    fprintf (dump_file, " offset cleared");
+	}
+      if (dump_file)
+	fprintf (dump_file, "\n");
+    }
+}
+
+/* Merge in access A if it is possible to do without losing
+   precision.  Return true if successful.
+   If RECORD_ADJUSTMENTs is true, remember how many interval
+   was prolonged and punt when there are too many.  */
+bool
+modref_access_node::merge (const modref_access_node &a,
+			   bool record_adjustments)
+{
+  poly_int64 offset1 = 0;
+  poly_int64 aoffset1 = 0;
+  poly_int64 new_parm_offset = 0;
+
+  /* We assume that containment was tested earlier.  */
+  gcc_checking_assert (!contains (a) && !a.contains (*this));
+  if (parm_index != MODREF_UNKNOWN_PARM)
+    {
+      if (parm_index != a.parm_index)
+	return false;
+      if (parm_offset_known)
+	{
+	  if (!a.parm_offset_known)
+	    return false;
+	  if (!combined_offsets (a, &new_parm_offset, &offset1, &aoffset1))
+	    return false;
+	}
+    }
+  /* See if we can merge ranges.  */
+  if (range_info_useful_p ())
+    {
+      /* In this case we have containment that should be
+	 handled earlier.  */
+      gcc_checking_assert (a.range_info_useful_p ());
+
+      /* If a.size is less specified than size, merge only
+	 if intervals are otherwise equivalent.  */
+      if (known_size_p (size)
+	  && (!known_size_p (a.size) || known_lt (a.size, size)))
+	{
+	  if (((known_size_p (max_size) || known_size_p (a.max_size))
+	       && !known_eq (max_size, a.max_size))
+	       || !known_eq (offset1, aoffset1))
+	    return false;
+	  update (new_parm_offset, offset1, a.size, max_size,
+		  record_adjustments);
+	  return true;
+	}
+      /* If sizes are same, we can extend the interval.  */
+      if ((known_size_p (size) || known_size_p (a.size))
+	  && !known_eq (size, a.size))
+	return false;
+      if (known_le (offset1, aoffset1))
+	{
+	  if (!known_size_p (max_size)
+	      || known_ge (offset1 + max_size, aoffset1))
+	    {
+	      update2 (new_parm_offset, offset1, size, max_size,
+		       aoffset1, a.size, a.max_size,
+		       record_adjustments);
+	      return true;
+	    }
+	}
+      else if (known_le (aoffset1, offset1))
+	{
+	  if (!known_size_p (a.max_size)
+	      || known_ge (aoffset1 + a.max_size, offset1))
+	    {
+	      update2 (new_parm_offset, offset1, size, max_size,
+		       aoffset1, a.size, a.max_size,
+		       record_adjustments);
+	      return true;
+	    }
+	}
+      return false;
+    }
+  update (new_parm_offset, offset1,
+	  size, max_size, record_adjustments);
+  return true;
+}
+
+/* Return true if A1 and B1 can be merged with lower information
+   less than A2 and B2.
+   Assume that no containment or lossless merging is possible.  */
+bool
+modref_access_node::closer_pair_p (const modref_access_node &a1,
+				   const modref_access_node &b1,
+				   const modref_access_node &a2,
+				   const modref_access_node &b2)
+{
+  /* Merging different parm indexes comes to complete loss
+     of range info.  */
+  if (a1.parm_index != b1.parm_index)
+    return false;
+  if (a2.parm_index != b2.parm_index)
+    return true;
+  /* If parm is known and parm indexes are the same we should
+     already have containment.  */
+  gcc_checking_assert (a1.parm_offset_known && b1.parm_offset_known);
+  gcc_checking_assert (a2.parm_offset_known && b2.parm_offset_known);
+
+  /* First normalize offsets for parm offsets.  */
+  poly_int64 new_parm_offset, offseta1, offsetb1, offseta2, offsetb2;
+  if (!a1.combined_offsets (b1, &new_parm_offset, &offseta1, &offsetb1)
+      || !a2.combined_offsets (b2, &new_parm_offset, &offseta2, &offsetb2))
+    gcc_unreachable ();
+
+
+  /* Now compute distnace of the intervals.  */
+  poly_int64 dist1, dist2;
+  if (known_le (offseta1, offsetb1))
+    {
+      if (!known_size_p (a1.max_size))
+	dist1 = 0;
+      else
+	dist1 = offsetb1 - offseta1 - a1.max_size;
+    }
+  else
+    {
+      if (!known_size_p (b1.max_size))
+	dist1 = 0;
+      else
+	dist1 = offseta1 - offsetb1 - b1.max_size;
+    }
+  if (known_le (offseta2, offsetb2))
+    {
+      if (!known_size_p (a2.max_size))
+	dist2 = 0;
+      else
+	dist2 = offsetb2 - offseta2 - a2.max_size;
+    }
+  else
+    {
+      if (!known_size_p (b2.max_size))
+	dist2 = 0;
+      else
+	dist2 = offseta2 - offsetb2 - b2.max_size;
+    }
+  /* It may happen that intervals overlap in case size
+     is different.  Preffer the overlap to non-overlap.  */
+  if (known_lt (dist1, 0) && known_ge (dist2, 0))
+    return true;
+  if (known_lt (dist2, 0) && known_ge (dist1, 0))
+    return false;
+  if (known_lt (dist1, 0))
+    /* If both overlaps minimize overlap.  */
+    return known_le (dist2, dist1);
+  else
+    /* If both are disjoint look for smaller distance.  */
+    return known_le (dist1, dist2);
+}
+
+/* Merge in access A while losing precision.  */
+void
+modref_access_node::forced_merge (const modref_access_node &a,
+				  bool record_adjustments)
+{
+  if (parm_index != a.parm_index)
+    {
+      gcc_checking_assert (parm_index != MODREF_UNKNOWN_PARM);
+      parm_index = MODREF_UNKNOWN_PARM;
+      return;
+    }
+
+  /* We assume that containment and lossless merging
+     was tested earlier.  */
+  gcc_checking_assert (!contains (a) && !a.contains (*this)
+		       && !merge (a, record_adjustments));
+  gcc_checking_assert (parm_offset_known && a.parm_offset_known);
+
+  poly_int64 new_parm_offset, offset1, aoffset1;
+  if (!combined_offsets (a, &new_parm_offset, &offset1, &aoffset1))
+    {
+      parm_offset_known = false;
+      return;
+    }
+  gcc_checking_assert (range_info_useful_p ()
+		       && a.range_info_useful_p ());
+  if (record_adjustments)
+    adjustments += a.adjustments;
+  update2 (new_parm_offset,
+	   offset1, size, max_size,
+	   aoffset1, a.size, a.max_size,
+	   record_adjustments);
+}
+
+/* Merge two ranges both starting at parm_offset1 and update THIS
+   with result.  */
+void
+modref_access_node::update2 (poly_int64 parm_offset1,
+			     poly_int64 offset1, poly_int64 size1,
+			     poly_int64 max_size1,
+			     poly_int64 offset2, poly_int64 size2,
+			     poly_int64 max_size2,
+			     bool record_adjustments)
+{
+  poly_int64 new_size = size1;
+
+  if (!known_size_p (size2)
+      || known_le (size2, size1))
+    new_size = size2;
+  else
+    gcc_checking_assert (known_le (size1, size2));
+
+  if (known_le (offset1, offset2))
+    ;
+  else if (known_le (offset2, offset1))
+    {
+      std::swap (offset1, offset2);
+      std::swap (max_size1, max_size2);
+    }
+  else
+    gcc_unreachable ();
+
+  poly_int64 new_max_size;
+
+  if (!known_size_p (max_size1))
+    new_max_size = max_size1;
+  else if (!known_size_p (max_size2))
+    new_max_size = max_size2;
+  else
+    {
+      new_max_size = max_size2 + offset2 - offset1;
+      if (known_le (new_max_size, max_size1))
+	new_max_size = max_size1;
+    }
+
+  update (parm_offset1, offset1,
+	  new_size, new_max_size, record_adjustments);
+}
+
+/* Given access nodes THIS and A, return true if they
+   can be done with common parm_offsets.  In this case
+   return parm offset in new_parm_offset, new_offset
+   which is start of range in THIS and new_aoffset that
+   is start of range in A.  */
+bool
+modref_access_node::combined_offsets (const modref_access_node &a,
+				      poly_int64 *new_parm_offset,
+				      poly_int64 *new_offset,
+				      poly_int64 *new_aoffset) const
+{
+  gcc_checking_assert (parm_offset_known && a.parm_offset_known);
+  if (known_le (a.parm_offset, parm_offset))
+    {
+      *new_offset = offset
+		    + ((parm_offset - a.parm_offset)
+		       << LOG2_BITS_PER_UNIT);
+      *new_aoffset = a.offset;
+      *new_parm_offset = a.parm_offset;
+      return true;
+    }
+  else if (known_le (parm_offset, a.parm_offset))
+    {
+      *new_aoffset = a.offset
+		      + ((a.parm_offset - parm_offset)
+			 << LOG2_BITS_PER_UNIT);
+      *new_offset = offset;
+      *new_parm_offset = parm_offset;
+      return true;
+    }
+  else
+    return false;
+}
+
+/* Try to optimize the access ACCESSES list after entry INDEX was modified.  */
+void
+modref_access_node::try_merge_with (vec <modref_access_node, va_gc> *&accesses,
+				    size_t index)
+{
+  size_t i;
+
+  for (i = 0; i < accesses->length ();)
+    if (i != index)
+      {
+	bool found = false, restart = false;
+	modref_access_node *a = &(*accesses)[i];
+	modref_access_node *n = &(*accesses)[index];
+
+	if (n->contains (*a))
+	  found = true;
+	if (!found && n->merge (*a, false))
+	  found = restart = true;
+	gcc_checking_assert (found || !a->merge (*n, false));
+	if (found)
+	  {
+	    accesses->unordered_remove (i);
+	    if (index == accesses->length ())
+	      {
+		index = i;
+		i++;
+	      }
+	    if (restart)
+	      i = 0;
+	  }
+	else
+	  i++;
+      }
+    else
+      i++;
+}
+
+/* Insert access with OFFSET and SIZE.
+   Collapse tree if it has more than MAX_ACCESSES entries.
+   If RECORD_ADJUSTMENTs is true avoid too many interval extensions.
+   Return true if record was changed.
+
+   Reutrn 0 if nothing changed, 1 if insert was successful and -1
+   if entries should be collapsed.  */
+int
+modref_access_node::insert (vec <modref_access_node, va_gc> *&accesses,
+			    modref_access_node a, size_t max_accesses,
+			    bool record_adjustments)
+{
+  size_t i, j;
+  modref_access_node *a2;
+
+  /* Verify that list does not contain redundant accesses.  */
+  if (flag_checking)
+    {
+      size_t i, i2;
+      modref_access_node *a, *a2;
+
+      FOR_EACH_VEC_SAFE_ELT (accesses, i, a)
+	{
+	  FOR_EACH_VEC_SAFE_ELT (accesses, i2, a2)
+	    if (i != i2)
+	      gcc_assert (!a->contains (*a2));
+	}
+    }
+
+  FOR_EACH_VEC_SAFE_ELT (accesses, i, a2)
+    {
+      if (a2->contains (a))
+	return 0;
+      if (a.contains (*a2))
+	{
+	  a.adjustments = 0;
+	  a2->parm_index = a.parm_index;
+	  a2->parm_offset_known = a.parm_offset_known;
+	  a2->update (a.parm_offset, a.offset, a.size, a.max_size,
+		      record_adjustments);
+	  modref_access_node::try_merge_with (accesses, i);
+	  return 1;
+	}
+      if (a2->merge (a, record_adjustments))
+	{
+	  modref_access_node::try_merge_with (accesses, i);
+	  return 1;
+	}
+      gcc_checking_assert (!(a == *a2));
+    }
+
+  /* If this base->ref pair has too many accesses stored, we will clear
+     all accesses and bail out.  */
+  if (accesses && accesses->length () >= max_accesses)
+    {
+      if (max_accesses < 2)
+	return -1;
+      /* Find least harmful merge and perform it.  */
+      int best1 = -1, best2 = -1;
+      FOR_EACH_VEC_SAFE_ELT (accesses, i, a2)
+	{
+	  for (j = i + 1; j < accesses->length (); j++)
+	    if (best1 < 0
+		|| modref_access_node::closer_pair_p
+		     (*a2, (*accesses)[j],
+		      (*accesses)[best1],
+		      best2 < 0 ? a : (*accesses)[best2]))
+	      {
+		best1 = i;
+		best2 = j;
+	      }
+	  if (modref_access_node::closer_pair_p
+		     (*a2, a,
+		      (*accesses)[best1],
+		      best2 < 0 ? a : (*accesses)[best2]))
+	    {
+	      best1 = i;
+	      best2 = -1;
+	    }
+	}
+      (*accesses)[best1].forced_merge (best2 < 0 ? a : (*accesses)[best2],
+				       record_adjustments);
+      /* Check that merging indeed merged ranges.  */
+      gcc_checking_assert ((*accesses)[best1].contains
+			       (best2 < 0 ? a : (*accesses)[best2]));
+      if (!(*accesses)[best1].useful_p ())
+	return -1;
+      if (dump_file && best2 >= 0)
+	fprintf (dump_file,
+		 "--param param=modref-max-accesses limit reached;"
+		 " merging %i and %i\n", best1, best2);
+      else if (dump_file)
+	fprintf (dump_file,
+		 "--param param=modref-max-accesses limit reached;"
+		 " merging with %i\n", best1);
+      modref_access_node::try_merge_with (accesses, best1);
+      if (best2 >= 0)
+	insert (accesses, a, max_accesses, record_adjustments);
+      return 1;
+    }
+  a.adjustments = 0;
+  vec_safe_push (accesses, a);
+  return 1;
+}
+
+/* Return true if range info is useful.  */
+bool
+modref_access_node::range_info_useful_p () const
+{
+  return parm_index != MODREF_UNKNOWN_PARM && parm_offset_known
+	 && (known_size_p (size)
+	     || known_size_p (max_size)
+	     || known_ge (offset, 0));
+}
+
+/* Dump range to debug OUT.  */
+void
+modref_access_node::dump (FILE *out)
+{
+  if (parm_index != MODREF_UNKNOWN_PARM)
+    {
+      if (parm_index >= 0)
+	fprintf (out, " Parm %i", parm_index);
+      else if (parm_index == MODREF_STATIC_CHAIN_PARM)
+	fprintf (out, " Static chain");
+      else
+	gcc_unreachable ();
+      if (parm_offset_known)
+	{
+	  fprintf (out, " param offset:");
+	  print_dec ((poly_int64_pod)parm_offset, out, SIGNED);
+	}
+    }
+  if (range_info_useful_p ())
+    {
+      fprintf (out, " offset:");
+      print_dec ((poly_int64_pod)offset, out, SIGNED);
+      fprintf (out, " size:");
+      print_dec ((poly_int64_pod)size, out, SIGNED);
+      fprintf (out, " max_size:");
+      print_dec ((poly_int64_pod)max_size, out, SIGNED);
+      if (adjustments)
+	fprintf (out, " adjusted %i times", adjustments);
+    }
+  fprintf (out, "\n");
+}
+
+/* Return tree corresponding to parameter of the range in STMT.  */
+tree
+modref_access_node::get_call_arg (const gcall *stmt) const
+{
+  if (parm_index == MODREF_UNKNOWN_PARM)
+    return NULL;
+  if (parm_index == MODREF_STATIC_CHAIN_PARM)
+    return gimple_call_chain (stmt);
+  /* MODREF_RETSLOT_PARM should not happen in access trees since the store
+     is seen explicitly in the caller.  */
+  gcc_checking_assert (parm_index >= 0);
+  if (parm_index >= (int)gimple_call_num_args (stmt))
+    return NULL;
+  return gimple_call_arg (stmt, parm_index);
+}
+
+/* Return tree corresponding to parameter of the range in STMT.  */
+bool
+modref_access_node::get_ao_ref (const gcall *stmt, ao_ref *ref) const
+{
+  tree arg;
+
+  if (!parm_offset_known || !(arg = get_call_arg (stmt)))
+    return false;
+  poly_offset_int off = (poly_offset_int)offset
+	+ ((poly_offset_int)parm_offset << LOG2_BITS_PER_UNIT);
+  poly_int64 off2;
+  if (!off.to_shwi (&off2))
+    return false;
+  ao_ref_init_from_ptr_and_range (ref, arg, true, off2, size, max_size);
+  return true;
+}
+
+/* Return true A is a subkill.  */
+bool
+modref_access_node::contains_for_kills (const modref_access_node &a) const
+{
+  poly_int64 aoffset_adj = 0;
+
+  gcc_checking_assert (parm_index != MODREF_UNKNOWN_PARM
+		       && a.parm_index != MODREF_UNKNOWN_PARM);
+  if (parm_index != a.parm_index)
+    return false;
+  gcc_checking_assert (parm_offset_known && a.parm_offset_known);
+  aoffset_adj = (a.parm_offset - parm_offset)
+		* BITS_PER_UNIT;
+  gcc_checking_assert (range_info_useful_p () && a.range_info_useful_p ());
+  return known_subrange_p (a.offset + aoffset_adj,
+			   a.max_size, offset, max_size);
+}
+
+/* Merge two ranges both starting at parm_offset1 and update THIS
+   with result.  */
+bool
+modref_access_node::update_for_kills (poly_int64 parm_offset1,
+				      poly_int64 offset1,
+				      poly_int64 max_size1,
+				      poly_int64 offset2,
+				      poly_int64 max_size2,
+				      bool record_adjustments)
+{
+  if (known_le (offset1, offset2))
+    ;
+  else if (known_le (offset2, offset1))
+    {
+      std::swap (offset1, offset2);
+      std::swap (max_size1, max_size2);
+    }
+  else
+    gcc_unreachable ();
+
+  poly_int64 new_max_size = max_size2 + offset2 - offset1;
+  if (known_le (new_max_size, max_size1))
+    new_max_size = max_size1;
+  if (known_eq (parm_offset, parm_offset1)
+      && known_eq (offset, offset1)
+      && known_eq (size, new_max_size)
+      && known_eq (max_size, new_max_size))
+    return false;
+
+  if (!record_adjustments
+      || (++adjustments) < param_modref_max_adjustments)
+    {
+      parm_offset = parm_offset1;
+      offset = offset1;
+      max_size = new_max_size;
+      size = new_max_size;
+      gcc_checking_assert (useful_for_kill_p ());
+      return true;
+    }
+  return false;
+}
+
+/* Merge in access A if it is possible to do without losing
+   precision.  Return true if successful.
+   Unlike merge assume that both accesses are always executed
+   and merge size the same was as max_size.  */
+bool
+modref_access_node::merge_for_kills (const modref_access_node &a,
+				     bool record_adjustments)
+{
+  poly_int64 offset1 = 0;
+  poly_int64 aoffset1 = 0;
+  poly_int64 new_parm_offset = 0;
+
+  /* We assume that containment was tested earlier.  */
+  gcc_checking_assert (!contains_for_kills (a) && !a.contains_for_kills (*this)
+		       && useful_for_kill_p () && a.useful_for_kill_p ());
+
+  if (parm_index != a.parm_index
+      || !combined_offsets (a, &new_parm_offset, &offset1, &aoffset1))
+    return false;
+
+  if (known_le (offset1, aoffset1))
+   {
+     if (!known_size_p (max_size)
+	 || known_ge (offset1 + max_size, aoffset1))
+       return update_for_kills (new_parm_offset, offset1, max_size,
+				aoffset1, a.max_size, record_adjustments);
+   }
+  else if (known_le (aoffset1, offset1))
+   {
+     if (!known_size_p (a.max_size)
+	 || known_ge (aoffset1 + a.max_size, offset1))
+       return update_for_kills (new_parm_offset, offset1, max_size,
+				aoffset1, a.max_size, record_adjustments);
+   }
+  return false;
+}
+
+/* Insert new kill A into KILLS.  If RECORD_ADJUSTMENTS is true limit number
+   of changes to each entry.  Return true if something changed.  */
+
+bool
+modref_access_node::insert_kill (vec<modref_access_node> &kills,
+				 modref_access_node &a, bool record_adjustments)
+{
+  size_t index;
+  modref_access_node *a2;
+  bool merge = false;
+
+  gcc_checking_assert (a.useful_for_kill_p ());
+
+  /* See if we have corresponding entry already or we can merge with
+     neighbouring entry.  */
+  FOR_EACH_VEC_ELT (kills, index, a2)
+    {
+      if (a2->contains_for_kills (a))
+	return false;
+      if (a.contains_for_kills (*a2))
+	{
+	  a.adjustments = 0;
+	  *a2 = a;
+	  merge = true;
+	  break;
+	}
+      if (a2->merge_for_kills (a, record_adjustments))
+	{
+	  merge = true;
+	  break;
+	}
+    }
+  /* If entry was not found, insert it.  */
+  if (!merge)
+    {
+      if ((int)kills.length () >= param_modref_max_accesses)
+	{
+	  if (dump_file)
+	    fprintf (dump_file,
+		     "--param param=modref-max-accesses limit reached:");
+	  return false;
+	}
+      a.adjustments = 0;
+      kills.safe_push (a);
+      return true;
+    }
+  /* Extending range in an entry may make it possible to merge it with
+     other entries.  */
+  size_t i;
+
+  for (i = 0; i < kills.length ();)
+    if (i != index)
+      {
+	bool found = false, restart = false;
+	modref_access_node *a = &kills[i];
+	modref_access_node *n = &kills[index];
+
+	if (n->contains_for_kills (*a))
+	  found = true;
+	if (!found && n->merge_for_kills (*a, false))
+	  found = restart = true;
+	gcc_checking_assert (found || !a->merge_for_kills (*n, false));
+	if (found)
+	  {
+	    kills.unordered_remove (i);
+	    if (index == kills.length ())
+	      {
+		index = i;
+		i++;
+	      }
+	    if (restart)
+	      i = 0;
+	  }
+	else
+	  i++;
+      }
+    else
+      i++;
+  return true;
+}
+
 
 #if CHECKING_P
 
@@ -138,7 +929,7 @@ test_merge ()
   t2->insert (3, 4, a, false);
   t2->insert (3, 5, a, false);
 
-  t1->merge (t2, NULL, false);
+  t1->merge (t2, NULL, NULL, false);
 
   ASSERT_FALSE (t1->every_base);
   ASSERT_NE (t1->bases, NULL);
diff --git a/gcc/ipa-modref-tree.h b/gcc/ipa-modref-tree.h
index 9976e48..1bf2aa8 100644
--- a/gcc/ipa-modref-tree.h
+++ b/gcc/ipa-modref-tree.h
@@ -42,10 +42,25 @@ along with GCC; see the file COPYING3.  If not see
 
 struct ipa_modref_summary;
 
-/* Memory access.  */
+/* parm indexes greater than 0 are normal parms.
+   Some negative values have special meaning.  */
+enum modref_special_parms {
+  MODREF_UNKNOWN_PARM = -1,
+  MODREF_STATIC_CHAIN_PARM = -2,
+  MODREF_RETSLOT_PARM = -3,
+  /* Used in modref_parm_map to tak references which can be removed
+     from the summary during summary update since they now points to loca
+     memory.  */
+  MODREF_LOCAL_MEMORY_PARM = -4
+};
+
+/* Modref record accesses relative to function parameters.
+   This is entry for single access specifying its base and access range.
+
+   Accesses can be collected to boundedly sized arrays using
+   modref_access_node::insert.  */
 struct GTY(()) modref_access_node
 {
-
   /* Access range information (in bits).  */
   poly_int64 offset;
   poly_int64 size;
@@ -62,394 +77,61 @@ struct GTY(()) modref_access_node
      This has to be limited in order to keep dataflow finite.  */
   unsigned char adjustments;
 
-  /* Return true if access node holds no useful info.  */
+  /* Return true if access node holds some useful info.  */
   bool useful_p () const
     {
-      return parm_index != -1;
+      return parm_index != MODREF_UNKNOWN_PARM;
     }
-  /* Return true if range info is useful.  */
-  bool range_info_useful_p () const
+  /* Return true if access can be used to determine a kill.  */
+  bool useful_for_kill_p () const
     {
-      return parm_index != -1 && parm_offset_known
-	     && (known_size_p (size)
-		 || known_size_p (max_size)
-		 || known_ge (offset, 0));
+      return parm_offset_known && parm_index != MODREF_UNKNOWN_PARM
+	     && parm_index != MODREF_RETSLOT_PARM && known_size_p (size)
+	     && known_eq (max_size, size);
     }
+  /* Dump range to debug OUT.  */
+  void dump (FILE *out);
   /* Return true if both accesses are the same.  */
-  bool operator == (modref_access_node &a) const
-    {
-      if (parm_index != a.parm_index)
-	return false;
-      if (parm_index >= 0)
-	{
-	  if (parm_offset_known != a.parm_offset_known)
-	    return false;
-	  if (parm_offset_known
-	      && !known_eq (parm_offset, a.parm_offset))
-	    return false;
-	}
-      if (range_info_useful_p () != a.range_info_useful_p ())
-	return false;
-      if (range_info_useful_p ()
-	  && (!known_eq (a.offset, offset)
-	      || !known_eq (a.size, size)
-	      || !known_eq (a.max_size, max_size)))
-	return false;
-      return true;
-    }
-  /* Return true A is a subaccess.  */
-  bool contains (const modref_access_node &a) const
-    {
-      poly_int64 aoffset_adj = 0;
-      if (parm_index >= 0)
-	{
-	  if (parm_index != a.parm_index)
-	    return false;
-	  if (parm_offset_known)
-	    {
-	       if (!a.parm_offset_known)
-		 return false;
-	       /* Accesses are never below parm_offset, so look
-		  for smaller offset.
-		  If access ranges are known still allow merging
-		  when bit offsets comparsion passes.  */
-	       if (!known_le (parm_offset, a.parm_offset)
-		   && !range_info_useful_p ())
-		 return false;
-	       aoffset_adj = (a.parm_offset - parm_offset)
-			     << LOG2_BITS_PER_UNIT;
-	    }
-	}
-      if (range_info_useful_p ())
-	{
-	  if (!a.range_info_useful_p ())
-	    return false;
-	  /* Sizes of stores are used to check that object is big enough
-	     to fit the store, so smaller or unknown sotre is more general
-	     than large store.  */
-	  if (known_size_p (size)
-	      && (!known_size_p (a.size)
-		  || !known_le (size, a.size)))
-	    return false;
-	  if (known_size_p (max_size))
-	    return known_subrange_p (a.offset + aoffset_adj,
-				     a.max_size, offset, max_size);
-	  else
-	    return known_le (offset, a.offset + aoffset_adj);
-	}
-      return true;
-    }
-  /* Update access range to new parameters.
-     If RECORD_ADJUSTMENTS is true, record number of changes in the access
-     and if threshold is exceeded start dropping precision
-     so only constantly many updates are possible.  This makes dataflow
-     to converge.  */
-  void update (poly_int64 parm_offset1,
-	       poly_int64 offset1, poly_int64 size1, poly_int64 max_size1,
-	       bool record_adjustments)
-    {
-      if (known_eq (parm_offset, parm_offset1)
-	  && known_eq (offset, offset1)
-	  && known_eq (size, size1)
-	  && known_eq (max_size, max_size1))
-	return;
-      if (!record_adjustments
-	  || (++adjustments) < param_modref_max_adjustments)
-	{
-	  parm_offset = parm_offset1;
-	  offset = offset1;
-	  size = size1;
-	  max_size = max_size1;
-	}
-      else
-	{
-	  if (dump_file)
-	    fprintf (dump_file,
-		     "--param param=modref-max-adjustments limit reached:");
-	  if (!known_eq (parm_offset, parm_offset1))
-	    {
-	      if (dump_file)
-		fprintf (dump_file, " parm_offset cleared");
-	      parm_offset_known = false;
-	    }
-	  if (!known_eq (size, size1))
-	    {
-	      size = -1;
-	      if (dump_file)
-		fprintf (dump_file, " size cleared");
-	    }
-	  if (!known_eq (max_size, max_size1))
-	    {
-	      max_size = -1;
-	      if (dump_file)
-		fprintf (dump_file, " max_size cleared");
-	    }
-	  if (!known_eq (offset, offset1))
-	    {
-	      offset = 0;
-	      if (dump_file)
-		fprintf (dump_file, " offset cleared");
-	    }
-	  if (dump_file)
-	    fprintf (dump_file, "\n");
-	}
-    }
-  /* Merge in access A if it is possible to do without losing
-     precision.  Return true if successful.
-     If RECORD_ADJUSTMENTs is true, remember how many interval
-     was prolonged and punt when there are too many.  */
-  bool merge (const modref_access_node &a, bool record_adjustments)
-    {
-      poly_int64 offset1 = 0;
-      poly_int64 aoffset1 = 0;
-      poly_int64 new_parm_offset = 0;
-
-      /* We assume that containment was tested earlier.  */
-      gcc_checking_assert (!contains (a) && !a.contains (*this));
-      if (parm_index >= 0)
-	{
-	  if (parm_index != a.parm_index)
-	    return false;
-	  if (parm_offset_known)
-	    {
-	      if (!a.parm_offset_known)
-		return false;
-	      if (!combined_offsets (a, &new_parm_offset, &offset1, &aoffset1))
-		return false;
-	    }
-	}
-      /* See if we can merge ranges.  */
-      if (range_info_useful_p ())
-	{
-	  /* In this case we have containment that should be
-	     handled earlier.  */
-	  gcc_checking_assert (a.range_info_useful_p ());
-
-	  /* If a.size is less specified than size, merge only
-	     if intervals are otherwise equivalent.  */
-	  if (known_size_p (size)
-	      && (!known_size_p (a.size) || known_lt (a.size, size)))
-	    {
-	      if (((known_size_p (max_size) || known_size_p (a.max_size))
-		   && !known_eq (max_size, a.max_size))
-		   || !known_eq (offset1, aoffset1))
-		return false;
-	      update (new_parm_offset, offset1, a.size, max_size,
-		      record_adjustments);
-	      return true;
-	    }
-	  /* If sizes are same, we can extend the interval.  */
-	  if ((known_size_p (size) || known_size_p (a.size))
-	      && !known_eq (size, a.size))
-	    return false;
-	  if (known_le (offset1, aoffset1))
-	    {
-	      if (!known_size_p (max_size)
-		  || known_ge (offset1 + max_size, aoffset1))
-		{
-		  update2 (new_parm_offset, offset1, size, max_size,
-			   aoffset1, a.size, a.max_size,
-			   record_adjustments);
-		  return true;
-		}
-	    }
-	  else if (known_le (aoffset1, offset1))
-	    {
-	      if (!known_size_p (a.max_size)
-		  || known_ge (aoffset1 + a.max_size, offset1))
-		{
-		  update2 (new_parm_offset, offset1, size, max_size,
-			   aoffset1, a.size, a.max_size,
-			   record_adjustments);
-		  return true;
-		}
-	    }
-	  return false;
-	}
-      update (new_parm_offset, offset1,
-	      size, max_size, record_adjustments);
-      return true;
-    }
-  /* Return true if A1 and B1 can be merged with lower informatoin
-     less than A2 and B2.
-     Assume that no containment or lossless merging is possible.  */
-  static bool closer_pair_p (const modref_access_node &a1,
-			     const modref_access_node &b1,
-			     const modref_access_node &a2,
-			     const modref_access_node &b2)
-    {
-      /* Merging different parm indexes comes to complete loss
-	 of range info.  */
-      if (a1.parm_index != b1.parm_index)
-	return false;
-      if (a2.parm_index != b2.parm_index)
-	return true;
-      /* If parm is known and parm indexes are the same we should
-	 already have containment.  */
-      gcc_checking_assert (a1.parm_offset_known && b1.parm_offset_known);
-      gcc_checking_assert (a2.parm_offset_known && b2.parm_offset_known);
-
-      /* First normalize offsets for parm offsets.  */
-      poly_int64 new_parm_offset, offseta1, offsetb1, offseta2, offsetb2;
-      if (!a1.combined_offsets (b1, &new_parm_offset, &offseta1, &offsetb1)
-	  || !a2.combined_offsets (b2, &new_parm_offset, &offseta2, &offsetb2))
-	gcc_unreachable ();
-
-
-      /* Now compute distnace of the intervals.  */
-      poly_int64 dist1, dist2;
-      if (known_le (offseta1, offsetb1))
-	{
-	  if (!known_size_p (a1.max_size))
-	    dist1 = 0;
-	  else
-	    dist1 = offsetb1 - offseta1 - a1.max_size;
-	}
-      else
-	{
-	  if (!known_size_p (b1.max_size))
-	    dist1 = 0;
-	  else
-	    dist1 = offseta1 - offsetb1 - b1.max_size;
-	}
-      if (known_le (offseta2, offsetb2))
-	{
-	  if (!known_size_p (a2.max_size))
-	    dist2 = 0;
-	  else
-	    dist2 = offsetb2 - offseta2 - a2.max_size;
-	}
-      else
-	{
-	  if (!known_size_p (b2.max_size))
-	    dist2 = 0;
-	  else
-	    dist2 = offseta2 - offsetb2 - b2.max_size;
-	}
-      /* It may happen that intervals overlap in case size
-	 is different.  Preffer the overlap to non-overlap.  */
-      if (known_lt (dist1, 0) && known_ge (dist2, 0))
-	return true;
-      if (known_lt (dist2, 0) && known_ge (dist1, 0))
-	return false;
-      if (known_lt (dist1, 0))
-	/* If both overlaps minimize overlap.  */
-	return known_le (dist2, dist1);
-      else
-	/* If both are disjoint look for smaller distance.  */
-	return known_le (dist1, dist2);
-    }
-
-  /* Merge in access A while losing precision.  */
-  void forced_merge (const modref_access_node &a, bool record_adjustments)
-    {
-      if (parm_index != a.parm_index)
-	{
-	  gcc_checking_assert (parm_index != -1);
-	  parm_index = -1;
-	  return;
-	}
-
-      /* We assume that containment and lossless merging
-	 was tested earlier.  */
-      gcc_checking_assert (!contains (a) && !a.contains (*this)
-			   && !merge (a, record_adjustments));
-      gcc_checking_assert (parm_offset_known && a.parm_offset_known);
-
-      poly_int64 new_parm_offset, offset1, aoffset1;
-      if (!combined_offsets (a, &new_parm_offset, &offset1, &aoffset1))
-	{
-	  parm_offset_known = false;
-	  return;
-	}
-      gcc_checking_assert (range_info_useful_p ()
-			   && a.range_info_useful_p ());
-      if (record_adjustments)
-	adjustments += a.adjustments;
-      update2 (new_parm_offset,
-	       offset1, size, max_size,
-	       aoffset1, a.size, a.max_size,
-	       record_adjustments);
-    }
+  bool operator == (modref_access_node &a) const;
+  /* Return true if range info is useful.  */
+  bool range_info_useful_p () const;
+  /* Return tree corresponding to parameter of the range in STMT.  */
+  tree get_call_arg (const gcall *stmt) const;
+  /* Build ao_ref corresponding to the access and return true if succesful.  */
+  bool get_ao_ref (const gcall *stmt, class ao_ref *ref) const;
+  /* Insert A into vector ACCESSES.  Limit size of vector to MAX_ACCESSES and
+     if RECORD_ADJUSTMENT is true keep track of adjustment counts.
+     Return 0 if nothing changed, 1 is insertion suceeded and -1 if failed.  */
+  static int insert (vec <modref_access_node, va_gc> *&accesses,
+		     modref_access_node a, size_t max_accesses,
+		     bool record_adjustments);
+  /* Same as insert but for kills where we are conservative the other way
+     around: if information is lost, the kill is lost.  */
+  static bool insert_kill (vec<modref_access_node> &kills,
+			   modref_access_node &a, bool record_adjustments);
 private:
-  /* Merge two ranges both starting at parm_offset1 and update THIS
-     with result.  */
-  void update2 (poly_int64 parm_offset1,
-		poly_int64 offset1, poly_int64 size1, poly_int64 max_size1,
-		poly_int64 offset2, poly_int64 size2, poly_int64 max_size2,
-		bool record_adjustments)
-    {
-      poly_int64 new_size = size1;
-
-      if (!known_size_p (size2)
-	  || known_le (size2, size1))
-	new_size = size2;
-      else
-	gcc_checking_assert (known_le (size1, size2));
-
-      if (known_le (offset1, offset2))
-	;
-      else if (known_le (offset2, offset1))
-	{
-	  std::swap (offset1, offset2);
-	  std::swap (max_size1, max_size2);
-	}
-      else
-	gcc_unreachable ();
-
-      poly_int64 new_max_size;
-
-      if (!known_size_p (max_size1))
-	new_max_size = max_size1;
-      else if (!known_size_p (max_size2))
-	new_max_size = max_size2;
-      else
-	{
-	  new_max_size = max_size2 + offset2 - offset1;
-	  if (known_le (new_max_size, max_size1))
-	    new_max_size = max_size1;
-	}
-
-      update (parm_offset1, offset1,
-	      new_size, new_max_size, record_adjustments);
-    }
-  /* Given access nodes THIS and A, return true if they
-     can be done with common parm_offsets.  In this case
-     return parm offset in new_parm_offset, new_offset
-     which is start of range in THIS and new_aoffset that
-     is start of range in A.  */
-  bool combined_offsets (const modref_access_node &a,
-			 poly_int64 *new_parm_offset,
-			 poly_int64 *new_offset,
-			 poly_int64 *new_aoffset) const
-    {
-      gcc_checking_assert (parm_offset_known && a.parm_offset_known);
-      if (known_le (a.parm_offset, parm_offset))
-	{
-	  *new_offset = offset
-			+ ((parm_offset - a.parm_offset)
-			   << LOG2_BITS_PER_UNIT);
-	  *new_aoffset = a.offset;
-	  *new_parm_offset = a.parm_offset;
-	  return true;
-	}
-      else if (known_le (parm_offset, a.parm_offset))
-	{
-	  *new_aoffset = a.offset
-	  		  + ((a.parm_offset - parm_offset)
-			     << LOG2_BITS_PER_UNIT);
-	  *new_offset = offset;
-	  *new_parm_offset = parm_offset;
-	  return true;
-	}
-      else
-	return false;
-    }
+  bool contains (const modref_access_node &) const;
+  bool contains_for_kills (const modref_access_node &) const;
+  void update (poly_int64, poly_int64, poly_int64, poly_int64, bool);
+  bool update_for_kills (poly_int64, poly_int64, poly_int64,
+			 poly_int64, poly_int64, bool);
+  bool merge (const modref_access_node &, bool);
+  bool merge_for_kills (const modref_access_node &, bool);
+  static bool closer_pair_p (const modref_access_node &,
+			     const modref_access_node &,
+			     const modref_access_node &,
+			     const modref_access_node &);
+  void forced_merge (const modref_access_node &, bool);
+  void update2 (poly_int64, poly_int64, poly_int64, poly_int64,
+		poly_int64, poly_int64, poly_int64, bool);
+  bool combined_offsets (const modref_access_node &,
+			 poly_int64 *, poly_int64 *, poly_int64 *) const;
+  static void try_merge_with (vec <modref_access_node, va_gc> *&, size_t);
 };
 
 /* Access node specifying no useful info.  */
 const modref_access_node unspecified_modref_access_node
-		 = {0, -1, -1, 0, -1, false, 0};
+		 = {0, -1, -1, 0, MODREF_UNKNOWN_PARM, false, 0};
 
 template <typename T>
 struct GTY((user)) modref_ref_node
@@ -472,20 +154,6 @@ struct GTY((user)) modref_ref_node
     every_access = true;
   }
 
-  /* Verify that list does not contain redundant accesses.  */
-  void verify ()
-  {
-    size_t i, i2;
-    modref_access_node *a, *a2;
-
-    FOR_EACH_VEC_SAFE_ELT (accesses, i, a)
-      {
-	FOR_EACH_VEC_SAFE_ELT (accesses, i2, a2)
-	  if (i != i2)
-	    gcc_assert (!a->contains (*a2));
-      }
-  }
-
   /* Insert access with OFFSET and SIZE.
      Collapse tree if it has more than MAX_ACCESSES entries.
      If RECORD_ADJUSTMENTs is true avoid too many interval extensions.
@@ -497,12 +165,12 @@ struct GTY((user)) modref_ref_node
     if (every_access)
       return false;
 
-    /* Otherwise, insert a node for the ref of the access under the base.  */
-    size_t i, j;
-    modref_access_node *a2;
-
-    if (flag_checking)
-      verify ();
+    /* Only the following kind of paramters needs to be tracked.
+       We do not track return slots because they are seen as a direct store
+       in the caller.  */
+    gcc_checking_assert (a.parm_index >= 0
+			 || a.parm_index == MODREF_STATIC_CHAIN_PARM
+			 || a.parm_index == MODREF_UNKNOWN_PARM);
 
     if (!a.useful_p ())
       {
@@ -514,130 +182,17 @@ struct GTY((user)) modref_ref_node
 	return false;
       }
 
-    FOR_EACH_VEC_SAFE_ELT (accesses, i, a2)
-      {
-	if (a2->contains (a))
-	  return false;
-	if (a.contains (*a2))
-	  {
-	    a.adjustments = 0;
-	    a2->parm_index = a.parm_index;
-	    a2->parm_offset_known = a.parm_offset_known;
-	    a2->update (a.parm_offset, a.offset, a.size, a.max_size,
-			record_adjustments);
-	    try_merge_with (i);
-	    return true;
-	  }
-	if (a2->merge (a, record_adjustments))
-	  {
-	    try_merge_with (i);
-	    return true;
-	  }
-	gcc_checking_assert (!(a == *a2));
-      }
-
-    /* If this base->ref pair has too many accesses stored, we will clear
-       all accesses and bail out.  */
-    if (accesses && accesses->length () >= max_accesses)
+    int ret = modref_access_node::insert (accesses, a, max_accesses,
+					  record_adjustments);
+    if (ret == -1)
       {
-	if (max_accesses < 2)
-	  {
-	    collapse ();
-	    if (dump_file)
-	      fprintf (dump_file,
-		       "--param param=modref-max-accesses limit reached;"
-		       " collapsing\n");
-	    return true;
-	  }
-	/* Find least harmful merge and perform it.  */
-	int best1 = -1, best2 = -1;
-	FOR_EACH_VEC_SAFE_ELT (accesses, i, a2)
-	  {
-	    for (j = i + 1; j < accesses->length (); j++)
-	      if (best1 < 0
-		  || modref_access_node::closer_pair_p
-		       (*a2, (*accesses)[j],
-			(*accesses)[best1],
-			best2 < 0 ? a : (*accesses)[best2]))
-		{
-		  best1 = i;
-		  best2 = j;
-		}
-	    if (modref_access_node::closer_pair_p
-		       (*a2, a,
-			(*accesses)[best1],
-			best2 < 0 ? a : (*accesses)[best2]))
-	      {
-		best1 = i;
-		best2 = -1;
-	      }
-	  }
-	(*accesses)[best1].forced_merge (best2 < 0 ? a : (*accesses)[best2],
-					 record_adjustments);
-	/* Check that merging indeed merged ranges.  */
-	gcc_checking_assert ((*accesses)[best1].contains
-				 (best2 < 0 ? a : (*accesses)[best2]));
-	if (!(*accesses)[best1].useful_p ())
-	  {
-	    collapse ();
-	    if (dump_file)
-	      fprintf (dump_file,
-		       "--param param=modref-max-accesses limit reached;"
-		       " collapsing\n");
-	    return true;
-	  }
-	if (dump_file && best2 >= 0)
-	  fprintf (dump_file,
-		   "--param param=modref-max-accesses limit reached;"
-		   " merging %i and %i\n", best1, best2);
-	else if (dump_file)
+	if (dump_file)
 	  fprintf (dump_file,
 		   "--param param=modref-max-accesses limit reached;"
-		   " merging with %i\n", best1);
-	try_merge_with (best1);
-	if (best2 >= 0)
-	  insert_access (a, max_accesses, record_adjustments);
-	return 1;
+		   " collapsing\n");
+	collapse ();
       }
-    a.adjustments = 0;
-    vec_safe_push (accesses, a);
-    return true;
-  }
-private:
-  /* Try to optimize the access list after entry INDEX was modified.  */
-  void
-  try_merge_with (size_t index)
-  {
-    size_t i;
-
-    for (i = 0; i < accesses->length ();)
-      if (i != index)
-	{
-	  bool found = false, restart = false;
-	  modref_access_node *a = &(*accesses)[i];
-	  modref_access_node *n = &(*accesses)[index];
-
-	  if (n->contains (*a))
-	    found = true;
-	  if (!found && n->merge (*a, false))
-	    found = restart = true;
-	  gcc_checking_assert (found || !a->merge (*n, false));
-	  if (found)
-	    {
-	      accesses->unordered_remove (i);
-	      if (index == accesses->length ())
-		{
-		  index = i;
-		  i++;
-		}
-	      if (restart)
-		i = 0;
-	    }
-	  else
-	    i++;
-	}
-      else
-	i++;
+    return ret != 0;
   }
 };
 
@@ -729,9 +284,7 @@ struct GTY((user)) modref_base_node
 struct modref_parm_map
 {
   /* Index of parameter we translate to.
-     -1 indicates that parameter is unknown
-     -2 indicates that parameter points to local memory and access can be
-	discarded.  */
+     Values from special_params enum are permitted too.  */
   int parm_index;
   bool parm_offset_known;
   poly_int64 parm_offset;
@@ -813,6 +366,36 @@ struct GTY((user)) modref_tree
 
     bool changed = false;
 
+    /* We may end up with max_size being less than size for accesses past the
+       end of array.  Those are undefined and safe to ignore.  */
+    if (a.range_info_useful_p ()
+	&& known_size_p (a.size) && known_size_p (a.max_size)
+	&& known_lt (a.max_size, a.size))
+      {
+	if (dump_file)
+	  fprintf (dump_file,
+		   "   - Paradoxical range. Ignoring\n");
+	return false;
+      }
+    if (known_size_p (a.size)
+	&& known_eq (a.size, 0))
+      {
+	if (dump_file)
+	  fprintf (dump_file,
+		   "   - Zero size. Ignoring\n");
+	return false;
+      }
+    if (known_size_p (a.max_size)
+	&& known_eq (a.max_size, 0))
+      {
+	if (dump_file)
+	  fprintf (dump_file,
+		   "   - Zero max_size. Ignoring\n");
+	return false;
+      }
+    gcc_checking_assert (!known_size_p (a.max_size)
+			 || !known_le (a.max_size, 0));
+
     /* No useful information tracked; collapse everything.  */
     if (!base && !ref && !a.useful_p ())
       {
@@ -911,10 +494,11 @@ struct GTY((user)) modref_tree
  }
 
   /* Merge OTHER into the tree.
-     PARM_MAP, if non-NULL, maps parm indexes of callee to caller.  -2 is used
-     to signalize that parameter is local and does not need to be tracked.
+     PARM_MAP, if non-NULL, maps parm indexes of callee to caller.
+     Similar CHAIN_MAP, if non-NULL, maps static chain of callee to caller.
      Return true if something has changed.  */
   bool merge (modref_tree <T> *other, vec <modref_parm_map> *parm_map,
+	      modref_parm_map *static_chain_map,
 	      bool record_accesses)
   {
     if (!other || every_base)
@@ -968,21 +552,21 @@ struct GTY((user)) modref_tree
 		  {
 		    modref_access_node a = *access_node;
 
-		    if (a.parm_index != -1 && parm_map)
+		    if (a.parm_index != MODREF_UNKNOWN_PARM && parm_map)
 		      {
 			if (a.parm_index >= (int)parm_map->length ())
-			  a.parm_index = -1;
-			else if ((*parm_map) [a.parm_index].parm_index == -2)
-			  continue;
+			  a.parm_index = MODREF_UNKNOWN_PARM;
 			else
 			  {
-			    a.parm_offset
-				 += (*parm_map) [a.parm_index].parm_offset;
-			    a.parm_offset_known
-				 &= (*parm_map)
-					 [a.parm_index].parm_offset_known;
-			    a.parm_index
-				 = (*parm_map) [a.parm_index].parm_index;
+			    modref_parm_map &m
+				    = a.parm_index == MODREF_STATIC_CHAIN_PARM
+				      ? *static_chain_map
+				      : (*parm_map) [a.parm_index];
+			    if (m.parm_index == MODREF_LOCAL_MEMORY_PARM)
+			      continue;
+			    a.parm_offset += m.parm_offset;
+			    a.parm_offset_known &= m.parm_offset_known;
+			    a.parm_index = m.parm_index;
 			  }
 		      }
 		    changed |= insert (base_node->base, ref_node->ref, a,
@@ -998,7 +582,7 @@ struct GTY((user)) modref_tree
   /* Copy OTHER to THIS.  */
   void copy_from (modref_tree <T> *other)
   {
-    merge (other, NULL, false);
+    merge (other, NULL, NULL, false);
   }
 
   /* Search BASE in tree; return NULL if failed.  */
@@ -1030,7 +614,7 @@ struct GTY((user)) modref_tree
 	    if (ref_node->every_access)
 	      return true;
 	    FOR_EACH_VEC_SAFE_ELT (ref_node->accesses, k, access_node)
-	      if (access_node->parm_index < 0)
+	      if (access_node->parm_index == MODREF_UNKNOWN_PARM)
 		return true;
 	  }
       }
@@ -1087,12 +671,12 @@ struct GTY((user)) modref_tree
 	    size_t k;
 	    modref_access_node *access_node;
 	    FOR_EACH_VEC_SAFE_ELT (ref_node->accesses, k, access_node)
-	      if (access_node->parm_index > 0)
+	      if (access_node->parm_index >= 0)
 		{
 		  if (access_node->parm_index < (int)map->length ())
 		    access_node->parm_index = (*map)[access_node->parm_index];
 		  else
-		    access_node->parm_index = -1;
+		    access_node->parm_index = MODREF_UNKNOWN_PARM;
 		}
 	  }
       }
diff --git a/gcc/ipa-modref.c b/gcc/ipa-modref.c
index 0bbec8d..5783430 100644
--- a/gcc/ipa-modref.c
+++ b/gcc/ipa-modref.c
@@ -86,6 +86,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "stringpool.h"
 #include "tree-ssanames.h"
 #include "attribs.h"
+#include "tree-cfg.h"
+#include "tree-eh.h"
 
 
 namespace {
@@ -133,7 +135,7 @@ static fnspec_summaries_t *fnspec_summaries = NULL;
 struct escape_entry
 {
   /* Parameter that escapes at a given call.  */
-  unsigned int parm_index;
+  int parm_index;
   /* Argument it escapes to.  */
   unsigned int arg;
   /* Minimal flags known about the argument.  */
@@ -147,20 +149,24 @@ struct escape_entry
 static void
 dump_eaf_flags (FILE *out, int flags, bool newline = true)
 {
-  if (flags & EAF_DIRECT)
-    fprintf (out, " direct");
-  if (flags & EAF_NOCLOBBER)
-    fprintf (out, " noclobber");
-  if (flags & EAF_NOESCAPE)
-    fprintf (out, " noescape");
-  if (flags & EAF_NODIRECTESCAPE)
-    fprintf (out, " nodirectescape");
   if (flags & EAF_UNUSED)
     fprintf (out, " unused");
-  if (flags & EAF_NOT_RETURNED)
-    fprintf (out, " not_returned");
-  if (flags & EAF_NOREAD)
-    fprintf (out, " noread");
+  if (flags & EAF_NO_DIRECT_CLOBBER)
+    fprintf (out, " no_direct_clobber");
+  if (flags & EAF_NO_INDIRECT_CLOBBER)
+    fprintf (out, " no_indirect_clobber");
+  if (flags & EAF_NO_DIRECT_ESCAPE)
+    fprintf (out, " no_direct_escape");
+  if (flags & EAF_NO_INDIRECT_ESCAPE)
+    fprintf (out, " no_indirect_escape");
+  if (flags & EAF_NOT_RETURNED_DIRECTLY)
+    fprintf (out, " not_returned_directly");
+  if (flags & EAF_NOT_RETURNED_INDIRECTLY)
+    fprintf (out, " not_returned_indirectly");
+  if (flags & EAF_NO_DIRECT_READ)
+    fprintf (out, " no_direct_read");
+  if (flags & EAF_NO_INDIRECT_READ)
+    fprintf (out, " no_indirect_read");
   if (newline)
   fprintf (out, "\n");
 }
@@ -269,7 +275,10 @@ static GTY(()) fast_function_summary <modref_summary_lto *, va_gc>
 /* Summary for a single function which this pass produces.  */
 
 modref_summary::modref_summary ()
-  : loads (NULL), stores (NULL), writes_errno (NULL)
+  : loads (NULL), stores (NULL), retslot_flags (0), static_chain_flags (0),
+    writes_errno (false), side_effects (false), nondeterministic (false),
+    calls_interposable (false), global_memory_read (false),
+    global_memory_written (false), try_dse (false)
 {
 }
 
@@ -287,14 +296,12 @@ modref_summary::~modref_summary ()
 static int
 remove_useless_eaf_flags (int eaf_flags, int ecf_flags, bool returns_void)
 {
-  if (ecf_flags & ECF_NOVOPS)
-    return 0;
-  if (ecf_flags & ECF_CONST)
+  if (ecf_flags & (ECF_CONST | ECF_NOVOPS))
     eaf_flags &= ~implicit_const_eaf_flags;
   else if (ecf_flags & ECF_PURE)
     eaf_flags &= ~implicit_pure_eaf_flags;
   else if ((ecf_flags & ECF_NORETURN) || returns_void)
-    eaf_flags &= ~EAF_NOT_RETURNED;
+    eaf_flags &= ~(EAF_NOT_RETURNED_DIRECTLY | EAF_NOT_RETURNED_INDIRECTLY);
   return eaf_flags;
 }
 
@@ -315,42 +322,29 @@ eaf_flags_useful_p (vec <eaf_flags_t> &flags, int ecf_flags)
 bool
 modref_summary::useful_p (int ecf_flags, bool check_flags)
 {
-  if (ecf_flags & ECF_NOVOPS)
-    return false;
   if (arg_flags.length () && !check_flags)
     return true;
   if (check_flags && eaf_flags_useful_p (arg_flags, ecf_flags))
     return true;
   arg_flags.release ();
-  if (ecf_flags & ECF_CONST)
-    return false;
+  if (check_flags && remove_useless_eaf_flags (retslot_flags, ecf_flags, false))
+    return true;
+  if (check_flags
+      && remove_useless_eaf_flags (static_chain_flags, ecf_flags, false))
+    return true;
+  if (ecf_flags & (ECF_CONST | ECF_NOVOPS))
+    return ((!side_effects || !nondeterministic)
+	    && (ecf_flags & ECF_LOOPING_CONST_OR_PURE));
   if (loads && !loads->every_base)
     return true;
+  else
+    kills.release ();
   if (ecf_flags & ECF_PURE)
-    return false;
+    return ((!side_effects || !nondeterministic)
+	    && (ecf_flags & ECF_LOOPING_CONST_OR_PURE));
   return stores && !stores->every_base;
 }
 
-/* Return true if global memory is read
-   (that is loads summary contains global memory access).  */
-bool
-modref_summary::global_memory_read_p ()
-{
-  if (!loads)
-    return true;
-  return loads->global_access_p ();
-}
-
-/* Return true if global memory is written.  */
-bool
-modref_summary::global_memory_written_p ()
-{
-  if (!stores)
-    return true;
-  return stores->global_access_p ();
-}
-
-
 /* Single function summary used for LTO.  */
 
 typedef modref_tree <tree> modref_records_lto;
@@ -362,8 +356,14 @@ struct GTY(()) modref_summary_lto
      more verbose and thus more likely to hit the limits.  */
   modref_records_lto *loads;
   modref_records_lto *stores;
+  auto_vec<modref_access_node> GTY((skip)) kills;
   auto_vec<eaf_flags_t> GTY((skip)) arg_flags;
-  bool writes_errno;
+  eaf_flags_t retslot_flags;
+  eaf_flags_t static_chain_flags;
+  unsigned writes_errno : 1;
+  unsigned side_effects : 1;
+  unsigned nondeterministic : 1;
+  unsigned calls_interposable : 1;
 
   modref_summary_lto ();
   ~modref_summary_lto ();
@@ -374,7 +374,9 @@ struct GTY(()) modref_summary_lto
 /* Summary for a single function which this pass produces.  */
 
 modref_summary_lto::modref_summary_lto ()
-  : loads (NULL), stores (NULL), writes_errno (NULL)
+  : loads (NULL), stores (NULL), retslot_flags (0), static_chain_flags (0),
+    writes_errno (false), side_effects (false), nondeterministic (false),
+    calls_interposable (false)
 {
 }
 
@@ -393,51 +395,27 @@ modref_summary_lto::~modref_summary_lto ()
 bool
 modref_summary_lto::useful_p (int ecf_flags, bool check_flags)
 {
-  if (ecf_flags & ECF_NOVOPS)
-    return false;
   if (arg_flags.length () && !check_flags)
     return true;
   if (check_flags && eaf_flags_useful_p (arg_flags, ecf_flags))
     return true;
   arg_flags.release ();
-  if (ecf_flags & ECF_CONST)
-    return false;
+  if (check_flags && remove_useless_eaf_flags (retslot_flags, ecf_flags, false))
+    return true;
+  if (check_flags
+      && remove_useless_eaf_flags (static_chain_flags, ecf_flags, false))
+    return true;
+  if (ecf_flags & (ECF_CONST | ECF_NOVOPS))
+    return ((!side_effects || !nondeterministic)
+	    && (ecf_flags & ECF_LOOPING_CONST_OR_PURE));
   if (loads && !loads->every_base)
     return true;
   if (ecf_flags & ECF_PURE)
-    return false;
+    return ((!side_effects || !nondeterministic)
+	    && (ecf_flags & ECF_LOOPING_CONST_OR_PURE));
   return stores && !stores->every_base;
 }
 
-/* Dump A to OUT.  */
-
-static void
-dump_access (modref_access_node *a, FILE *out)
-{
-  fprintf (out, "          access:");
-  if (a->parm_index != -1)
-    {
-      fprintf (out, " Parm %i", a->parm_index);
-      if (a->parm_offset_known)
-	{
-	  fprintf (out, " param offset:");
-	  print_dec ((poly_int64_pod)a->parm_offset, out, SIGNED);
-	}
-    }
-  if (a->range_info_useful_p ())
-    {
-      fprintf (out, " offset:");
-      print_dec ((poly_int64_pod)a->offset, out, SIGNED);
-      fprintf (out, " size:");
-      print_dec ((poly_int64_pod)a->size, out, SIGNED);
-      fprintf (out, " max_size:");
-      print_dec ((poly_int64_pod)a->max_size, out, SIGNED);
-      if (a->adjustments)
-	fprintf (out, " adjusted %i times", a->adjustments);
-    }
-  fprintf (out, "\n");
-}
-
 /* Dump records TT to OUT.  */
 
 static void
@@ -473,7 +451,10 @@ dump_records (modref_records *tt, FILE *out)
 	  size_t k;
 	  modref_access_node *a;
 	  FOR_EACH_VEC_SAFE_ELT (r->accesses, k, a)
-	    dump_access (a, out);
+	    {
+	      fprintf (out, "          access:");
+	      a->dump (out);
+	    }
 	}
     }
 }
@@ -519,7 +500,10 @@ dump_lto_records (modref_records_lto *tt, FILE *out)
 	  size_t k;
 	  modref_access_node *a;
 	  FOR_EACH_VEC_SAFE_ELT (r->accesses, k, a)
-	    dump_access (a, out);
+	    {
+	      fprintf (out, "          access:");
+	      a->dump (out);
+	    }
 	}
     }
 }
@@ -597,8 +581,29 @@ modref_summary::dump (FILE *out)
       fprintf (out, "  stores:\n");
       dump_records (stores, out);
     }
+  if (kills.length ())
+    {
+      fprintf (out, "  kills:\n");
+      for (auto kill : kills)
+	{
+	  fprintf (out, "    ");
+	  kill.dump (out);
+	}
+    }
   if (writes_errno)
     fprintf (out, "  Writes errno\n");
+  if (side_effects)
+    fprintf (out, "  Side effects\n");
+  if (nondeterministic)
+    fprintf (out, "  Nondeterministic\n");
+  if (calls_interposable)
+    fprintf (out, "  Calls interposable\n");
+  if (global_memory_read)
+    fprintf (out, "  Global memory read\n");
+  if (global_memory_written)
+    fprintf (out, "  Global memory written\n");
+  if (try_dse)
+    fprintf (out, "  Try dse\n");
   if (arg_flags.length ())
     {
       for (unsigned int i = 0; i < arg_flags.length (); i++)
@@ -608,6 +613,16 @@ modref_summary::dump (FILE *out)
 	    dump_eaf_flags (out, arg_flags[i]);
 	  }
     }
+  if (retslot_flags)
+    {
+      fprintf (out, "  Retslot flags:");
+      dump_eaf_flags (out, retslot_flags);
+    }
+  if (static_chain_flags)
+    {
+      fprintf (out, "  Static chain flags:");
+      dump_eaf_flags (out, static_chain_flags);
+    }
 }
 
 /* Dump summary.  */
@@ -621,6 +636,12 @@ modref_summary_lto::dump (FILE *out)
   dump_lto_records (stores, out);
   if (writes_errno)
     fprintf (out, "  Writes errno\n");
+  if (side_effects)
+    fprintf (out, "  Side effects\n");
+  if (nondeterministic)
+    fprintf (out, "  Nondeterministic\n");
+  if (calls_interposable)
+    fprintf (out, "  Calls interposable\n");
   if (arg_flags.length ())
     {
       for (unsigned int i = 0; i < arg_flags.length (); i++)
@@ -630,6 +651,69 @@ modref_summary_lto::dump (FILE *out)
 	    dump_eaf_flags (out, arg_flags[i]);
 	  }
     }
+  if (retslot_flags)
+    {
+      fprintf (out, "  Retslot flags:");
+      dump_eaf_flags (out, retslot_flags);
+    }
+  if (static_chain_flags)
+    {
+      fprintf (out, "  Static chain flags:");
+      dump_eaf_flags (out, static_chain_flags);
+    }
+}
+
+/* Called after summary is produced and before it is used by local analysis.
+   Can be called multiple times in case summary needs to update signature.
+   FUN is decl of function summary is attached to.  */
+void
+modref_summary::finalize (tree fun)
+{
+  global_memory_read = !loads || loads->global_access_p ();
+  global_memory_written = !stores || stores->global_access_p ();
+
+  /* We can do DSE if we know function has no side effects and
+     we can analyse all stores.  Disable dse if there are too many
+     stores to try.  */
+  if (side_effects || global_memory_written || writes_errno)
+    try_dse = false;
+  else
+    {
+      try_dse = true;
+      size_t i, j, k;
+      int num_tests = 0, max_tests
+       	= opt_for_fn (fun, param_modref_max_tests);
+      modref_base_node <alias_set_type> *base_node;
+      modref_ref_node <alias_set_type> *ref_node;
+      modref_access_node *access_node;
+      FOR_EACH_VEC_SAFE_ELT (stores->bases, i, base_node)
+	{
+	  if (base_node->every_ref)
+	    {
+	      try_dse = false;
+	      break;
+	    }
+	  FOR_EACH_VEC_SAFE_ELT (base_node->refs, j, ref_node)
+	    {
+	      if (base_node->every_ref)
+		{
+		  try_dse = false;
+		  break;
+		}
+	      FOR_EACH_VEC_SAFE_ELT (ref_node->accesses, k, access_node)
+		if (num_tests++ > max_tests
+		    || !access_node->parm_offset_known)
+		  {
+		    try_dse = false;
+		    break;
+		  }
+	      if (!try_dse)
+		break;
+	    }
+	  if (!try_dse)
+	    break;
+	}
+    }
 }
 
 /* Get function summary for FUNC if it exists, return NULL otherwise.  */
@@ -656,6 +740,8 @@ get_modref_function_summary (cgraph_node *func)
   return r;
 }
 
+namespace {
+
 /* Construct modref_access_node from REF.  */
 static modref_access_node
 get_access (ao_ref *ref)
@@ -664,57 +750,57 @@ get_access (ao_ref *ref)
 
   base = ao_ref_base (ref);
   modref_access_node a = {ref->offset, ref->size, ref->max_size,
-			  0, -1, false, 0};
+			  0, MODREF_UNKNOWN_PARM, false, 0};
   if (TREE_CODE (base) == MEM_REF || TREE_CODE (base) == TARGET_MEM_REF)
     {
       tree memref = base;
       base = TREE_OPERAND (base, 0);
+
       if (TREE_CODE (base) == SSA_NAME
 	  && SSA_NAME_IS_DEFAULT_DEF (base)
 	  && TREE_CODE (SSA_NAME_VAR (base)) == PARM_DECL)
 	{
 	  a.parm_index = 0;
-	  for (tree t = DECL_ARGUMENTS (current_function_decl);
-	       t != SSA_NAME_VAR (base); t = DECL_CHAIN (t))
-	    {
-	      if (!t)
-		{
-		  a.parm_index = -1;
-		  break;
-		}
-	      a.parm_index++;
-	    }
-	  if (TREE_CODE (memref) == MEM_REF)
-	    {
-	      a.parm_offset_known
-		 = wi::to_poly_wide (TREE_OPERAND
-					 (memref, 1)).to_shwi (&a.parm_offset);
-	    }
+	  if (cfun->static_chain_decl
+	      && base == ssa_default_def (cfun, cfun->static_chain_decl))
+	    a.parm_index = MODREF_STATIC_CHAIN_PARM;
 	  else
-	    a.parm_offset_known = false;
+	    for (tree t = DECL_ARGUMENTS (current_function_decl);
+		 t != SSA_NAME_VAR (base); t = DECL_CHAIN (t))
+	      a.parm_index++;
 	}
       else
-	a.parm_index = -1;
+	a.parm_index = MODREF_UNKNOWN_PARM;
+
+      if (a.parm_index != MODREF_UNKNOWN_PARM
+	  && TREE_CODE (memref) == MEM_REF)
+	{
+	  a.parm_offset_known
+	     = wi::to_poly_wide (TREE_OPERAND
+				     (memref, 1)).to_shwi (&a.parm_offset);
+	}
+      else
+	a.parm_offset_known = false;
     }
   else
-    a.parm_index = -1;
+    a.parm_index = MODREF_UNKNOWN_PARM;
   return a;
 }
 
 /* Record access into the modref_records data structure.  */
 
 static void
-record_access (modref_records *tt, ao_ref *ref)
+record_access (modref_records *tt, ao_ref *ref, modref_access_node &a)
 {
   alias_set_type base_set = !flag_strict_aliasing ? 0
 			    : ao_ref_base_alias_set (ref);
   alias_set_type ref_set = !flag_strict_aliasing ? 0
 			    : (ao_ref_alias_set (ref));
-  modref_access_node a = get_access (ref);
   if (dump_file)
     {
-       fprintf (dump_file, "   - Recording base_set=%i ref_set=%i parm=%i\n",
-		base_set, ref_set, a.parm_index);
+       fprintf (dump_file, "   - Recording base_set=%i ref_set=%i ",
+		base_set, ref_set);
+       a.dump (dump_file);
     }
   tt->insert (base_set, ref_set, a, false);
 }
@@ -722,7 +808,7 @@ record_access (modref_records *tt, ao_ref *ref)
 /* IPA version of record_access_tree.  */
 
 static void
-record_access_lto (modref_records_lto *tt, ao_ref *ref)
+record_access_lto (modref_records_lto *tt, ao_ref *ref, modref_access_node &a)
 {
   /* get_alias_set sometimes use different type to compute the alias set
      than TREE_TYPE (base).  Do same adjustments.  */
@@ -769,7 +855,6 @@ record_access_lto (modref_records_lto *tt, ao_ref *ref)
 		       || variably_modified_type_p (ref_type, NULL_TREE)))
 	ref_type = NULL_TREE;
     }
-  modref_access_node a = get_access (ref);
   if (dump_file)
     {
       fprintf (dump_file, "   - Recording base type:");
@@ -777,9 +862,9 @@ record_access_lto (modref_records_lto *tt, ao_ref *ref)
       fprintf (dump_file, " (alias set %i) ref type:",
 	       base_type ? get_alias_set (base_type) : 0);
       print_generic_expr (dump_file, ref_type);
-      fprintf (dump_file, " (alias set %i) parm:%i\n",
-	       ref_type ? get_alias_set (ref_type) : 0,
-	       a.parm_index);
+      fprintf (dump_file, " (alias set %i) ",
+	       ref_type ? get_alias_set (ref_type) : 0);
+       a.dump (dump_file);
     }
 
   tt->insert (base_type, ref_type, a, false);
@@ -800,6 +885,20 @@ record_access_p (tree expr)
   return true;
 }
 
+/* Return true if ECF flags says that nondeterminsm can be ignored.  */
+
+static bool
+ignore_nondeterminism_p (tree caller, int flags)
+{
+  if ((flags & (ECF_CONST | ECF_PURE))
+      && !(flags & ECF_LOOPING_CONST_OR_PURE))
+    return true;
+  if ((flags & (ECF_NORETURN | ECF_NOTHROW)) == (ECF_NORETURN | ECF_NOTHROW)
+      || (!opt_for_fn (caller, flag_exceptions) && (flags & ECF_NORETURN)))
+    return true;
+  return false;
+}
+
 /* Return true if ECF flags says that return value can be ignored.  */
 
 static bool
@@ -824,12 +923,11 @@ ignore_stores_p (tree caller, int flags)
   return false;
 }
 
-/* Determine parm_map for argument I of STMT.  */
+/* Determine parm_map for argument OP.  */
 
 modref_parm_map
-parm_map_for_arg (gimple *stmt, int i)
+parm_map_for_arg (tree op)
 {
-  tree op = gimple_call_arg (stmt, i);
   bool offset_known;
   poly_int64 offset;
   struct modref_parm_map parm_map;
@@ -848,7 +946,7 @@ parm_map_for_arg (gimple *stmt, int i)
 	{
 	  if (!t)
 	    {
-	      index = -1;
+	      index = MODREF_UNKNOWN_PARM;
 	      break;
 	    }
 	  index++;
@@ -858,9 +956,9 @@ parm_map_for_arg (gimple *stmt, int i)
       parm_map.parm_offset = offset;
     }
   else if (points_to_local_or_readonly_memory_p (op))
-    parm_map.parm_index = -2;
+    parm_map.parm_index = MODREF_LOCAL_MEMORY_PARM;
   else
-    parm_map.parm_index = -1;
+    parm_map.parm_index = MODREF_UNKNOWN_PARM;
   return parm_map;
 }
 
@@ -874,20 +972,59 @@ bool
 merge_call_side_effects (modref_summary *cur_summary,
 			 gimple *stmt, modref_summary *callee_summary,
 			 bool ignore_stores, cgraph_node *callee_node,
-			 bool record_adjustments)
+			 bool record_adjustments, bool always_executed)
 {
   auto_vec <modref_parm_map, 32> parm_map;
+  modref_parm_map chain_map;
   bool changed = false;
+  int flags = gimple_call_flags (stmt);
+
+  if ((flags & (ECF_CONST | ECF_NOVOPS))
+      && !(flags & ECF_LOOPING_CONST_OR_PURE))
+    return changed;
+
+  if (!(flags & (ECF_CONST | ECF_NOVOPS | ECF_PURE))
+      || (flags & ECF_LOOPING_CONST_OR_PURE))
+    {
+      if (!cur_summary->side_effects && callee_summary->side_effects)
+	{
+	  if (dump_file)
+	    fprintf (dump_file, " - merging side effects.\n");
+	  cur_summary->side_effects = true;
+	  changed = true;
+	}
+      if (!cur_summary->nondeterministic && callee_summary->nondeterministic
+	  && !ignore_nondeterminism_p (current_function_decl, flags))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, " - merging nondeterministic.\n");
+	  cur_summary->nondeterministic = true;
+	  changed = true;
+	}
+     }
+
+  if (flags & (ECF_CONST | ECF_NOVOPS))
+    return changed;
+
+  if (!cur_summary->calls_interposable && callee_summary->calls_interposable)
+    {
+      if (dump_file)
+	fprintf (dump_file, " - merging calls interposable.\n");
+      cur_summary->calls_interposable = true;
+      changed = true;
+    }
 
   /* We can not safely optimize based on summary of callee if it does
      not always bind to current def: it is possible that memory load
      was optimized out earlier which may not happen in the interposed
      variant.  */
-  if (!callee_node->binds_to_current_def_p ())
+  if (!callee_node->binds_to_current_def_p ()
+      && !cur_summary->calls_interposable)
     {
       if (dump_file)
-	fprintf (dump_file, " - May be interposed: collapsing loads.\n");
-      cur_summary->loads->collapse ();
+	fprintf (dump_file, " - May be interposed.\n");
+      cur_summary->calls_interposable = true;
+      changed = true;
     }
 
   if (dump_file)
@@ -897,7 +1034,7 @@ merge_call_side_effects (modref_summary *cur_summary,
   parm_map.safe_grow_cleared (gimple_call_num_args (stmt), true);
   for (unsigned i = 0; i < gimple_call_num_args (stmt); i++)
     {
-      parm_map[i] = parm_map_for_arg (stmt, i);
+      parm_map[i] = parm_map_for_arg (gimple_call_arg (stmt, i));
       if (dump_file)
 	{
 	  fprintf (dump_file, " %i", parm_map[i].parm_index);
@@ -909,16 +1046,62 @@ merge_call_side_effects (modref_summary *cur_summary,
 	    }
 	}
     }
+  if (gimple_call_chain (stmt))
+    {
+      chain_map = parm_map_for_arg (gimple_call_chain (stmt));
+      if (dump_file)
+	{
+	  fprintf (dump_file, "static chain %i", chain_map.parm_index);
+	  if (chain_map.parm_offset_known)
+	    {
+	      fprintf (dump_file, " offset:");
+	      print_dec ((poly_int64_pod)chain_map.parm_offset,
+			 dump_file, SIGNED);
+	    }
+	}
+    }
   if (dump_file)
     fprintf (dump_file, "\n");
 
+  if (always_executed
+      && callee_summary->kills.length ()
+      && (!cfun->can_throw_non_call_exceptions
+	  || !stmt_could_throw_p (cfun, stmt)))
+    {
+      /* Watch for self recursive updates.  */
+      auto_vec<modref_access_node, 32> saved_kills;
+
+      saved_kills.reserve_exact (callee_summary->kills.length ());
+      saved_kills.splice (callee_summary->kills);
+      for (auto kill : saved_kills)
+	{
+	  if (kill.parm_index >= (int)parm_map.length ())
+	    continue;
+	  modref_parm_map &m
+		  = kill.parm_index == MODREF_STATIC_CHAIN_PARM
+		    ? chain_map
+		    : parm_map[kill.parm_index];
+	  if (m.parm_index == MODREF_LOCAL_MEMORY_PARM
+	      || m.parm_index == MODREF_UNKNOWN_PARM
+	      || m.parm_index == MODREF_RETSLOT_PARM
+	      || !m.parm_offset_known)
+	    continue;
+	  modref_access_node n = kill;
+	  n.parm_index = m.parm_index;
+	  n.parm_offset += m.parm_offset;
+	  if (modref_access_node::insert_kill (cur_summary->kills, n,
+					       record_adjustments))
+	    changed = true;
+	}
+    }
+
   /* Merge with callee's summary.  */
   changed |= cur_summary->loads->merge (callee_summary->loads, &parm_map,
-					record_adjustments);
+					&chain_map, record_adjustments);
   if (!ignore_stores)
     {
       changed |= cur_summary->stores->merge (callee_summary->stores,
-					     &parm_map,
+					     &parm_map, &chain_map,
 					     record_adjustments);
       if (!cur_summary->writes_errno
 	  && callee_summary->writes_errno)
@@ -1022,6 +1205,28 @@ process_fnspec (modref_summary *cur_summary,
 		gcall *call, bool ignore_stores)
 {
   attr_fnspec fnspec = gimple_call_fnspec (call);
+  int flags = gimple_call_flags (call);
+
+  if (!(flags & (ECF_CONST | ECF_NOVOPS | ECF_PURE))
+      || (flags & ECF_LOOPING_CONST_OR_PURE)
+      || (cfun->can_throw_non_call_exceptions
+	  && stmt_could_throw_p (cfun, call)))
+    {
+      if (cur_summary)
+	{
+	  cur_summary->side_effects = true;
+	  if (!ignore_nondeterminism_p (current_function_decl, flags))
+	    cur_summary->nondeterministic = true;
+	}
+      if (cur_summary_lto)
+	{
+	  cur_summary_lto->side_effects = true;
+	  if (!ignore_nondeterminism_p (current_function_decl, flags))
+	    cur_summary_lto->nondeterministic = true;
+	}
+    }
+  if (flags & (ECF_CONST | ECF_NOVOPS))
+    return true;
   if (!fnspec.known_p ())
     {
       if (dump_file && gimple_call_builtin_p (call, BUILT_IN_NORMAL))
@@ -1044,11 +1249,12 @@ process_fnspec (modref_summary *cur_summary,
 	else if (!fnspec.arg_specified_p (i)
 		 || fnspec.arg_maybe_read_p (i))
 	  {
-	    modref_parm_map map = parm_map_for_arg (call, i);
+	    modref_parm_map map = parm_map_for_arg
+					(gimple_call_arg (call, i));
 
-	    if (map.parm_index == -2)
+	    if (map.parm_index == MODREF_LOCAL_MEMORY_PARM)
 	      continue;
-	    if (map.parm_index == -1)
+	    if (map.parm_index == MODREF_UNKNOWN_PARM)
 	      {
 		collapse_loads (cur_summary, cur_summary_lto);
 		break;
@@ -1079,11 +1285,12 @@ process_fnspec (modref_summary *cur_summary,
 	else if (!fnspec.arg_specified_p (i)
 		 || fnspec.arg_maybe_written_p (i))
 	  {
-	    modref_parm_map map = parm_map_for_arg (call, i);
+	    modref_parm_map map = parm_map_for_arg
+					 (gimple_call_arg (call, i));
 
-	    if (map.parm_index == -2)
+	    if (map.parm_index == MODREF_LOCAL_MEMORY_PARM)
 	      continue;
-	    if (map.parm_index == -1)
+	    if (map.parm_index == MODREF_UNKNOWN_PARM)
 	      {
 		collapse_stores (cur_summary, cur_summary_lto);
 		break;
@@ -1117,12 +1324,14 @@ process_fnspec (modref_summary *cur_summary,
 
 static bool
 analyze_call (modref_summary *cur_summary, modref_summary_lto *cur_summary_lto,
-	      gcall *stmt, vec <gimple *> *recursive_calls)
+	      gcall *stmt, vec <gimple *> *recursive_calls,
+	      bool always_executed)
 {
   /* Check flags on the function call.  In certain cases, analysis can be
      simplified.  */
   int flags = gimple_call_flags (stmt);
-  if (flags & (ECF_CONST | ECF_NOVOPS))
+  if ((flags & (ECF_CONST | ECF_NOVOPS))
+      && !(flags & ECF_LOOPING_CONST_OR_PURE))
     {
       if (dump_file)
 	fprintf (dump_file,
@@ -1157,6 +1366,10 @@ analyze_call (modref_summary *cur_summary, modref_summary_lto *cur_summary_lto,
   if (recursive_call_p (current_function_decl, callee))
     {
       recursive_calls->safe_push (stmt);
+      if (cur_summary)
+	cur_summary->side_effects = true;
+      if (cur_summary_lto)
+	cur_summary_lto->side_effects = true;
       if (dump_file)
 	fprintf (dump_file, " - Skipping recursive call.\n");
       return true;
@@ -1167,6 +1380,20 @@ analyze_call (modref_summary *cur_summary, modref_summary_lto *cur_summary_lto,
   /* Get the function symbol and its availability.  */
   enum availability avail;
   callee_node = callee_node->function_symbol (&avail);
+  bool looping;
+  if (builtin_safe_for_const_function_p (&looping, callee))
+    {
+      if (looping)
+	{
+	  if (cur_summary)
+	    cur_summary->side_effects = true;
+	  if (cur_summary_lto)
+	    cur_summary_lto->side_effects = true;
+	}
+      if (dump_file)
+	fprintf (dump_file, " - Bulitin is safe for const.\n");
+      return true;
+    }
   if (avail <= AVAIL_INTERPOSABLE)
     {
       if (dump_file)
@@ -1185,7 +1412,7 @@ analyze_call (modref_summary *cur_summary, modref_summary_lto *cur_summary_lto,
     }
 
   merge_call_side_effects (cur_summary, stmt, callee_summary, ignore_stores,
-			   callee_node, false);
+			   callee_node, false, always_executed);
 
   return true;
 }
@@ -1196,6 +1423,7 @@ struct summary_ptrs
 {
   struct modref_summary *nolto;
   struct modref_summary_lto *lto;
+  bool always_executed;
 };
 
 /* Helper for analyze_stmt.  */
@@ -1213,23 +1441,36 @@ analyze_load (gimple *, tree, tree op, void *data)
       fprintf (dump_file, "\n");
     }
 
+  if (TREE_THIS_VOLATILE (op)
+      || (cfun->can_throw_non_call_exceptions
+	  && tree_could_throw_p (op)))
+    {
+      if (dump_file)
+	fprintf (dump_file, " (volatile or can throw; marking side effects) ");
+      if (summary)
+	summary->side_effects = summary->nondeterministic = true;
+      if (summary_lto)
+	summary_lto->side_effects = summary_lto->nondeterministic = true;
+    }
+
   if (!record_access_p (op))
     return false;
 
   ao_ref r;
   ao_ref_init (&r, op);
+  modref_access_node a = get_access (&r);
 
   if (summary)
-    record_access (summary->loads, &r);
+    record_access (summary->loads, &r, a);
   if (summary_lto)
-    record_access_lto (summary_lto->loads, &r);
+    record_access_lto (summary_lto->loads, &r, a);
   return false;
 }
 
 /* Helper for analyze_stmt.  */
 
 static bool
-analyze_store (gimple *, tree, tree op, void *data)
+analyze_store (gimple *stmt, tree, tree op, void *data)
 {
   modref_summary *summary = ((summary_ptrs *)data)->nolto;
   modref_summary_lto *summary_lto = ((summary_ptrs *)data)->lto;
@@ -1241,16 +1482,39 @@ analyze_store (gimple *, tree, tree op, void *data)
       fprintf (dump_file, "\n");
     }
 
+  if (TREE_THIS_VOLATILE (op)
+      || (cfun->can_throw_non_call_exceptions
+	  && tree_could_throw_p (op)))
+    {
+      if (dump_file)
+	fprintf (dump_file, " (volatile or can throw; marking side effects) ");
+      if (summary)
+	summary->side_effects = summary->nondeterministic = true;
+      if (summary_lto)
+	summary_lto->side_effects = summary_lto->nondeterministic = true;
+    }
+
   if (!record_access_p (op))
     return false;
 
   ao_ref r;
   ao_ref_init (&r, op);
+  modref_access_node a = get_access (&r);
 
   if (summary)
-    record_access (summary->stores, &r);
+    record_access (summary->stores, &r, a);
   if (summary_lto)
-    record_access_lto (summary_lto->stores, &r);
+    record_access_lto (summary_lto->stores, &r, a);
+  if (summary
+      && ((summary_ptrs *)data)->always_executed
+      && a.useful_for_kill_p ()
+      && (!cfun->can_throw_non_call_exceptions
+	  || !stmt_could_throw_p (cfun, stmt)))
+    {
+      if (dump_file)
+	fprintf (dump_file, "   - Recording kill\n");
+      modref_access_node::insert_kill (summary->kills, a, false);
+    }
   return false;
 }
 
@@ -1259,16 +1523,32 @@ analyze_store (gimple *, tree, tree op, void *data)
 
 static bool
 analyze_stmt (modref_summary *summary, modref_summary_lto *summary_lto,
-	      gimple *stmt, bool ipa, vec <gimple *> *recursive_calls)
+	      gimple *stmt, bool ipa, vec <gimple *> *recursive_calls,
+	      bool always_executed)
 {
   /* In general we can not ignore clobbers because they are barriers for code
      motion, however after inlining it is safe to do because local optimization
      passes do not consider clobbers from other functions.
      Similar logic is in ipa-pure-const.c.  */
   if ((ipa || cfun->after_inlining) && gimple_clobber_p (stmt))
-    return true;
+    {
+      if (summary
+	  && always_executed && record_access_p (gimple_assign_lhs (stmt)))
+	{
+	  ao_ref r;
+	  ao_ref_init (&r, gimple_assign_lhs (stmt));
+	  modref_access_node a = get_access (&r);
+	  if (a.useful_for_kill_p ())
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "   - Recording kill\n");
+	      modref_access_node::insert_kill (summary->kills, a, false);
+	    }
+	}
+      return true;
+    }
 
-  struct summary_ptrs sums = {summary, summary_lto};
+  struct summary_ptrs sums = {summary, summary_lto, always_executed};
 
   /* Analyze all loads and stores in STMT.  */
   walk_stmt_load_store_ops (stmt, &sums,
@@ -1277,6 +1557,21 @@ analyze_stmt (modref_summary *summary, modref_summary_lto *summary_lto,
   switch (gimple_code (stmt))
    {
    case GIMPLE_ASM:
+      if (gimple_asm_volatile_p (as_a <gasm *> (stmt)))
+	{
+	  if (summary)
+	    summary->side_effects = summary->nondeterministic = true;
+	  if (summary_lto)
+	    summary_lto->side_effects = summary_lto->nondeterministic = true;
+	}
+      if (cfun->can_throw_non_call_exceptions
+	  && stmt_could_throw_p (cfun, stmt))
+	{
+	  if (summary)
+	    summary->side_effects = true;
+	  if (summary_lto)
+	    summary_lto->side_effects = true;
+	}
      /* If the ASM statement does not read nor write memory, there's nothing
 	to do.  Otherwise just give up.  */
      if (!gimple_asm_clobbers_memory_p (as_a <gasm *> (stmt)))
@@ -1288,7 +1583,8 @@ analyze_stmt (modref_summary *summary, modref_summary_lto *summary_lto,
    case GIMPLE_CALL:
      if (!ipa || gimple_call_internal_p (stmt))
        return analyze_call (summary, summary_lto,
-			    as_a <gcall *> (stmt), recursive_calls);
+			    as_a <gcall *> (stmt), recursive_calls,
+			    always_executed);
      else
       {
 	attr_fnspec fnspec = gimple_call_fnspec (as_a <gcall *>(stmt));
@@ -1308,7 +1604,14 @@ analyze_stmt (modref_summary *summary, modref_summary_lto *summary_lto,
       }
      return true;
    default:
-     /* Nothing to do for other types of statements.  */
+     if (cfun->can_throw_non_call_exceptions
+	 && stmt_could_throw_p (cfun, stmt))
+	{
+	  if (summary)
+	    summary->side_effects = true;
+	  if (summary_lto)
+	    summary_lto->side_effects = true;
+	}
      return true;
    }
 }
@@ -1357,40 +1660,36 @@ memory_access_to (tree op, tree ssa_name)
 static int
 deref_flags (int flags, bool ignore_stores)
 {
-  int ret = EAF_NODIRECTESCAPE;
+  /* Dereference is also a direct read but dereferenced value does not
+     yield any other direct use.  */
+  int ret = EAF_NO_DIRECT_CLOBBER | EAF_NO_DIRECT_ESCAPE
+	    | EAF_NOT_RETURNED_DIRECTLY;
   /* If argument is unused just account for
      the read involved in dereference.  */
   if (flags & EAF_UNUSED)
-    ret |= EAF_DIRECT | EAF_NOCLOBBER | EAF_NOESCAPE | EAF_NOT_RETURNED;
+    ret |= EAF_NO_INDIRECT_READ | EAF_NO_INDIRECT_CLOBBER
+	   | EAF_NO_INDIRECT_ESCAPE;
   else
     {
-      if ((flags & EAF_NOCLOBBER) || ignore_stores)
-	ret |= EAF_NOCLOBBER;
-      if ((flags & EAF_NOESCAPE) || ignore_stores)
-	ret |= EAF_NOESCAPE;
-      /* If the value dereferenced is not used for another load or store
-	 we can still consider ARG as used only directly.
-
-	 Consider
-
-	 int
-	 test (int *a)
-	   {
-	     return *a!=0;
-	   }
-
-	*/
-      if ((flags & (EAF_NOREAD | EAF_NOT_RETURNED | EAF_NOESCAPE | EAF_DIRECT))
-	  == (EAF_NOREAD | EAF_NOT_RETURNED | EAF_NOESCAPE | EAF_DIRECT)
-	  && ((flags & EAF_NOCLOBBER) || ignore_stores))
-	ret |= EAF_DIRECT;
-      if (flags & EAF_NOT_RETURNED)
-	ret |= EAF_NOT_RETURNED;
+      /* Direct or indirect accesses leads to indirect accesses.  */
+      if (((flags & EAF_NO_DIRECT_CLOBBER)
+	   && (flags & EAF_NO_INDIRECT_CLOBBER))
+	  || ignore_stores)
+	ret |= EAF_NO_INDIRECT_CLOBBER;
+      if (((flags & EAF_NO_DIRECT_ESCAPE)
+	   && (flags & EAF_NO_INDIRECT_ESCAPE))
+	  || ignore_stores)
+	ret |= EAF_NO_INDIRECT_ESCAPE;
+      if ((flags & EAF_NO_DIRECT_READ)
+	   && (flags & EAF_NO_INDIRECT_READ))
+	ret |= EAF_NO_INDIRECT_READ;
+      if ((flags & EAF_NOT_RETURNED_DIRECTLY)
+	  && (flags & EAF_NOT_RETURNED_INDIRECTLY))
+	ret |= EAF_NOT_RETURNED_INDIRECTLY;
     }
   return ret;
 }
 
-namespace {
 
 /* Description of an escape point.  */
 
@@ -1412,14 +1711,32 @@ class modref_lattice
 public:
   /* EAF flags of the SSA name.  */
   eaf_flags_t flags;
-  /* DFS bookkkeeping: we don't do real dataflow yet.  */
+  /* Used during DFS walk to mark names where final value was determined
+     without need for dataflow.  */
   bool known;
+  /* Used during DFS walk to mark open vertices (for cycle detection).  */
   bool open;
+  /* Set during DFS walk for names that needs dataflow propagation.  */
+  bool do_dataflow;
+  /* Used during the iterative dataflow.  */
+  bool changed;
 
   /* When doing IPA analysis we can not merge in callee escape points;
      Only remember them and do the merging at IPA propagation time.  */
   vec <escape_point, va_heap, vl_ptr> escape_points;
 
+  /* Representation of a graph for dataaflow.  This graph is built on-demand
+     using modref_eaf_analysis::analyze_ssa and later solved by
+     modref_eaf_analysis::propagate.
+     Each edge represents the fact that flags of current lattice should be
+     propagated to lattice of SSA_NAME.  */
+  struct propagate_edge
+  {
+    int ssa_name;
+    bool deref;
+  };
+  vec <propagate_edge, va_heap, vl_ptr> propagate_to;
+
   void init ();
   void release ();
   bool merge (const modref_lattice &with);
@@ -1436,8 +1753,11 @@ void
 modref_lattice::init ()
 {
   /* All flags we track.  */
-  int f = EAF_DIRECT | EAF_NOCLOBBER | EAF_NOESCAPE | EAF_UNUSED
-	  | EAF_NODIRECTESCAPE | EAF_NOT_RETURNED | EAF_NOREAD;
+  int f = EAF_NO_DIRECT_CLOBBER | EAF_NO_INDIRECT_CLOBBER
+	  | EAF_NO_DIRECT_ESCAPE | EAF_NO_INDIRECT_ESCAPE
+	  | EAF_NO_DIRECT_READ | EAF_NO_INDIRECT_READ
+	  | EAF_NOT_RETURNED_DIRECTLY | EAF_NOT_RETURNED_INDIRECTLY
+	  | EAF_UNUSED;
   flags = f;
   /* Check that eaf_flags_t is wide enough to hold all flags.  */
   gcc_checking_assert (f == flags);
@@ -1450,6 +1770,7 @@ void
 modref_lattice::release ()
 {
   escape_points.release ();
+  propagate_to.release ();
 }
 
 /* Dump lattice to OUT; indent with INDENT spaces.  */
@@ -1515,10 +1836,13 @@ modref_lattice::merge (int f)
 {
   if (f & EAF_UNUSED)
     return false;
-  /* Noescape implies that value also does not escape directly.
-     Fnspec machinery does set both so compensate for this.  */
-  if (f & EAF_NOESCAPE)
-    f |= EAF_NODIRECTESCAPE;
+  /* Check that flags seems sane: if function does not read the parameter
+     it can not access it indirectly.  */
+  gcc_checking_assert (!(f & EAF_NO_DIRECT_READ)
+		       || ((f & EAF_NO_INDIRECT_READ)
+			   && (f & EAF_NO_INDIRECT_CLOBBER)
+			   && (f & EAF_NO_INDIRECT_ESCAPE)
+			   && (f & EAF_NOT_RETURNED_INDIRECTLY)));
   if ((flags & f) != flags)
     {
       flags &= f;
@@ -1540,7 +1864,7 @@ bool
 modref_lattice::merge (const modref_lattice &with)
 {
   if (!with.known)
-    return merge (0);
+    do_dataflow = true;
 
   bool changed = merge (with.flags);
 
@@ -1561,7 +1885,7 @@ bool
 modref_lattice::merge_deref (const modref_lattice &with, bool ignore_stores)
 {
   if (!with.known)
-    return merge (0);
+    do_dataflow = true;
 
   bool changed = merge (deref_flags (with.flags, ignore_stores));
 
@@ -1588,7 +1912,7 @@ modref_lattice::merge_deref (const modref_lattice &with, bool ignore_stores)
 bool
 modref_lattice::merge_direct_load ()
 {
-  return merge (~(EAF_UNUSED | EAF_NOREAD));
+  return merge (~(EAF_UNUSED | EAF_NO_DIRECT_READ));
 }
 
 /* Merge in flags for direct store.  */
@@ -1596,52 +1920,146 @@ modref_lattice::merge_direct_load ()
 bool
 modref_lattice::merge_direct_store ()
 {
-  return merge (~(EAF_UNUSED | EAF_NOCLOBBER));
+  return merge (~(EAF_UNUSED | EAF_NO_DIRECT_CLOBBER));
 }
 
-}  /* ANON namespace.  */
+/* Analyzer of EAF flags.
+   This is genrally dataflow problem over the SSA graph, however we only
+   care about flags of few selected ssa names (arguments, return slot and
+   static chain).  So we first call analyze_ssa_name on all relevant names
+   and perform a DFS walk to discover SSA names where flags needs to be
+   determined.  For acyclic graphs we try to determine final flags during
+   this walk.  Once cycles or recursin depth is met we enlist SSA names
+   for dataflow which is done by propagate call.
+
+   After propagation the flags can be obtained using get_ssa_name_flags.  */
+
+class modref_eaf_analysis
+{
+public:
+  /* Mark NAME as relevant for analysis.  */
+  void analyze_ssa_name (tree name);
+  /* Dataflow slover.  */
+  void propagate ();
+  /* Return flags computed earlier for NAME.  */
+  int get_ssa_name_flags (tree name)
+  {
+    int version = SSA_NAME_VERSION (name);
+    gcc_checking_assert (m_lattice[version].known);
+    return m_lattice[version].flags;
+  }
+  /* In IPA mode this will record all escape points
+     determined for NAME to PARM_IDNEX.  Flags are minimal
+     flags known.  */
+  void record_escape_points (tree name, int parm_index, int flags);
+  modref_eaf_analysis (bool ipa)
+  {
+    m_ipa = ipa;
+    m_depth = 0;
+    m_lattice.safe_grow_cleared (num_ssa_names, true);
+  }
+  ~modref_eaf_analysis ()
+  {
+    gcc_checking_assert (!m_depth);
+    if (m_ipa || m_names_to_propagate.length ())
+      for (unsigned int i = 0; i < num_ssa_names; i++)
+	m_lattice[i].release ();
+  }
+private:
+  /* If true, we produce analysis for IPA mode.  In this case escape points ar
+     collected.  */
+  bool m_ipa;
+  /* Depth of recursion of analyze_ssa_name.  */
+  int m_depth;
+  /* Propagation lattice for individual ssa names.  */
+  auto_vec<modref_lattice> m_lattice;
+  auto_vec<tree> m_deferred_names;
+  auto_vec<int> m_names_to_propagate;
+
+  void merge_with_ssa_name (tree dest, tree src, bool deref);
+  void merge_call_lhs_flags (gcall *call, int arg, tree name, bool direct,
+			     bool deref);
+};
 
-static void analyze_ssa_name_flags (tree name,
-				    vec<modref_lattice> &lattice,
-				    int depth, bool ipa);
 
-/* Call statements may return their parameters.  Consider argument number
+/* Call statements may return tgeir parameters.  Consider argument number
    ARG of USE_STMT and determine flags that can needs to be cleared
    in case pointer possibly indirectly references from ARG I is returned.
-   LATTICE, DEPTH and ipa are same as in analyze_ssa_name_flags.  */
+   If DIRECT is true consider direct returns and if INDIRECT consider
+   indirect returns.
+   LATTICE, DEPTH and ipa are same as in analyze_ssa_name.
+   ARG is set to -1 for static chain.  */
 
-static void
-merge_call_lhs_flags (gcall *call, int arg, int index, bool deref,
-		      vec<modref_lattice> &lattice,
-		      int depth, bool ipa)
+void
+modref_eaf_analysis::merge_call_lhs_flags (gcall *call, int arg,
+					   tree name, bool direct,
+					   bool indirect)
 {
+  int index = SSA_NAME_VERSION (name);
+
+  /* If value is not returned at all, do nothing.  */
+  if (!direct && !indirect)
+    return;
+
   /* If there is no return value, no flags are affected.  */
   if (!gimple_call_lhs (call))
     return;
 
   /* If we know that function returns given argument and it is not ARG
      we can still be happy.  */
-  int flags = gimple_call_return_flags (call);
-  if ((flags & ERF_RETURNS_ARG)
-      && (flags & ERF_RETURN_ARG_MASK) != arg)
-    return;
-
-  if (gimple_call_arg_flags (call, arg) & (EAF_NOT_RETURNED | EAF_UNUSED))
-    return;
+  if (arg >= 0)
+    {
+      int flags = gimple_call_return_flags (call);
+      if ((flags & ERF_RETURNS_ARG)
+	  && (flags & ERF_RETURN_ARG_MASK) != arg)
+	return;
+    }
 
   /* If return value is SSA name determine its flags.  */
   if (TREE_CODE (gimple_call_lhs (call)) == SSA_NAME)
     {
       tree lhs = gimple_call_lhs (call);
-      analyze_ssa_name_flags (lhs, lattice, depth + 1, ipa);
-      if (deref)
-	lattice[index].merge_deref (lattice[SSA_NAME_VERSION (lhs)], false);
-      else
-	lattice[index].merge (lattice[SSA_NAME_VERSION (lhs)]);
+      if (direct)
+	merge_with_ssa_name (name, lhs, false);
+      if (indirect)
+	merge_with_ssa_name (name, lhs, true);
     }
   /* In the case of memory store we can do nothing.  */
+  else if (!direct)
+    m_lattice[index].merge (deref_flags (0, false));
   else
-    lattice[index].merge (0);
+    m_lattice[index].merge (0);
+}
+
+/* CALL_FLAGS are EAF_FLAGS of the argument.  Turn them
+   into flags for caller, update LATTICE of corresponding
+   argument if needed.  */
+
+static int
+callee_to_caller_flags (int call_flags, bool ignore_stores,
+			modref_lattice &lattice)
+{
+  /* call_flags is about callee returning a value
+     that is not the same as caller returning it.  */
+  call_flags |= EAF_NOT_RETURNED_DIRECTLY
+		| EAF_NOT_RETURNED_INDIRECTLY;
+  if (!ignore_stores && !(call_flags & EAF_UNUSED))
+    {
+      /* If value escapes we are no longer able to track what happens
+	 with it because we can read it from the escaped location
+	 anytime.  */
+      if (!(call_flags & EAF_NO_DIRECT_ESCAPE))
+	lattice.merge (0);
+      else if (!(call_flags & EAF_NO_INDIRECT_ESCAPE))
+	lattice.merge (~(EAF_NOT_RETURNED_INDIRECTLY
+			 | EAF_NO_DIRECT_READ
+			 | EAF_NO_INDIRECT_READ
+			 | EAF_NO_INDIRECT_CLOBBER
+			 | EAF_UNUSED));
+    }
+  else
+    call_flags |= ignore_stores_eaf_flags;
+  return call_flags;
 }
 
 /* Analyze EAF flags for SSA name NAME and store result to LATTICE.
@@ -1650,67 +2068,75 @@ merge_call_lhs_flags (gcall *call, int arg, int index, bool deref,
    If IPA is true we analyze for IPA propagation (and thus call escape points
    are processed later)  */
 
-static void
-analyze_ssa_name_flags (tree name, vec<modref_lattice> &lattice, int depth,
-			bool ipa)
+void
+modref_eaf_analysis::analyze_ssa_name (tree name)
 {
   imm_use_iterator ui;
   gimple *use_stmt;
   int index = SSA_NAME_VERSION (name);
 
   /* See if value is already computed.  */
-  if (lattice[index].known)
+  if (m_lattice[index].known || m_lattice[index].do_dataflow)
    return;
-  if (lattice[index].open)
+  if (m_lattice[index].open)
     {
       if (dump_file)
 	fprintf (dump_file,
-		 "%*sGiving up on a cycle in SSA graph\n", depth * 4, "");
+		 "%*sCycle in SSA graph\n",
+		 m_depth * 4, "");
       return;
     }
-  if (depth == param_modref_max_depth)
+  /* Recursion guard.  */
+  m_lattice[index].init ();
+  if (m_depth == param_modref_max_depth)
     {
       if (dump_file)
 	fprintf (dump_file,
-		 "%*sGiving up on max depth\n", depth * 4, "");
+		 "%*sMax recursion depth reached; postponing\n",
+		 m_depth * 4, "");
+      m_deferred_names.safe_push (name);
       return;
     }
-  /* Recursion guard.  */
-  lattice[index].init ();
 
   if (dump_file)
     {
       fprintf (dump_file,
-	       "%*sAnalyzing flags of ssa name: ", depth * 4, "");
+	       "%*sAnalyzing flags of ssa name: ", m_depth * 4, "");
       print_generic_expr (dump_file, name);
       fprintf (dump_file, "\n");
     }
 
   FOR_EACH_IMM_USE_STMT (use_stmt, ui, name)
     {
-      if (lattice[index].flags == 0)
+      if (m_lattice[index].flags == 0)
 	break;
       if (is_gimple_debug (use_stmt))
 	continue;
       if (dump_file)
 	{
-	  fprintf (dump_file, "%*s  Analyzing stmt: ", depth * 4, "");
+	  fprintf (dump_file, "%*s  Analyzing stmt: ", m_depth * 4, "");
 	  print_gimple_stmt (dump_file, use_stmt, 0);
 	}
       /* If we see a direct non-debug use, clear unused bit.
 	 All dereferneces should be accounted below using deref_flags.  */
-      lattice[index].merge (~EAF_UNUSED);
+      m_lattice[index].merge (~EAF_UNUSED);
 
       /* Gimple return may load the return value.
 	 Returning name counts as an use by tree-ssa-structalias.c  */
       if (greturn *ret = dyn_cast <greturn *> (use_stmt))
 	{
-	  if (gimple_return_retval (ret) == name)
-	    lattice[index].merge (~(EAF_UNUSED | EAF_NOT_RETURNED));
+	  /* Returning through return slot is seen as memory write earlier.  */
+	  if (DECL_RESULT (current_function_decl)
+	      && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
+	    ;
+	  else if (gimple_return_retval (ret) == name)
+	    m_lattice[index].merge (~(EAF_UNUSED | EAF_NOT_RETURNED_DIRECTLY
+				      | EAF_NOT_RETURNED_DIRECTLY));
 	  else if (memory_access_to (gimple_return_retval (ret), name))
 	    {
-	      lattice[index].merge_direct_load ();
-	      lattice[index].merge (~(EAF_UNUSED | EAF_NOT_RETURNED));
+	      m_lattice[index].merge_direct_load ();
+	      m_lattice[index].merge (~(EAF_UNUSED
+					| EAF_NOT_RETURNED_INDIRECTLY));
 	    }
 	}
       /* Account for LHS store, arg loads and flags from callee function.  */
@@ -1724,12 +2150,12 @@ analyze_ssa_name_flags (tree name, vec<modref_lattice> &lattice, int depth,
 	     is on since that would allow propagation of this from -fno-ipa-pta
 	     to -fipa-pta functions.  */
 	  if (gimple_call_fn (use_stmt) == name)
-	    lattice[index].merge (~(EAF_NOCLOBBER | EAF_UNUSED));
+	    m_lattice[index].merge (~(EAF_NO_DIRECT_CLOBBER | EAF_UNUSED));
 
 	  /* Recursion would require bit of propagation; give up for now.  */
-	  if (callee && !ipa && recursive_call_p (current_function_decl,
+	  if (callee && !m_ipa && recursive_call_p (current_function_decl,
 						  callee))
-	    lattice[index].merge (0);
+	    m_lattice[index].merge (0);
 	  else
 	    {
 	      int ecf_flags = gimple_call_flags (call);
@@ -1742,66 +2168,113 @@ analyze_ssa_name_flags (tree name, vec<modref_lattice> &lattice, int depth,
 	      if (gimple_call_lhs (call)
 		  && memory_access_to (gimple_call_lhs (call), name))
 		{
-		  lattice[index].merge_direct_store ();
+		  m_lattice[index].merge_direct_store ();
 		  /* Return slot optimization passes address of
 		     LHS to callee via hidden parameter and this
 		     may make LHS to escape.  See PR 98499.  */
 		  if (gimple_call_return_slot_opt_p (call)
 		      && TREE_ADDRESSABLE (TREE_TYPE (gimple_call_lhs (call))))
-		    lattice[index].merge (EAF_NOREAD | EAF_DIRECT);
+		    {
+		      int call_flags = gimple_call_retslot_flags (call);
+		      bool isretslot = false;
+
+		      if (DECL_RESULT (current_function_decl)
+			  && DECL_BY_REFERENCE
+				(DECL_RESULT (current_function_decl)))
+			isretslot = ssa_default_def
+					 (cfun,
+					  DECL_RESULT (current_function_decl))
+					 == name;
+
+		      /* Passing returnslot to return slot is special because
+			 not_returned and escape has same meaning.
+			 However passing arg to return slot is different.  If
+			 the callee's return slot is returned it means that
+			 arg is written to itself which is an escape.
+			 Since we do not track the memory it is written to we
+			 need to give up on analysisng it.  */
+		      if (!isretslot)
+			{
+			  if (!(call_flags & (EAF_NOT_RETURNED_DIRECTLY
+					      | EAF_UNUSED)))
+			    m_lattice[index].merge (0);
+			  else gcc_checking_assert
+				(call_flags & (EAF_NOT_RETURNED_INDIRECTLY
+					       | EAF_UNUSED));
+			  call_flags = callee_to_caller_flags
+					   (call_flags, false,
+					    m_lattice[index]);
+			}
+		      m_lattice[index].merge (call_flags);
+		    }
 		}
 
-	      /* We do not track accesses to the static chain (we could)
-		 so give up.  */
 	      if (gimple_call_chain (call)
 		  && (gimple_call_chain (call) == name))
-		lattice[index].merge (0);
+		{
+		  int call_flags = gimple_call_static_chain_flags (call);
+		  if (!ignore_retval && !(call_flags & EAF_UNUSED))
+		    merge_call_lhs_flags
+			 (call, -1, name,
+			  !(call_flags & EAF_NOT_RETURNED_DIRECTLY),
+			  !(call_flags & EAF_NOT_RETURNED_INDIRECTLY));
+		  call_flags = callee_to_caller_flags
+				   (call_flags, ignore_stores,
+				    m_lattice[index]);
+		  if (!(ecf_flags & (ECF_CONST | ECF_NOVOPS)))
+		    m_lattice[index].merge (call_flags);
+		}
 
 	      /* Process internal functions and right away.  */
-	      bool record_ipa = ipa && !gimple_call_internal_p (call);
+	      bool record_ipa = m_ipa && !gimple_call_internal_p (call);
 
 	      /* Handle all function parameters.  */
 	      for (unsigned i = 0;
-		   i < gimple_call_num_args (call) && lattice[index].flags; i++)
+		   i < gimple_call_num_args (call)
+		   && m_lattice[index].flags; i++)
 		/* Name is directly passed to the callee.  */
 		if (gimple_call_arg (call, i) == name)
 		  {
+		    int call_flags = gimple_call_arg_flags (call, i);
+		    if (!ignore_retval && !(call_flags & EAF_UNUSED))
+		      merge_call_lhs_flags
+			      (call, i, name,
+			       !(call_flags & EAF_NOT_RETURNED_DIRECTLY),
+			       !(call_flags & EAF_NOT_RETURNED_INDIRECTLY));
 		    if (!(ecf_flags & (ECF_CONST | ECF_NOVOPS)))
 		      {
-			int call_flags = gimple_call_arg_flags (call, i)
-					 | EAF_NOT_RETURNED;
-			if (ignore_stores)
-			  call_flags |= ignore_stores_eaf_flags;
-
+			call_flags = callee_to_caller_flags
+					 (call_flags, ignore_stores,
+					  m_lattice[index]);
 			if (!record_ipa)
-			  lattice[index].merge (call_flags);
+			  m_lattice[index].merge (call_flags);
 			else
-			  lattice[index].add_escape_point (call, i,
-			     				   call_flags, true);
+			  m_lattice[index].add_escape_point (call, i,
+							   call_flags, true);
 		      }
-		    if (!ignore_retval)
-		      merge_call_lhs_flags (call, i, index, false,
-					    lattice, depth, ipa);
 		  }
 		/* Name is dereferenced and passed to a callee.  */
 		else if (memory_access_to (gimple_call_arg (call, i), name))
 		  {
+		    int call_flags = deref_flags
+			    (gimple_call_arg_flags (call, i), ignore_stores);
+		    if (!ignore_retval && !(call_flags & EAF_UNUSED)
+			&& !(call_flags & EAF_NOT_RETURNED_DIRECTLY)
+			&& !(call_flags & EAF_NOT_RETURNED_INDIRECTLY))
+		      merge_call_lhs_flags (call, i, name, false, true);
 		    if (ecf_flags & (ECF_CONST | ECF_NOVOPS))
-		      lattice[index].merge_direct_load ();
+		      m_lattice[index].merge_direct_load ();
 		    else
 		      {
-			int call_flags = deref_flags
-			   (gimple_call_arg_flags (call, i)
-			    | EAF_NOT_RETURNED, ignore_stores);
+			call_flags = callee_to_caller_flags
+					 (call_flags, ignore_stores,
+					  m_lattice[index]);
 			if (!record_ipa)
-			  lattice[index].merge (call_flags);
+			  m_lattice[index].merge (call_flags);
 			else
-			  lattice[index].add_escape_point (call, i,
-							   call_flags, false);
+			  m_lattice[index].add_escape_point (call, i,
+							     call_flags, false);
 		      }
-		    if (!ignore_retval)
-		      merge_call_lhs_flags (call, i, index, true,
-					    lattice, depth, ipa);
 		  }
 	    }
 	}
@@ -1816,18 +2289,16 @@ analyze_ssa_name_flags (tree name, vec<modref_lattice> &lattice, int depth,
 		 We do not track memory locations, so assume that value
 		 is used arbitrarily.  */
 	      if (memory_access_to (gimple_assign_rhs1 (assign), name))
-		lattice[index].merge (0);
+		m_lattice[index].merge (deref_flags (0, false));
 	      /* Handle *name = *exp.  */
 	      else if (memory_access_to (gimple_assign_lhs (assign), name))
-		lattice[index].merge_direct_store ();
+		m_lattice[index].merge_direct_store ();
 	    }
 	  /* Handle lhs = *name.  */
 	  else if (memory_access_to (gimple_assign_rhs1 (assign), name))
 	    {
 	      tree lhs = gimple_assign_lhs (assign);
-	      analyze_ssa_name_flags (lhs, lattice, depth + 1, ipa);
-	      lattice[index].merge_deref (lattice[SSA_NAME_VERSION (lhs)],
-					  false);
+	      merge_with_ssa_name (name, lhs, true);
 	    }
 	}
       else if (gimple_store_p (use_stmt))
@@ -1839,8 +2310,8 @@ analyze_ssa_name_flags (tree name, vec<modref_lattice> &lattice, int depth,
 	    {
 	      if (dump_file)
 		fprintf (dump_file, "%*s  ssa name saved to memory\n",
-			 depth * 4, "");
-	      lattice[index].merge (0);
+			 m_depth * 4, "");
+	      m_lattice[index].merge (0);
 	    }
 	  /* Handle *name = exp.  */
 	  else if (assign
@@ -1849,17 +2320,17 @@ analyze_ssa_name_flags (tree name, vec<modref_lattice> &lattice, int depth,
 	      /* In general we can not ignore clobbers because they are
 		 barriers for code motion, however after inlining it is safe to
 		 do because local optimization passes do not consider clobbers
-		 from other functions.  Similar logic is in ipa-pure-const.c.  */
+		 from other functions.
+		 Similar logic is in ipa-pure-const.c.  */
 	      if (!cfun->after_inlining || !gimple_clobber_p (assign))
-		lattice[index].merge_direct_store ();
+		m_lattice[index].merge_direct_store ();
 	    }
 	  /* ASM statements etc.  */
 	  else if (!assign)
 	    {
 	      if (dump_file)
-		fprintf (dump_file, "%*s  Unhandled store\n",
-			 depth * 4, "");
-	      lattice[index].merge (0);
+		fprintf (dump_file, "%*s  Unhandled store\n", m_depth * 4, "");
+	      m_lattice[index].merge (0);
 	    }
 	}
       else if (gassign *assign = dyn_cast <gassign *> (use_stmt))
@@ -1874,15 +2345,13 @@ analyze_ssa_name_flags (tree name, vec<modref_lattice> &lattice, int depth,
 		  || gimple_assign_rhs1 (assign) == name))
 	    {
 	      tree lhs = gimple_assign_lhs (assign);
-	      analyze_ssa_name_flags (lhs, lattice, depth + 1, ipa);
-	      lattice[index].merge (lattice[SSA_NAME_VERSION (lhs)]);
+	      merge_with_ssa_name (name, lhs, false);
 	    }
 	}
       else if (gphi *phi = dyn_cast <gphi *> (use_stmt))
 	{
 	  tree result = gimple_phi_result (phi);
-	  analyze_ssa_name_flags (result, lattice, depth + 1, ipa);
-	  lattice[index].merge (lattice[SSA_NAME_VERSION (result)]);
+	  merge_with_ssa_name (name, result, false);
 	}
       /* Conditions are not considered escape points
 	 by tree-ssa-structalias.  */
@@ -1891,51 +2360,286 @@ analyze_ssa_name_flags (tree name, vec<modref_lattice> &lattice, int depth,
       else
 	{
 	  if (dump_file)
-	    fprintf (dump_file, "%*s  Unhandled stmt\n", depth * 4, "");
-	  lattice[index].merge (0);
+	    fprintf (dump_file, "%*s  Unhandled stmt\n", m_depth * 4, "");
+	  m_lattice[index].merge (0);
 	}
 
       if (dump_file)
 	{
-	  fprintf (dump_file, "%*s  current flags of ", depth * 4, "");
+	  fprintf (dump_file, "%*s  current flags of ", m_depth * 4, "");
 	  print_generic_expr (dump_file, name);
-	  lattice[index].dump (dump_file, depth * 4 + 4);
+	  m_lattice[index].dump (dump_file, m_depth * 4 + 4);
 	}
     }
   if (dump_file)
     {
-      fprintf (dump_file, "%*sflags of ssa name ", depth * 4, "");
+      fprintf (dump_file, "%*sflags of ssa name ", m_depth * 4, "");
       print_generic_expr (dump_file, name);
-      lattice[index].dump (dump_file, depth * 4 + 2);
+      m_lattice[index].dump (dump_file, m_depth * 4 + 2);
+    }
+  m_lattice[index].open = false;
+  if (!m_lattice[index].do_dataflow)
+    m_lattice[index].known = true;
+}
+
+/* Propagate info from SRC to DEST.  If DEREF it true, assume that SRC
+   is dereferenced.  */
+
+void
+modref_eaf_analysis::merge_with_ssa_name (tree dest, tree src, bool deref)
+{
+  int index = SSA_NAME_VERSION (dest);
+  int src_index = SSA_NAME_VERSION (src);
+
+  /* Merging lattice with itself is a no-op.  */
+  if (!deref && src == dest)
+    return;
+
+  m_depth++;
+  analyze_ssa_name (src);
+  m_depth--;
+  if (deref)
+    m_lattice[index].merge_deref (m_lattice[src_index], false);
+  else
+    m_lattice[index].merge (m_lattice[src_index]);
+
+  /* If we failed to produce final solution add an edge to the dataflow
+     graph.  */
+  if (!m_lattice[src_index].known)
+    {
+      modref_lattice::propagate_edge e = {index, deref};
+
+      if (!m_lattice[src_index].propagate_to.length ())
+	m_names_to_propagate.safe_push (src_index);
+      m_lattice[src_index].propagate_to.safe_push (e);
+      m_lattice[src_index].changed = true;
+      m_lattice[src_index].do_dataflow = true;
+      if (dump_file)
+	fprintf (dump_file,
+		 "%*sWill propgate from ssa_name %i to %i%s\n",
+		 m_depth * 4 + 4,
+		 "", src_index, index, deref ? " (deref)" : "");
+    }
+}
+
+/* In the case we deferred some SSA names, reprocess them.  In the case some
+   dataflow edges were introduced, do the actual iterative dataflow.  */
+
+void
+modref_eaf_analysis::propagate ()
+{
+  int iterations = 0;
+  size_t i;
+  int index;
+  bool changed = true;
+
+  while (m_deferred_names.length ())
+    {
+      tree name = m_deferred_names.pop ();
+      m_lattice[SSA_NAME_VERSION (name)].open = false;
+      if (dump_file)
+	fprintf (dump_file, "Analyzing deferred SSA name\n");
+      analyze_ssa_name (name);
+    }
+
+  if (!m_names_to_propagate.length ())
+    return;
+  if (dump_file)
+    fprintf (dump_file, "Propagating EAF flags\n");
+
+  /* Compute reverse postorder.  */
+  auto_vec <int> rpo;
+  struct stack_entry
+  {
+    int name;
+    unsigned pos;
+  };
+  auto_vec <struct stack_entry> stack;
+  int pos = m_names_to_propagate.length () - 1;
+
+  rpo.safe_grow (m_names_to_propagate.length (), true);
+  stack.reserve_exact (m_names_to_propagate.length ());
+
+  /* We reuse known flag for RPO DFS walk bookeeping.  */
+  if (flag_checking)
+    FOR_EACH_VEC_ELT (m_names_to_propagate, i, index)
+      gcc_assert (!m_lattice[index].known && m_lattice[index].changed);
+
+  FOR_EACH_VEC_ELT (m_names_to_propagate, i, index)
+    {
+      if (!m_lattice[index].known)
+	{
+	  stack_entry e = {index, 0};
+
+	  stack.quick_push (e);
+	  m_lattice[index].known = true;
+	}
+      while (stack.length ())
+	{
+	  bool found = false;
+	  int index1 = stack.last ().name;
+
+	  while (stack.last ().pos < m_lattice[index1].propagate_to.length ())
+	    {
+	      int index2 = m_lattice[index1]
+		      .propagate_to[stack.last ().pos].ssa_name;
+
+	      stack.last ().pos++;
+	      if (!m_lattice[index2].known
+		  && m_lattice[index2].propagate_to.length ())
+		{
+		  stack_entry e = {index2, 0};
+
+		  stack.quick_push (e);
+		  m_lattice[index2].known = true;
+		  found = true;
+		  break;
+		}
+	    }
+	  if (!found
+	      && stack.last ().pos == m_lattice[index1].propagate_to.length ())
+	    {
+	      rpo[pos--] = index1;
+	      stack.pop ();
+	    }
+	}
+    }
+
+  /* Perform itrative dataflow.  */
+  while (changed)
+    {
+      changed = false;
+      iterations++;
+      if (dump_file)
+	fprintf (dump_file, " iteration %i\n", iterations);
+      FOR_EACH_VEC_ELT (rpo, i, index)
+	{
+	  if (m_lattice[index].changed)
+	    {
+	      size_t j;
+
+	      m_lattice[index].changed = false;
+	      if (dump_file)
+		fprintf (dump_file, "  Visiting ssa name %i\n", index);
+	      for (j = 0; j < m_lattice[index].propagate_to.length (); j++)
+		{
+		  bool ch;
+		  int target = m_lattice[index].propagate_to[j].ssa_name;
+		  bool deref = m_lattice[index].propagate_to[j].deref;
+
+		  if (dump_file)
+		    fprintf (dump_file, "   Propagating flags of ssa name"
+			     " %i to %i%s\n",
+			     index, target, deref ? " (deref)" : "");
+		  m_lattice[target].known = true;
+		  if (!m_lattice[index].propagate_to[j].deref)
+		    ch = m_lattice[target].merge (m_lattice[index]);
+		  else
+		    ch = m_lattice[target].merge_deref (m_lattice[index],
+							false);
+		  if (!ch)
+		    continue;
+		  if (dump_file)
+		    {
+		      fprintf (dump_file, "   New lattice: ");
+		      m_lattice[target].dump (dump_file);
+		    }
+		  changed = true;
+		  m_lattice[target].changed = true;
+		}
+	    }
+	}
+    }
+  if (dump_file)
+    fprintf (dump_file, "EAF flags propagated in %i iterations\n", iterations);
+}
+
+/* Record escape points of PARM_INDEX according to LATTICE.  */
+
+void
+modref_eaf_analysis::record_escape_points (tree name, int parm_index, int flags)
+{
+  modref_lattice &lattice = m_lattice[SSA_NAME_VERSION (name)];
+
+  if (lattice.escape_points.length ())
+    {
+      escape_point *ep;
+      unsigned int ip;
+      cgraph_node *node = cgraph_node::get (current_function_decl);
+
+      gcc_assert (m_ipa);
+      FOR_EACH_VEC_ELT (lattice.escape_points, ip, ep)
+	if ((ep->min_flags & flags) != flags)
+	  {
+	    cgraph_edge *e = node->get_edge (ep->call);
+	    struct escape_entry ee = {parm_index, ep->arg,
+				      ep->min_flags, ep->direct};
+
+	    escape_summaries->get_create (e)->esc.safe_push (ee);
+	  }
     }
-  lattice[index].open = false;
-  lattice[index].known = true;
 }
 
-/* Determine EAF flags for function parameters.  */
+/* Determine EAF flags for function parameters
+   and fill in SUMMARY/SUMMARY_LTO.  If IPA is true work in IPA mode
+   where we also collect scape points.
+   PAST_FLAGS, PAST_RETSLOT_FLAGS, PAST_STATIC_CHAIN_FLAGS can be
+   used to preserve flags from prevoius (IPA) run for cases where
+   late optimizations changed code in a way we can no longer analyze
+   it easily.  */
 
 static void
 analyze_parms (modref_summary *summary, modref_summary_lto *summary_lto,
-	       bool ipa)
+	       bool ipa, vec<eaf_flags_t> &past_flags,
+	       int past_retslot_flags, int past_static_chain_flags)
 {
   unsigned int parm_index = 0;
   unsigned int count = 0;
   int ecf_flags = flags_from_decl_or_type (current_function_decl);
+  tree retslot = NULL;
+  tree static_chain = NULL;
 
-  /* For novops functions we have nothing to gain by EAF flags.  */
-  if (ecf_flags & ECF_NOVOPS)
-    return;
+  /* If there is return slot, look up its SSA name.  */
+  if (DECL_RESULT (current_function_decl)
+      && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
+    retslot = ssa_default_def (cfun, DECL_RESULT (current_function_decl));
+  if (cfun->static_chain_decl)
+    static_chain = ssa_default_def (cfun, cfun->static_chain_decl);
 
   for (tree parm = DECL_ARGUMENTS (current_function_decl); parm;
        parm = TREE_CHAIN (parm))
     count++;
 
-  if (!count)
+  if (!count && !retslot && !static_chain)
     return;
 
-  auto_vec<modref_lattice> lattice;
-  lattice.safe_grow_cleared (num_ssa_names, true);
+  modref_eaf_analysis eaf_analysis (ipa);
+
+  /* Determine all SSA names we need to know flags for.  */
+  for (tree parm = DECL_ARGUMENTS (current_function_decl); parm;
+       parm = TREE_CHAIN (parm))
+    {
+      tree name = ssa_default_def (cfun, parm);
+      if (name)
+	eaf_analysis.analyze_ssa_name (name);
+    }
+  if (retslot)
+    eaf_analysis.analyze_ssa_name (retslot);
+  if (static_chain)
+    eaf_analysis.analyze_ssa_name (static_chain);
+
+  /* Do the dataflow.  */
+  eaf_analysis.propagate ();
 
+  tree attr = lookup_attribute ("fn spec",
+				TYPE_ATTRIBUTES
+				  (TREE_TYPE (current_function_decl)));
+  attr_fnspec fnspec (attr
+		      ? TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr)))
+		      : "");
+
+
+  /* Store results to summaries.  */
   for (tree parm = DECL_ARGUMENTS (current_function_decl); parm; parm_index++,
        parm = TREE_CHAIN (parm))
     {
@@ -1960,8 +2664,19 @@ analyze_parms (modref_summary *summary, modref_summary_lto *summary_lto,
 	    }
 	  continue;
 	}
-      analyze_ssa_name_flags (name, lattice, 0, ipa);
-      int flags = lattice[SSA_NAME_VERSION (name)].flags;
+      int flags = eaf_analysis.get_ssa_name_flags (name);
+      int attr_flags = fnspec.arg_eaf_flags (parm_index);
+
+      if (dump_file && (flags | attr_flags) != flags && !(flags & EAF_UNUSED))
+	{
+	  fprintf (dump_file,
+		   "  Flags for param %i combined with fnspec flags:",
+		   (int)parm_index);
+	  dump_eaf_flags (dump_file, attr_flags, false);
+	  fprintf (dump_file, " determined: ");
+	  dump_eaf_flags (dump_file, flags, true);
+	}
+      flags |= attr_flags;
 
       /* Eliminate useless flags so we do not end up storing unnecessary
 	 summaries.  */
@@ -1969,6 +2684,25 @@ analyze_parms (modref_summary *summary, modref_summary_lto *summary_lto,
       flags = remove_useless_eaf_flags
 		 (flags, ecf_flags,
 		  VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))));
+      if (past_flags.length () > parm_index)
+	{
+	  int past = past_flags[parm_index];
+	  past = remove_useless_eaf_flags
+		     (past, ecf_flags,
+		      VOID_TYPE_P (TREE_TYPE
+			  (TREE_TYPE (current_function_decl))));
+	  if (dump_file && (flags | past) != flags && !(flags & EAF_UNUSED))
+	    {
+	      fprintf (dump_file,
+		       "  Flags for param %i combined with IPA pass:",
+		       (int)parm_index);
+	      dump_eaf_flags (dump_file, past, false);
+	      fprintf (dump_file, " determined: ");
+	      dump_eaf_flags (dump_file, flags, true);
+	    }
+	  if (!(flags & EAF_UNUSED))
+	    flags |= past;
+	}
 
       if (flags)
 	{
@@ -1984,37 +2718,80 @@ analyze_parms (modref_summary *summary, modref_summary_lto *summary_lto,
 		summary_lto->arg_flags.safe_grow_cleared (count, true);
 	      summary_lto->arg_flags[parm_index] = flags;
 	    }
-	  if (lattice[SSA_NAME_VERSION (name)].escape_points.length ())
-	    {
-	      escape_point *ep;
-	      unsigned int ip;
-	      cgraph_node *node = cgraph_node::get (current_function_decl);
-
-	      gcc_checking_assert (ipa);
-	      FOR_EACH_VEC_ELT
-		  (lattice[SSA_NAME_VERSION (name)].escape_points, ip, ep)
-		if ((ep->min_flags & flags) != flags)
-		  {
-		    cgraph_edge *e = node->get_edge (ep->call);
-		    struct escape_entry ee = {parm_index, ep->arg,
-		     			      ep->min_flags, ep->direct};
-
-		    escape_summaries->get_create (e)->esc.safe_push (ee);
-		  }
-	    }
+	  eaf_analysis.record_escape_points (name, parm_index, flags);
+	}
+    }
+  if (retslot)
+    {
+      int flags = eaf_analysis.get_ssa_name_flags (retslot);
+      int past = past_retslot_flags;
+
+      flags = remove_useless_eaf_flags (flags, ecf_flags, false);
+      past = remove_useless_eaf_flags
+		 (past, ecf_flags,
+		  VOID_TYPE_P (TREE_TYPE
+		      (TREE_TYPE (current_function_decl))));
+      if (dump_file && (flags | past) != flags && !(flags & EAF_UNUSED))
+	{
+	  fprintf (dump_file,
+		   "  Retslot flags combined with IPA pass:");
+	  dump_eaf_flags (dump_file, past, false);
+	  fprintf (dump_file, " determined: ");
+	  dump_eaf_flags (dump_file, flags, true);
+	}
+      if (!(flags & EAF_UNUSED))
+	flags |= past;
+      if (flags)
+	{
+	  if (summary)
+	    summary->retslot_flags = flags;
+	  if (summary_lto)
+	    summary_lto->retslot_flags = flags;
+	  eaf_analysis.record_escape_points (retslot,
+					     MODREF_RETSLOT_PARM, flags);
+	}
+    }
+  if (static_chain)
+    {
+      int flags = eaf_analysis.get_ssa_name_flags (static_chain);
+      int past = past_static_chain_flags;
+
+      flags = remove_useless_eaf_flags (flags, ecf_flags, false);
+      past = remove_useless_eaf_flags
+		 (past, ecf_flags,
+		  VOID_TYPE_P (TREE_TYPE
+		      (TREE_TYPE (current_function_decl))));
+      if (dump_file && (flags | past) != flags && !(flags & EAF_UNUSED))
+	{
+	  fprintf (dump_file,
+		   "  Static chain flags combined with IPA pass:");
+	  dump_eaf_flags (dump_file, past, false);
+	  fprintf (dump_file, " determined: ");
+	  dump_eaf_flags (dump_file, flags, true);
+	}
+      if (!(flags & EAF_UNUSED))
+	flags |= past;
+      if (flags)
+	{
+	  if (summary)
+	    summary->static_chain_flags = flags;
+	  if (summary_lto)
+	    summary_lto->static_chain_flags = flags;
+	  eaf_analysis.record_escape_points (static_chain,
+					     MODREF_STATIC_CHAIN_PARM,
+					     flags);
 	}
     }
-  if (ipa)
-    for (unsigned int i = 0; i < num_ssa_names; i++)
-      lattice[i].release ();
 }
 
 /* Analyze function F.  IPA indicates whether we're running in local mode
-   (false) or the IPA mode (true).  */
+   (false) or the IPA mode (true).
+   Return true if fixup cfg is needed after the pass.  */
 
-static void
+static bool
 analyze_function (function *f, bool ipa)
 {
+  bool fixup_cfg = false;
   if (dump_file)
     fprintf (dump_file, "modref analyzing '%s' (ipa=%i)%s%s\n",
 	     function_name (f), ipa,
@@ -2024,7 +2801,7 @@ analyze_function (function *f, bool ipa)
   /* Don't analyze this function if it's compiled with -fno-strict-aliasing.  */
   if (!flag_ipa_modref
       || lookup_attribute ("noipa", DECL_ATTRIBUTES (current_function_decl)))
-    return;
+    return false;
 
   /* Compute no-LTO summaries when local optimization is going to happen.  */
   bool nolto = (!ipa || ((!flag_lto || flag_fat_lto_objects) && !in_lto_p)
@@ -2039,6 +2816,11 @@ analyze_function (function *f, bool ipa)
   modref_summary *summary = NULL;
   modref_summary_lto *summary_lto = NULL;
 
+  bool past_flags_known = false;
+  auto_vec <eaf_flags_t> past_flags;
+  int past_retslot_flags = 0;
+  int past_static_chain_flags = 0;
+
   /* Initialize the summary.
      If we run in local mode there is possibly pre-existing summary from
      IPA pass.  Dump it so it is easy to compare if mod-ref info has
@@ -2058,6 +2840,11 @@ analyze_function (function *f, bool ipa)
 	      fprintf (dump_file, "Past summary:\n");
 	      optimization_summaries->get
 		 (cgraph_node::get (f->decl))->dump (dump_file);
+	      past_flags.reserve_exact (summary->arg_flags.length ());
+	      past_flags.splice (summary->arg_flags);
+	      past_retslot_flags = summary->retslot_flags;
+	      past_static_chain_flags = summary->static_chain_flags;
+	      past_flags_known = true;
 	    }
 	  optimization_summaries->remove (cgraph_node::get (f->decl));
 	}
@@ -2104,6 +2891,9 @@ analyze_function (function *f, bool ipa)
 						    param_modref_max_refs,
 						    param_modref_max_accesses);
       summary->writes_errno = false;
+      summary->side_effects = false;
+      summary->nondeterministic = false;
+      summary->calls_interposable = false;
     }
   if (lto)
     {
@@ -2118,9 +2908,13 @@ analyze_function (function *f, bool ipa)
 				  param_modref_max_refs,
 				  param_modref_max_accesses);
       summary_lto->writes_errno = false;
+      summary_lto->side_effects = false;
+      summary_lto->nondeterministic = false;
+      summary_lto->calls_interposable = false;
     }
 
-  analyze_parms (summary, summary_lto, ipa);
+  analyze_parms (summary, summary_lto, ipa,
+		 past_flags, past_retslot_flags, past_static_chain_flags);
 
   int ecf_flags = flags_from_decl_or_type (current_function_decl);
   auto_vec <gimple *, 32> recursive_calls;
@@ -2132,11 +2926,15 @@ analyze_function (function *f, bool ipa)
   FOR_EACH_BB_FN (bb, f)
     {
       gimple_stmt_iterator si;
+      bool always_executed
+	      = bb == single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun))->dest;
+
       for (si = gsi_start_nondebug_after_labels_bb (bb);
 	   !gsi_end_p (si); gsi_next_nondebug (&si))
 	{
 	  if (!analyze_stmt (summary, summary_lto,
-			     gsi_stmt (si), ipa, &recursive_calls)
+			     gsi_stmt (si), ipa, &recursive_calls,
+			     always_executed)
 	      || ((!summary || !summary->useful_p (ecf_flags, false))
 		  && (!summary_lto
 		      || !summary_lto->useful_p (ecf_flags, false))))
@@ -2145,6 +2943,9 @@ analyze_function (function *f, bool ipa)
 	      collapse_stores (summary, summary_lto);
 	      break;
 	    }
+	  if (always_executed
+	      && stmt_can_throw_external (cfun, gsi_stmt (si)))
+	    always_executed = false;
 	}
     }
 
@@ -2164,16 +2965,37 @@ analyze_function (function *f, bool ipa)
 			   ignore_stores_p (current_function_decl,
 					    gimple_call_flags
 						 (recursive_calls[i])),
-			   fnode, !first);
+			   fnode, !first, false);
 	      if (!summary->useful_p (ecf_flags, false))
 		{
 		  remove_summary (lto, nolto, ipa);
-		  return;
+		  return false;
 		}
 	    }
 	  first = false;
 	}
     }
+  if (summary && !summary->side_effects && !finite_function_p ())
+    summary->side_effects = true;
+  if (summary_lto && !summary_lto->side_effects && !finite_function_p ())
+    summary_lto->side_effects = true;
+
+  if (!ipa && flag_ipa_pure_const)
+    {
+      if (!summary->stores->every_base && !summary->stores->bases
+	  && !summary->nondeterministic)
+	{
+	  if (!summary->loads->every_base && !summary->loads->bases
+	      && !summary->calls_interposable)
+	    fixup_cfg = ipa_make_function_const
+		   (cgraph_node::get (current_function_decl),
+		    summary->side_effects, true);
+	  else
+	    fixup_cfg = ipa_make_function_pure
+		   (cgraph_node::get (current_function_decl),
+		    summary->side_effects, true);
+	}
+    }
   if (summary && !summary->useful_p (ecf_flags))
     {
       if (!ipa)
@@ -2182,11 +3004,14 @@ analyze_function (function *f, bool ipa)
 	summaries->remove (fnode);
       summary = NULL;
     }
+  if (summary)
+    summary->finalize (current_function_decl);
   if (summary_lto && !summary_lto->useful_p (ecf_flags))
     {
       summaries_lto->remove (fnode);
       summary_lto = NULL;
     }
+
   if (ipa && !summary && !summary_lto)
     remove_modref_edge_summaries (fnode);
 
@@ -2198,7 +3023,104 @@ analyze_function (function *f, bool ipa)
       if (summary_lto)
 	summary_lto->dump (dump_file);
       dump_modref_edge_summaries (dump_file, fnode, 2);
+      /* To simplify debugging, compare IPA and local solutions.  */
+      if (past_flags_known && summary)
+	{
+	  size_t len = summary->arg_flags.length ();
+
+	  if (past_flags.length () > len)
+	    len = past_flags.length ();
+	  for (size_t i = 0; i < len; i++)
+	    {
+	      int old_flags = i < past_flags.length () ? past_flags[i] : 0;
+	      int new_flags = i < summary->arg_flags.length ()
+			      ? summary->arg_flags[i] : 0;
+	      old_flags = remove_useless_eaf_flags
+		(old_flags, flags_from_decl_or_type (current_function_decl),
+		 VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))));
+	      if (old_flags != new_flags)
+		{
+		  if ((old_flags & ~new_flags) == 0
+		      || (new_flags & EAF_UNUSED))
+		    fprintf (dump_file, "  Flags for param %i improved:",
+			     (int)i);
+		  else
+		    gcc_unreachable ();
+		  dump_eaf_flags (dump_file, old_flags, false);
+		  fprintf (dump_file, " -> ");
+		  dump_eaf_flags (dump_file, new_flags, true);
+		}
+	    }
+	  past_retslot_flags = remove_useless_eaf_flags
+		(past_retslot_flags,
+		 flags_from_decl_or_type (current_function_decl),
+		 VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))));
+	  if (past_retslot_flags != summary->retslot_flags)
+	    {
+	      if ((past_retslot_flags & ~summary->retslot_flags) == 0
+		  || (summary->retslot_flags & EAF_UNUSED))
+		fprintf (dump_file, "  Flags for retslot improved:");
+	      else
+		gcc_unreachable ();
+	      dump_eaf_flags (dump_file, past_retslot_flags, false);
+	      fprintf (dump_file, " -> ");
+	      dump_eaf_flags (dump_file, summary->retslot_flags, true);
+	    }
+	  past_static_chain_flags = remove_useless_eaf_flags
+		(past_static_chain_flags,
+		 flags_from_decl_or_type (current_function_decl),
+		 VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))));
+	  if (past_static_chain_flags != summary->static_chain_flags)
+	    {
+	      if ((past_static_chain_flags & ~summary->static_chain_flags) == 0
+		  || (summary->static_chain_flags & EAF_UNUSED))
+		fprintf (dump_file, "  Flags for static chain improved:");
+	      else
+		gcc_unreachable ();
+	      dump_eaf_flags (dump_file, past_static_chain_flags, false);
+	      fprintf (dump_file, " -> ");
+	      dump_eaf_flags (dump_file, summary->static_chain_flags, true);
+	    }
+	}
+      else if (past_flags_known && !summary)
+	{
+	  for (size_t i = 0; i < past_flags.length (); i++)
+	    {
+	      int old_flags = past_flags[i];
+	      old_flags = remove_useless_eaf_flags
+		(old_flags, flags_from_decl_or_type (current_function_decl),
+		 VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))));
+	      if (old_flags)
+		{
+		  fprintf (dump_file, "  Flags for param %i worsened:",
+			   (int)i);
+		  dump_eaf_flags (dump_file, old_flags, false);
+		  fprintf (dump_file, " -> \n");
+		}
+	    }
+	  past_retslot_flags = remove_useless_eaf_flags
+		(past_retslot_flags,
+		 flags_from_decl_or_type (current_function_decl),
+		 VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))));
+	  if (past_retslot_flags)
+	    {
+	      fprintf (dump_file, "  Flags for retslot worsened:");
+	      dump_eaf_flags (dump_file, past_retslot_flags, false);
+	      fprintf (dump_file, " ->\n");
+	    }
+	  past_static_chain_flags = remove_useless_eaf_flags
+		(past_static_chain_flags,
+		 flags_from_decl_or_type (current_function_decl),
+		 VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))));
+	  if (past_static_chain_flags)
+	    {
+	      fprintf (dump_file, "  Flags for static chain worsened:");
+	      dump_eaf_flags (dump_file, past_static_chain_flags, false);
+	      fprintf (dump_file, " ->\n");
+	    }
+	}
     }
+  return fixup_cfg;
 }
 
 /* Callback for generate_summary.  */
@@ -2218,6 +3140,16 @@ modref_generate (void)
     }
 }
 
+}  /* ANON namespace.  */
+
+/* Debugging helper.  */
+
+void
+debug_eaf_flags (int flags)
+{
+   dump_eaf_flags (stderr, flags, true);
+}
+
 /* Called when a new function is inserted to callgraph late.  */
 
 void
@@ -2284,9 +3216,16 @@ modref_summaries::duplicate (cgraph_node *, cgraph_node *dst,
 			 src_data->loads->max_refs,
 			 src_data->loads->max_accesses);
   dst_data->loads->copy_from (src_data->loads);
+  dst_data->kills.reserve_exact (src_data->kills.length ());
+  dst_data->kills.splice (src_data->kills);
   dst_data->writes_errno = src_data->writes_errno;
+  dst_data->side_effects = src_data->side_effects;
+  dst_data->nondeterministic = src_data->nondeterministic;
+  dst_data->calls_interposable = src_data->calls_interposable;
   if (src_data->arg_flags.length ())
     dst_data->arg_flags = src_data->arg_flags.copy ();
+  dst_data->retslot_flags = src_data->retslot_flags;
+  dst_data->static_chain_flags = src_data->static_chain_flags;
 }
 
 /* Called when new clone is inserted to callgraph late.  */
@@ -2310,8 +3249,13 @@ modref_summaries_lto::duplicate (cgraph_node *, cgraph_node *,
 			 src_data->loads->max_accesses);
   dst_data->loads->copy_from (src_data->loads);
   dst_data->writes_errno = src_data->writes_errno;
+  dst_data->side_effects = src_data->side_effects;
+  dst_data->nondeterministic = src_data->nondeterministic;
+  dst_data->calls_interposable = src_data->calls_interposable;
   if (src_data->arg_flags.length ())
     dst_data->arg_flags = src_data->arg_flags.copy ();
+  dst_data->retslot_flags = src_data->retslot_flags;
+  dst_data->static_chain_flags = src_data->static_chain_flags;
 }
 
 namespace
@@ -2401,7 +3345,7 @@ write_modref_records (modref_records_lto *tt, struct output_block *ob)
    Either nolto_ret or lto_ret is initialized by the tree depending whether
    LTO streaming is expected or not.  */
 
-void
+static void
 read_modref_records (lto_input_block *ib, struct data_in *data_in,
 		     modref_records **nolto_ret,
 		     modref_records_lto **lto_ret)
@@ -2551,7 +3495,7 @@ modref_write_escape_summary (struct bitpack_d *bp, escape_summary *esum)
   escape_entry *ee;
   FOR_EACH_VEC_ELT (esum->esc, i, ee)
     {
-      bp_pack_var_len_unsigned (bp, ee->parm_index);
+      bp_pack_var_len_int (bp, ee->parm_index);
       bp_pack_var_len_unsigned (bp, ee->arg);
       bp_pack_var_len_unsigned (bp, ee->min_flags);
       bp_pack_value (bp, ee->direct, 1);
@@ -2571,7 +3515,7 @@ modref_read_escape_summary (struct bitpack_d *bp, cgraph_edge *e)
   for (unsigned int i = 0; i < n; i++)
     {
       escape_entry ee;
-      ee.parm_index = bp_unpack_var_len_unsigned (bp);
+      ee.parm_index = bp_unpack_var_len_int (bp);
       ee.arg = bp_unpack_var_len_unsigned (bp);
       ee.min_flags = bp_unpack_var_len_unsigned (bp);
       ee.direct = bp_unpack_value (bp, 1);
@@ -2628,12 +3572,17 @@ modref_write ()
 	  streamer_write_uhwi (ob, r->arg_flags.length ());
 	  for (unsigned int i = 0; i < r->arg_flags.length (); i++)
 	    streamer_write_uhwi (ob, r->arg_flags[i]);
+	  streamer_write_uhwi (ob, r->retslot_flags);
+	  streamer_write_uhwi (ob, r->static_chain_flags);
 
 	  write_modref_records (r->loads, ob);
 	  write_modref_records (r->stores, ob);
 
 	  struct bitpack_d bp = bitpack_create (ob->main_stream);
 	  bp_pack_value (&bp, r->writes_errno, 1);
+	  bp_pack_value (&bp, r->side_effects, 1);
+	  bp_pack_value (&bp, r->nondeterministic, 1);
+	  bp_pack_value (&bp, r->calls_interposable, 1);
 	  if (!flag_wpa)
 	    {
 	      for (cgraph_edge *e = cnode->indirect_calls;
@@ -2703,9 +3652,19 @@ read_section (struct lto_file_decl_data *file_data, const char *data,
 	modref_sum = optimization_summaries->get_create (node);
 
       if (modref_sum)
-	modref_sum->writes_errno = false;
+	{
+	  modref_sum->writes_errno = false;
+	  modref_sum->side_effects = false;
+	  modref_sum->nondeterministic = false;
+	  modref_sum->calls_interposable = false;
+	}
       if (modref_sum_lto)
-	modref_sum_lto->writes_errno = false;
+	{
+	  modref_sum_lto->writes_errno = false;
+	  modref_sum_lto->side_effects = false;
+	  modref_sum_lto->nondeterministic = false;
+	  modref_sum_lto->calls_interposable = false;
+	}
 
       gcc_assert (!modref_sum || (!modref_sum->loads
 				  && !modref_sum->stores));
@@ -2724,6 +3683,18 @@ read_section (struct lto_file_decl_data *file_data, const char *data,
 	  if (modref_sum_lto)
 	    modref_sum_lto->arg_flags.quick_push (flags);
 	}
+      eaf_flags_t flags = streamer_read_uhwi (&ib);
+      if (modref_sum)
+	modref_sum->retslot_flags = flags;
+      if (modref_sum_lto)
+	modref_sum_lto->retslot_flags = flags;
+
+      flags = streamer_read_uhwi (&ib);
+      if (modref_sum)
+	modref_sum->static_chain_flags = flags;
+      if (modref_sum_lto)
+	modref_sum_lto->static_chain_flags = flags;
+
       read_modref_records (&ib, data_in,
 			   modref_sum ? &modref_sum->loads : NULL,
 			   modref_sum_lto ? &modref_sum_lto->loads : NULL);
@@ -2738,6 +3709,27 @@ read_section (struct lto_file_decl_data *file_data, const char *data,
 	  if (modref_sum_lto)
 	    modref_sum_lto->writes_errno = true;
 	}
+      if (bp_unpack_value (&bp, 1))
+	{
+	  if (modref_sum)
+	    modref_sum->side_effects = true;
+	  if (modref_sum_lto)
+	    modref_sum_lto->side_effects = true;
+	}
+      if (bp_unpack_value (&bp, 1))
+	{
+	  if (modref_sum)
+	    modref_sum->nondeterministic = true;
+	  if (modref_sum_lto)
+	    modref_sum_lto->nondeterministic = true;
+	}
+      if (bp_unpack_value (&bp, 1))
+	{
+	  if (modref_sum)
+	    modref_sum->calls_interposable = true;
+	  if (modref_sum_lto)
+	    modref_sum_lto->calls_interposable = true;
+	}
       if (!flag_ltrans)
 	{
 	  for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
@@ -2759,6 +3751,8 @@ read_section (struct lto_file_decl_data *file_data, const char *data,
 	      modref_read_escape_summary (&bp, e);
 	    }
 	}
+      if (flag_ltrans)
+	modref_sum->finalize (node->decl);
       if (dump_file)
 	{
 	  fprintf (dump_file, "Read modref for %s\n",
@@ -2886,7 +3880,7 @@ update_signature (struct cgraph_node *node)
 
   map.reserve (max + 1);
   for (i = 0; i <= max; i++)
-    map.quick_push (-1);
+    map.quick_push (MODREF_UNKNOWN_PARM);
   FOR_EACH_VEC_SAFE_ELT (info->param_adjustments->m_adj_params, i, p)
     {
       int idx = info->param_adjustments->get_original_index (i);
@@ -2897,6 +3891,8 @@ update_signature (struct cgraph_node *node)
     {
       r->loads->remap_params (&map);
       r->stores->remap_params (&map);
+      /* TODO: One we do IPA kills analysis, update the table here.  */
+      r->kills.release ();
       if (r->arg_flags.length ())
 	remap_arg_flags (r->arg_flags, info);
     }
@@ -2904,6 +3900,8 @@ update_signature (struct cgraph_node *node)
     {
       r_lto->loads->remap_params (&map);
       r_lto->stores->remap_params (&map);
+      /* TODO: One we do IPA kills analysis, update the table here.  */
+      r_lto->kills.release ();
       if (r_lto->arg_flags.length ())
 	remap_arg_flags (r_lto->arg_flags, info);
     }
@@ -2915,6 +3913,8 @@ update_signature (struct cgraph_node *node)
       if (r_lto)
 	r_lto->dump (dump_file);
     }
+  if (r)
+    r->finalize (node->decl);
   return;
 }
 
@@ -2962,7 +3962,8 @@ public:
 
 unsigned int pass_modref::execute (function *f)
 {
-  analyze_function (f, false);
+  if (analyze_function (f, false))
+    return execute_fixup_cfg ();
   return 0;
 }
 
@@ -2978,6 +3979,8 @@ make_pass_ipa_modref (gcc::context *ctxt)
   return new pass_ipa_modref (ctxt);
 }
 
+namespace {
+
 /* Skip edges from and to nodes without ipa_pure_const enabled.
    Ignore not available symbols.  */
 
@@ -2995,9 +3998,7 @@ ignore_edge (struct cgraph_edge *e)
 
   return (avail <= AVAIL_INTERPOSABLE
 	  || ((!optimization_summaries || !optimization_summaries->get (callee))
-	      && (!summaries_lto || !summaries_lto->get (callee)))
-	  || flags_from_decl_or_type (e->callee->decl)
-	     & (ECF_CONST | ECF_NOVOPS));
+	      && (!summaries_lto || !summaries_lto->get (callee))));
 }
 
 /* Compute parm_map for CALLEE_EDGE.  */
@@ -3030,7 +4031,7 @@ compute_parm_map (cgraph_edge *callee_edge, vec<modref_parm_map> *parm_map)
 	{
 	  if (es && es->param[i].points_to_local_or_readonly_memory)
 	    {
-	      (*parm_map)[i].parm_index = -2;
+	      (*parm_map)[i].parm_index = MODREF_LOCAL_MEMORY_PARM;
 	      continue;
 	    }
 
@@ -3044,7 +4045,7 @@ compute_parm_map (cgraph_edge *callee_edge, vec<modref_parm_map> *parm_map)
 						 (callee_pi, i));
 	      if (cst && points_to_local_or_readonly_memory_p (cst))
 		{
-		  (*parm_map)[i].parm_index = -2;
+		  (*parm_map)[i].parm_index = MODREF_LOCAL_MEMORY_PARM;
 		  continue;
 		}
 	    }
@@ -3098,7 +4099,7 @@ struct escape_map
   bool direct;
 };
 
-/* Update escape map fo E.  */
+/* Update escape map for E.  */
 
 static void
 update_escape_summary_1 (cgraph_edge *e,
@@ -3117,7 +4118,10 @@ update_escape_summary_1 (cgraph_edge *e,
     {
       unsigned int j;
       struct escape_map *em;
-      if (ee->parm_index >= map.length ())
+      /* TODO: We do not have jump functions for return slots, so we
+	 never propagate them to outer function.  */
+      if (ee->parm_index >= (int)map.length ()
+	  || ee->parm_index < 0)
 	continue;
       FOR_EACH_VEC_ELT (map[ee->parm_index], j, em)
 	{
@@ -3125,7 +4129,7 @@ update_escape_summary_1 (cgraph_edge *e,
 	  if (ee->direct && !em->direct)
 	    min_flags = deref_flags (min_flags, ignore_stores);
 	  struct escape_entry entry = {em->parm_index, ee->arg,
-	    			       ee->min_flags,
+				       ee->min_flags,
 				       ee->direct & em->direct};
 	  sum->esc.safe_push (entry);
 	}
@@ -3154,180 +4158,6 @@ update_escape_summary (cgraph_node *node,
     }
 }
 
-/* Call EDGE was inlined; merge summary from callee to the caller.  */
-
-void
-ipa_merge_modref_summary_after_inlining (cgraph_edge *edge)
-{
-  if (!summaries && !summaries_lto)
-    return;
-
-  struct cgraph_node *to = (edge->caller->inlined_to
-			    ? edge->caller->inlined_to : edge->caller);
-  class modref_summary *to_info = summaries ? summaries->get (to) : NULL;
-  class modref_summary_lto *to_info_lto = summaries_lto
-					  ? summaries_lto->get (to) : NULL;
-
-  if (!to_info && !to_info_lto)
-    {
-      if (summaries)
-	summaries->remove (edge->callee);
-      if (summaries_lto)
-	summaries_lto->remove (edge->callee);
-      remove_modref_edge_summaries (edge->callee);
-      return;
-    }
-
-  class modref_summary *callee_info = summaries ? summaries->get (edge->callee)
-				      : NULL;
-  class modref_summary_lto *callee_info_lto
-		 = summaries_lto ? summaries_lto->get (edge->callee) : NULL;
-  int flags = flags_from_decl_or_type (edge->callee->decl);
-  bool ignore_stores = ignore_stores_p (edge->caller->decl, flags);
-
-  if (!callee_info && to_info)
-    {
-      if (!(flags & (ECF_CONST | ECF_NOVOPS)))
-	to_info->loads->collapse ();
-      if (!ignore_stores)
-	to_info->stores->collapse ();
-    }
-  if (!callee_info_lto && to_info_lto)
-    {
-      if (!(flags & (ECF_CONST | ECF_NOVOPS)))
-	to_info_lto->loads->collapse ();
-      if (!ignore_stores)
-	to_info_lto->stores->collapse ();
-    }
-  if (callee_info || callee_info_lto)
-    {
-      auto_vec <modref_parm_map, 32> parm_map;
-
-      compute_parm_map (edge, &parm_map);
-
-      if (!ignore_stores)
-	{
-	  if (to_info && callee_info)
-	    to_info->stores->merge (callee_info->stores, &parm_map, false);
-	  if (to_info_lto && callee_info_lto)
-	    to_info_lto->stores->merge (callee_info_lto->stores, &parm_map,
-					false);
-	}
-      if (!(flags & (ECF_CONST | ECF_NOVOPS)))
-	{
-	  if (to_info && callee_info)
-	    to_info->loads->merge (callee_info->loads, &parm_map, false);
-	  if (to_info_lto && callee_info_lto)
-	    to_info_lto->loads->merge (callee_info_lto->loads, &parm_map,
-				       false);
-	}
-    }
-
-  /* Now merge escape summaries.
-     For every escape to the callee we need to merge calle flags
-     and remap calees escapes.  */
-  class escape_summary *sum = escape_summaries->get (edge);
-  int max_escape = -1;
-  escape_entry *ee;
-  unsigned int i;
-
-  if (sum && !(flags & (ECF_CONST | ECF_NOVOPS)))
-    FOR_EACH_VEC_ELT (sum->esc, i, ee)
-      if ((int)ee->arg > max_escape)
-	max_escape = ee->arg;
-
-  auto_vec <vec <struct escape_map>, 32> emap (max_escape + 1);
-  emap.safe_grow (max_escape + 1, true);
-  for (i = 0; (int)i < max_escape + 1; i++)
-    emap[i] = vNULL;
-
-  if (sum && !(flags & (ECF_CONST | ECF_NOVOPS)))
-    FOR_EACH_VEC_ELT (sum->esc, i, ee)
-      {
-	bool needed = false;
-	if (to_info && to_info->arg_flags.length () > ee->parm_index)
-	  {
-	    int flags = callee_info
-			&& callee_info->arg_flags.length () > ee->arg
-			? callee_info->arg_flags[ee->arg] : 0;
-	    if (!ee->direct)
-	      flags = deref_flags (flags, ignore_stores);
-	    else if (ignore_stores)
-	      flags |= ignore_stores_eaf_flags;
-	    flags |= ee->min_flags;
-	    to_info->arg_flags[ee->parm_index] &= flags;
-	    if (to_info->arg_flags[ee->parm_index])
-	      needed = true;
-	  }
-	if (to_info_lto && to_info_lto->arg_flags.length () > ee->parm_index)
-	  {
-	    int flags = callee_info_lto
-			&& callee_info_lto->arg_flags.length () > ee->arg
-			? callee_info_lto->arg_flags[ee->arg] : 0;
-	    if (!ee->direct)
-	      flags = deref_flags (flags, ignore_stores);
-	    else if (ignore_stores)
-	      flags |= ignore_stores_eaf_flags;
-	    flags |= ee->min_flags;
-	    to_info_lto->arg_flags[ee->parm_index] &= flags;
-	    if (to_info_lto->arg_flags[ee->parm_index])
-	      needed = true;
-	  }
-	struct escape_map entry = {ee->parm_index, ee->direct};
-	if (needed)
-	  emap[ee->arg].safe_push (entry);
-      }
-  update_escape_summary (edge->callee, emap, ignore_stores);
-  for (i = 0; (int)i < max_escape + 1; i++)
-    emap[i].release ();
-  if (sum)
-    escape_summaries->remove (edge);
-
-  if (summaries)
-    {
-      if (to_info && !to_info->useful_p (flags))
-	{
-	  if (dump_file)
-	    fprintf (dump_file, "Removed mod-ref summary for %s\n",
-		     to->dump_name ());
-	  summaries->remove (to);
-	  to_info = NULL;
-	}
-      else if (to_info && dump_file)
-	{
-	  if (dump_file)
-	    fprintf (dump_file, "Updated mod-ref summary for %s\n",
-		     to->dump_name ());
-	  to_info->dump (dump_file);
-	}
-      if (callee_info)
-	summaries->remove (edge->callee);
-    }
-  if (summaries_lto)
-    {
-      if (to_info_lto && !to_info_lto->useful_p (flags))
-	{
-	  if (dump_file)
-	    fprintf (dump_file, "Removed mod-ref summary for %s\n",
-		     to->dump_name ());
-	  summaries_lto->remove (to);
-	}
-      else if (to_info_lto && dump_file)
-	{
-	  if (dump_file)
-	    fprintf (dump_file, "Updated mod-ref summary for %s\n",
-		     to->dump_name ());
-	  to_info_lto->dump (dump_file);
-	  to_info_lto = NULL;
-	}
-      if (callee_info_lto)
-	summaries_lto->remove (edge->callee);
-    }
-  if (!to_info && !to_info_lto)
-    remove_modref_edge_summaries (to);
-  return;
-}
-
 /* Get parameter type from DECL.  This is only safe for special cases
    like builtins we create fnspec for because the type match is checked
    at fnspec creation time.  */
@@ -3389,11 +4219,60 @@ static bool
 propagate_unknown_call (cgraph_node *node,
 			cgraph_edge *e, int ecf_flags,
 			modref_summary *cur_summary,
-			modref_summary_lto *cur_summary_lto)
+			modref_summary_lto *cur_summary_lto,
+			bool nontrivial_scc)
 {
   bool changed = false;
   class fnspec_summary *fnspec_sum = fnspec_summaries->get (e);
   auto_vec <modref_parm_map, 32> parm_map;
+  bool looping;
+
+  if (e->callee
+      && builtin_safe_for_const_function_p (&looping, e->callee->decl))
+    {
+      if (looping && cur_summary && !cur_summary->side_effects)
+	{
+	  cur_summary->side_effects = true;
+	  changed = true;
+	}
+      if (looping && cur_summary_lto && !cur_summary_lto->side_effects)
+	{
+	  cur_summary_lto->side_effects = true;
+	  changed = true;
+	}
+      return changed;
+    }
+
+  if (!(ecf_flags & (ECF_CONST | ECF_NOVOPS | ECF_PURE))
+      || (ecf_flags & ECF_LOOPING_CONST_OR_PURE)
+      || nontrivial_scc)
+    {
+      if (cur_summary && !cur_summary->side_effects)
+	{
+	  cur_summary->side_effects = true;
+	  changed = true;
+	}
+      if (cur_summary_lto && !cur_summary_lto->side_effects)
+	{
+	  cur_summary_lto->side_effects = true;
+	  changed = true;
+	}
+      if (cur_summary && !cur_summary->nondeterministic
+	  && !ignore_nondeterminism_p (node->decl, ecf_flags))
+	{
+	  cur_summary->nondeterministic = true;
+	  changed = true;
+	}
+      if (cur_summary_lto && !cur_summary_lto->nondeterministic
+	  && !ignore_nondeterminism_p (node->decl, ecf_flags))
+	{
+	  cur_summary_lto->nondeterministic = true;
+	  changed = true;
+	}
+    }
+  if (ecf_flags & (ECF_CONST | ECF_NOVOPS))
+    return changed;
+
   if (fnspec_sum
       && compute_parm_map (e, &parm_map))
     {
@@ -3413,9 +4292,9 @@ propagate_unknown_call (cgraph_node *node,
 		   || fnspec.arg_maybe_read_p (i))
 	    {
 	      modref_parm_map map = parm_map[i];
-	      if (map.parm_index == -2)
+	      if (map.parm_index == MODREF_LOCAL_MEMORY_PARM)
 		continue;
-	      if (map.parm_index == -1)
+	      if (map.parm_index == MODREF_UNKNOWN_PARM)
 		{
 		  collapse_loads (cur_summary, cur_summary_lto);
 		  break;
@@ -3443,9 +4322,9 @@ propagate_unknown_call (cgraph_node *node,
 		   || fnspec.arg_maybe_written_p (i))
 	    {
 	      modref_parm_map map = parm_map[i];
-	      if (map.parm_index == -2)
+	      if (map.parm_index == MODREF_LOCAL_MEMORY_PARM)
 		continue;
-	      if (map.parm_index == -1)
+	      if (map.parm_index == MODREF_UNKNOWN_PARM)
 		{
 		  collapse_stores (cur_summary, cur_summary_lto);
 		  break;
@@ -3510,7 +4389,7 @@ remove_useless_summaries (cgraph_node *node,
 /* Perform iterative dataflow on SCC component starting in COMPONENT_NODE
    and propagate loads/stores.  */
 
-static void
+static bool
 modref_propagate_in_scc (cgraph_node *component_node)
 {
   bool changed = true;
@@ -3519,6 +4398,8 @@ modref_propagate_in_scc (cgraph_node *component_node)
 
   while (changed)
     {
+      bool nontrivial_scc
+		 = ((struct ipa_dfs_info *) component_node->aux)->next_cycle;
       changed = false;
       for (struct cgraph_node *cur = component_node; cur;
 	   cur = ((struct ipa_dfs_info *) cur->aux)->next_cycle)
@@ -3544,14 +4425,12 @@ modref_propagate_in_scc (cgraph_node *component_node)
 
 	  for (cgraph_edge *e = cur->indirect_calls; e; e = e->next_callee)
 	    {
-	      if (e->indirect_info->ecf_flags & (ECF_CONST | ECF_NOVOPS))
-		continue;
 	      if (dump_file)
-		fprintf (dump_file, "    Indirect call"
-			 "collapsing loads\n");
+		fprintf (dump_file, "    Indirect call\n");
 	      if (propagate_unknown_call
 			   (node, e, e->indirect_info->ecf_flags,
-			    cur_summary, cur_summary_lto))
+			    cur_summary, cur_summary_lto,
+			    nontrivial_scc))
 		{
 		  changed = true;
 		  remove_useless_summaries (node, &cur_summary,
@@ -3573,8 +4452,9 @@ modref_propagate_in_scc (cgraph_node *component_node)
 	      modref_summary_lto *callee_summary_lto = NULL;
 	      struct cgraph_node *callee;
 
-	      if (flags & (ECF_CONST | ECF_NOVOPS)
-		  || !callee_edge->inline_failed)
+	      if (!callee_edge->inline_failed
+		 || ((flags & (ECF_CONST | ECF_NOVOPS))
+		     && !(flags & ECF_LOOPING_CONST_OR_PURE)))
 		continue;
 
 	      /* Get the callee and its summary.  */
@@ -3603,7 +4483,8 @@ modref_propagate_in_scc (cgraph_node *component_node)
 			     " or not available\n");
 		  changed |= propagate_unknown_call
 			       (node, callee_edge, flags,
-				cur_summary, cur_summary_lto);
+				cur_summary, cur_summary_lto,
+				nontrivial_scc);
 		  if (!cur_summary && !cur_summary_lto)
 		    break;
 		  continue;
@@ -3619,7 +4500,8 @@ modref_propagate_in_scc (cgraph_node *component_node)
 		    fprintf (dump_file, "      No call target summary\n");
 		  changed |= propagate_unknown_call
 			       (node, callee_edge, flags,
-				cur_summary, NULL);
+				cur_summary, NULL,
+				nontrivial_scc);
 		}
 	      if (cur_summary_lto
 		  && !(callee_summary_lto = summaries_lto->get (callee)))
@@ -3628,16 +4510,57 @@ modref_propagate_in_scc (cgraph_node *component_node)
 		    fprintf (dump_file, "      No call target summary\n");
 		  changed |= propagate_unknown_call
 			       (node, callee_edge, flags,
-				NULL, cur_summary_lto);
+				NULL, cur_summary_lto,
+				nontrivial_scc);
 		}
 
+	      if (callee_summary && !cur_summary->side_effects
+		  && (callee_summary->side_effects
+		      || callee_edge->recursive_p ()))
+		{
+		  cur_summary->side_effects = true;
+		  changed = true;
+		}
+	      if (callee_summary_lto && !cur_summary_lto->side_effects
+		  && (callee_summary_lto->side_effects
+		      || callee_edge->recursive_p ()))
+		{
+		  cur_summary_lto->side_effects = true;
+		  changed = true;
+		}
+	      if (callee_summary && !cur_summary->nondeterministic
+		  && callee_summary->nondeterministic
+		  && !ignore_nondeterminism_p (cur->decl, flags))
+		{
+		  cur_summary->nondeterministic = true;
+		  changed = true;
+		}
+	      if (callee_summary_lto && !cur_summary_lto->nondeterministic
+		  && callee_summary_lto->nondeterministic
+		  && !ignore_nondeterminism_p (cur->decl, flags))
+		{
+		  cur_summary_lto->nondeterministic = true;
+		  changed = true;
+		}
+	      if (flags & (ECF_CONST | ECF_NOVOPS))
+		continue;
+
 	      /* We can not safely optimize based on summary of callee if it
 		 does not always bind to current def: it is possible that
 		 memory load was optimized out earlier which may not happen in
 		 the interposed variant.  */
 	      if (!callee_edge->binds_to_current_def_p ())
 		{
-		  changed |= collapse_loads (cur_summary, cur_summary_lto);
+		  if (cur_summary && !cur_summary->calls_interposable)
+		    {
+		      cur_summary->calls_interposable = true;
+		      changed = true;
+		    }
+		  if (cur_summary_lto && !cur_summary_lto->calls_interposable)
+		    {
+		      cur_summary_lto->calls_interposable = true;
+		      changed = true;
+		    }
 		  if (dump_file)
 		    fprintf (dump_file, "      May not bind local;"
 			     " collapsing loads\n");
@@ -3645,6 +4568,10 @@ modref_propagate_in_scc (cgraph_node *component_node)
 
 
 	      auto_vec <modref_parm_map, 32> parm_map;
+	      modref_parm_map chain_map;
+	      /* TODO: Once we get jump functions for static chains we could
+		 compute this.  */
+	      chain_map.parm_index = MODREF_UNKNOWN_PARM;
 
 	      compute_parm_map (callee_edge, &parm_map);
 
@@ -3652,12 +4579,13 @@ modref_propagate_in_scc (cgraph_node *component_node)
 	      if (callee_summary)
 		{
 		  changed |= cur_summary->loads->merge
-				  (callee_summary->loads, &parm_map, !first);
+				  (callee_summary->loads, &parm_map,
+				   &chain_map, !first);
 		  if (!ignore_stores)
 		    {
 		      changed |= cur_summary->stores->merge
 				      (callee_summary->stores, &parm_map,
-				       !first);
+				       &chain_map, !first);
 		      if (!cur_summary->writes_errno
 			  && callee_summary->writes_errno)
 			{
@@ -3670,12 +4598,12 @@ modref_propagate_in_scc (cgraph_node *component_node)
 		{
 		  changed |= cur_summary_lto->loads->merge
 				  (callee_summary_lto->loads, &parm_map,
-				   !first);
+				   &chain_map, !first);
 		  if (!ignore_stores)
 		    {
 		      changed |= cur_summary_lto->stores->merge
 				      (callee_summary_lto->stores, &parm_map,
-				       !first);
+				       &chain_map, !first);
 		      if (!cur_summary_lto->writes_errno
 			  && callee_summary_lto->writes_errno)
 			{
@@ -3706,6 +4634,41 @@ modref_propagate_in_scc (cgraph_node *component_node)
   if (dump_file)
     fprintf (dump_file,
 	     "Propagation finished in %i iterations\n", iteration);
+  bool pureconst = false;
+  for (struct cgraph_node *cur = component_node; cur;
+       cur = ((struct ipa_dfs_info *) cur->aux)->next_cycle)
+    if (!cur->inlined_to && opt_for_fn (cur->decl, flag_ipa_pure_const))
+      {
+	modref_summary *summary = optimization_summaries
+				  ? optimization_summaries->get (cur)
+				  : NULL;
+	modref_summary_lto *summary_lto = summaries_lto
+					  ? summaries_lto->get (cur)
+					  : NULL;
+	if (summary && !summary->stores->every_base && !summary->stores->bases
+	    && !summary->nondeterministic)
+	  {
+	    if (!summary->loads->every_base && !summary->loads->bases
+		&& !summary->calls_interposable)
+	      pureconst |= ipa_make_function_const
+		     (cur, summary->side_effects, false);
+	    else
+	      pureconst |= ipa_make_function_pure
+		     (cur, summary->side_effects, false);
+	  }
+	if (summary_lto && !summary_lto->stores->every_base
+	    && !summary_lto->stores->bases && !summary_lto->nondeterministic)
+	  {
+	    if (!summary_lto->loads->every_base && !summary_lto->loads->bases
+		&& !summary_lto->calls_interposable)
+	      pureconst |= ipa_make_function_const
+		     (cur, summary_lto->side_effects, false);
+	    else
+	      pureconst |= ipa_make_function_pure
+		     (cur, summary_lto->side_effects, false);
+	  }
+     }
+  return pureconst;
 }
 
 /* Dump results of propagation in SCC rooted in COMPONENT_NODE.  */
@@ -3756,12 +4719,15 @@ modref_merge_call_site_flags (escape_summary *sum,
 			      modref_summary *summary,
 			      modref_summary_lto *summary_lto,
 			      tree caller,
-			      int ecf_flags)
+			      cgraph_edge *e,
+			      int caller_ecf_flags,
+			      int callee_ecf_flags,
+			      bool binds_to_current_def)
 {
   escape_entry *ee;
   unsigned int i;
   bool changed = false;
-  bool ignore_stores = ignore_stores_p (caller, ecf_flags);
+  bool ignore_stores = ignore_stores_p (caller, callee_ecf_flags);
 
   /* If we have no useful info to propagate.  */
   if ((!cur_summary || !cur_summary->arg_flags.length ())
@@ -3772,6 +4738,9 @@ modref_merge_call_site_flags (escape_summary *sum,
     {
       int flags = 0;
       int flags_lto = 0;
+      /* Returning the value is already accounted to at local propagation.  */
+      int implicit_flags = EAF_NOT_RETURNED_DIRECTLY
+			   | EAF_NOT_RETURNED_INDIRECTLY;
 
       if (summary && ee->arg < summary->arg_flags.length ())
 	flags = summary->arg_flags[ee->arg];
@@ -3783,44 +4752,57 @@ modref_merge_call_site_flags (escape_summary *sum,
 	  flags = deref_flags (flags, ignore_stores);
 	  flags_lto = deref_flags (flags_lto, ignore_stores);
 	}
-      else if (ignore_stores)
+      if (ignore_stores)
+	 implicit_flags |= ignore_stores_eaf_flags;
+      if (callee_ecf_flags & ECF_PURE)
+	implicit_flags |= implicit_pure_eaf_flags;
+      if (callee_ecf_flags & (ECF_CONST | ECF_NOVOPS))
+	implicit_flags |= implicit_const_eaf_flags;
+      class fnspec_summary *fnspec_sum = fnspec_summaries->get (e);
+      if (fnspec_sum)
 	{
-	  flags |= ignore_stores_eaf_flags;
-	  flags_lto |= ignore_stores_eaf_flags;
+	  attr_fnspec fnspec (fnspec_sum->fnspec);
+	  implicit_flags |= fnspec.arg_eaf_flags (ee->arg);
+	}
+      if (!ee->direct)
+	implicit_flags = deref_flags (implicit_flags, ignore_stores);
+      flags |= implicit_flags;
+      flags_lto |= implicit_flags;
+      if (!binds_to_current_def && (flags || flags_lto))
+	{
+	  flags = interposable_eaf_flags (flags, implicit_flags);
+	  flags_lto = interposable_eaf_flags (flags_lto, implicit_flags);
 	}
-      /* Returning the value is already accounted to at local propagation.  */
-      flags |= ee->min_flags | EAF_NOT_RETURNED;
-      flags_lto |= ee->min_flags | EAF_NOT_RETURNED;
-      /* Noescape implies that value also does not escape directly.
-	 Fnspec machinery does set both so compensate for this.  */
-      if (flags & EAF_NOESCAPE)
-	flags |= EAF_NODIRECTESCAPE;
-      if (flags_lto & EAF_NOESCAPE)
-	flags_lto |= EAF_NODIRECTESCAPE;
       if (!(flags & EAF_UNUSED)
-	  && cur_summary && ee->parm_index < cur_summary->arg_flags.length ())
+	  && cur_summary && ee->parm_index < (int)cur_summary->arg_flags.length ())
 	{
-	  int f = cur_summary->arg_flags[ee->parm_index];
+	  eaf_flags_t &f = ee->parm_index == MODREF_RETSLOT_PARM
+			   ? cur_summary->retslot_flags
+			   : ee->parm_index == MODREF_STATIC_CHAIN_PARM
+			   ? cur_summary->static_chain_flags
+			   : cur_summary->arg_flags[ee->parm_index];
 	  if ((f & flags) != f)
 	    {
 	      f = remove_useless_eaf_flags
-			 (f & flags, ecf_flags,
+			 (f & flags, caller_ecf_flags,
 			  VOID_TYPE_P (TREE_TYPE (TREE_TYPE (caller))));
-	      cur_summary->arg_flags[ee->parm_index] = f;
 	      changed = true;
 	    }
 	}
       if (!(flags_lto & EAF_UNUSED)
 	  && cur_summary_lto
-	  && ee->parm_index < cur_summary_lto->arg_flags.length ())
+	  && ee->parm_index < (int)cur_summary_lto->arg_flags.length ())
 	{
-	  int f = cur_summary_lto->arg_flags[ee->parm_index];
+	  eaf_flags_t &f = ee->parm_index == MODREF_RETSLOT_PARM
+			   ? cur_summary_lto->retslot_flags
+			   : ee->parm_index == MODREF_STATIC_CHAIN_PARM
+			   ? cur_summary_lto->static_chain_flags
+			   : cur_summary_lto->arg_flags[ee->parm_index];
 	  if ((f & flags_lto) != f)
 	    {
 	      f = remove_useless_eaf_flags
-			 (f & flags_lto, ecf_flags,
+			 (f & flags_lto, caller_ecf_flags,
 			  VOID_TYPE_P (TREE_TYPE (TREE_TYPE (caller))));
-	      cur_summary_lto->arg_flags[ee->parm_index] = f;
 	      changed = true;
 	    }
 	}
@@ -3853,6 +4835,7 @@ modref_propagate_flags_in_scc (cgraph_node *component_node)
 
 	  if (!cur_summary && !cur_summary_lto)
 	    continue;
+	  int caller_ecf_flags = flags_from_decl_or_type (cur->decl);
 
 	  if (dump_file)
 	    fprintf (dump_file, "  Processing %s%s%s\n",
@@ -3871,7 +4854,11 @@ modref_propagate_flags_in_scc (cgraph_node *component_node)
 	      changed |= modref_merge_call_site_flags
 				(sum, cur_summary, cur_summary_lto,
 				 NULL, NULL,
-				 node->decl, e->indirect_info->ecf_flags);
+				 node->decl,
+				 e,
+				 caller_ecf_flags,
+				 e->indirect_info->ecf_flags,
+				 false);
 	    }
 
 	  if (!cur_summary && !cur_summary_lto)
@@ -3923,7 +4910,11 @@ modref_propagate_flags_in_scc (cgraph_node *component_node)
 	      changed |= modref_merge_call_site_flags
 				(sum, cur_summary, cur_summary_lto,
 				 callee_summary, callee_summary_lto,
-				 node->decl, ecf_flags);
+				 node->decl,
+				 callee_edge,
+				 caller_ecf_flags,
+				 ecf_flags,
+				 callee->binds_to_current_def_p ());
 	      if (dump_file && changed)
 		{
 		  if (cur_summary)
@@ -3940,6 +4931,193 @@ modref_propagate_flags_in_scc (cgraph_node *component_node)
 	     "Propagation of flags finished in %i iterations\n", iteration);
 }
 
+}  /* ANON namespace.  */
+
+/* Call EDGE was inlined; merge summary from callee to the caller.  */
+
+void
+ipa_merge_modref_summary_after_inlining (cgraph_edge *edge)
+{
+  if (!summaries && !summaries_lto)
+    return;
+
+  struct cgraph_node *to = (edge->caller->inlined_to
+			    ? edge->caller->inlined_to : edge->caller);
+  class modref_summary *to_info = summaries ? summaries->get (to) : NULL;
+  class modref_summary_lto *to_info_lto = summaries_lto
+					  ? summaries_lto->get (to) : NULL;
+
+  if (!to_info && !to_info_lto)
+    {
+      if (summaries)
+	summaries->remove (edge->callee);
+      if (summaries_lto)
+	summaries_lto->remove (edge->callee);
+      remove_modref_edge_summaries (edge->callee);
+      return;
+    }
+
+  class modref_summary *callee_info = summaries ? summaries->get (edge->callee)
+				      : NULL;
+  class modref_summary_lto *callee_info_lto
+		 = summaries_lto ? summaries_lto->get (edge->callee) : NULL;
+  int flags = flags_from_decl_or_type (edge->callee->decl);
+  bool ignore_stores = ignore_stores_p (edge->caller->decl, flags);
+
+  if (!callee_info && to_info)
+    {
+      if (!(flags & (ECF_CONST | ECF_NOVOPS)))
+	to_info->loads->collapse ();
+      if (!ignore_stores)
+	to_info->stores->collapse ();
+    }
+  if (!callee_info_lto && to_info_lto)
+    {
+      if (!(flags & (ECF_CONST | ECF_NOVOPS)))
+	to_info_lto->loads->collapse ();
+      if (!ignore_stores)
+	to_info_lto->stores->collapse ();
+    }
+  if (callee_info || callee_info_lto)
+    {
+      auto_vec <modref_parm_map, 32> parm_map;
+      modref_parm_map chain_map;
+      /* TODO: Once we get jump functions for static chains we could
+	 compute this.  */
+      chain_map.parm_index = MODREF_UNKNOWN_PARM;
+
+      compute_parm_map (edge, &parm_map);
+
+      if (!ignore_stores)
+	{
+	  if (to_info && callee_info)
+	    to_info->stores->merge (callee_info->stores, &parm_map,
+				    &chain_map, false);
+	  if (to_info_lto && callee_info_lto)
+	    to_info_lto->stores->merge (callee_info_lto->stores, &parm_map,
+					&chain_map, false);
+	}
+      if (!(flags & (ECF_CONST | ECF_NOVOPS)))
+	{
+	  if (to_info && callee_info)
+	    to_info->loads->merge (callee_info->loads, &parm_map,
+				   &chain_map, false);
+	  if (to_info_lto && callee_info_lto)
+	    to_info_lto->loads->merge (callee_info_lto->loads, &parm_map,
+				       &chain_map, false);
+	}
+    }
+
+  /* Now merge escape summaries.
+     For every escape to the callee we need to merge calle flags
+     and remap calees escapes.  */
+  class escape_summary *sum = escape_summaries->get (edge);
+  int max_escape = -1;
+  escape_entry *ee;
+  unsigned int i;
+
+  if (sum && !(flags & (ECF_CONST | ECF_NOVOPS)))
+    FOR_EACH_VEC_ELT (sum->esc, i, ee)
+      if ((int)ee->arg > max_escape)
+	max_escape = ee->arg;
+
+  auto_vec <vec <struct escape_map>, 32> emap (max_escape + 1);
+  emap.safe_grow (max_escape + 1, true);
+  for (i = 0; (int)i < max_escape + 1; i++)
+    emap[i] = vNULL;
+
+  if (sum && !(flags & (ECF_CONST | ECF_NOVOPS)))
+    FOR_EACH_VEC_ELT (sum->esc, i, ee)
+      {
+	bool needed = false;
+	/* TODO: We do not have jump functions for return slots, so we
+	   never propagate them to outer function.  */
+	if (ee->parm_index < 0)
+	  continue;
+	if (to_info && (int)to_info->arg_flags.length () > ee->parm_index)
+	  {
+	    int flags = callee_info
+			&& callee_info->arg_flags.length () > ee->arg
+			? callee_info->arg_flags[ee->arg] : 0;
+	    if (!ee->direct)
+	      flags = deref_flags (flags, ignore_stores);
+	    else if (ignore_stores)
+	      flags |= ignore_stores_eaf_flags;
+	    flags |= ee->min_flags;
+	    to_info->arg_flags[ee->parm_index] &= flags;
+	    if (to_info->arg_flags[ee->parm_index])
+	      needed = true;
+	  }
+	if (to_info_lto
+	    && (int)to_info_lto->arg_flags.length () > ee->parm_index)
+	  {
+	    int flags = callee_info_lto
+			&& callee_info_lto->arg_flags.length () > ee->arg
+			? callee_info_lto->arg_flags[ee->arg] : 0;
+	    if (!ee->direct)
+	      flags = deref_flags (flags, ignore_stores);
+	    else if (ignore_stores)
+	      flags |= ignore_stores_eaf_flags;
+	    flags |= ee->min_flags;
+	    to_info_lto->arg_flags[ee->parm_index] &= flags;
+	    if (to_info_lto->arg_flags[ee->parm_index])
+	      needed = true;
+	  }
+	struct escape_map entry = {ee->parm_index, ee->direct};
+	if (needed)
+	  emap[ee->arg].safe_push (entry);
+      }
+  update_escape_summary (edge->callee, emap, ignore_stores);
+  for (i = 0; (int)i < max_escape + 1; i++)
+    emap[i].release ();
+  if (sum)
+    escape_summaries->remove (edge);
+
+  if (summaries)
+    {
+      if (to_info && !to_info->useful_p (flags))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "Removed mod-ref summary for %s\n",
+		     to->dump_name ());
+	  summaries->remove (to);
+	  to_info = NULL;
+	}
+      else if (to_info && dump_file)
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "Updated mod-ref summary for %s\n",
+		     to->dump_name ());
+	  to_info->dump (dump_file);
+	}
+      if (callee_info)
+	summaries->remove (edge->callee);
+    }
+  if (summaries_lto)
+    {
+      if (to_info_lto && !to_info_lto->useful_p (flags))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "Removed mod-ref summary for %s\n",
+		     to->dump_name ());
+	  summaries_lto->remove (to);
+	}
+      else if (to_info_lto && dump_file)
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "Updated mod-ref summary for %s\n",
+		     to->dump_name ());
+	  to_info_lto->dump (dump_file);
+	  to_info_lto = NULL;
+	}
+      if (callee_info_lto)
+	summaries_lto->remove (edge->callee);
+    }
+  if (!to_info && !to_info_lto)
+    remove_modref_edge_summaries (to);
+  return;
+}
+
 /* Run the IPA pass.  This will take a function's summaries and calls and
    construct new summaries which represent a transitive closure.  So that
    summary of an analyzed function contains information about the loads and
@@ -3950,6 +5128,7 @@ pass_ipa_modref::execute (function *)
 {
   if (!summaries && !summaries_lto)
     return 0;
+  bool pureconst = false;
 
   if (optimization_summaries)
     ggc_delete (optimization_summaries);
@@ -3972,8 +5151,13 @@ pass_ipa_modref::execute (function *)
       if (dump_file)
 	fprintf (dump_file, "\n\nStart of SCC component\n");
 
-      modref_propagate_in_scc (component_node);
+      pureconst |= modref_propagate_in_scc (component_node);
       modref_propagate_flags_in_scc (component_node);
+      if (optimization_summaries)
+	for (struct cgraph_node *cur = component_node; cur;
+	     cur = ((struct ipa_dfs_info *) cur->aux)->next_cycle)
+	  if (modref_summary *sum = optimization_summaries->get (cur))
+	    sum->finalize (cur->decl);
       if (dump_file)
 	modref_propagate_dump_scc (component_node);
     }
@@ -3988,7 +5172,10 @@ pass_ipa_modref::execute (function *)
   fnspec_summaries = NULL;
   delete escape_summaries;
   escape_summaries = NULL;
-  return 0;
+
+  /* If we posibly made constructors const/pure we may need to remove
+     them.  */
+  return pureconst ? TODO_remove_functions : 0;
 }
 
 /* Summaries must stay alive until end of compilation.  */
@@ -3999,8 +5186,6 @@ ipa_modref_c_finalize ()
   if (optimization_summaries)
     ggc_delete (optimization_summaries);
   optimization_summaries = NULL;
-  gcc_checking_assert (!summaries
-		       || flag_incremental_link == INCREMENTAL_LINK_LTO);
   if (summaries_lto)
     ggc_delete (summaries_lto);
   summaries_lto = NULL;
diff --git a/gcc/ipa-modref.h b/gcc/ipa-modref.h
index 5afa3aa..4eb696d 100644
--- a/gcc/ipa-modref.h
+++ b/gcc/ipa-modref.h
@@ -21,7 +21,7 @@ along with GCC; see the file COPYING3.  If not see
 #define IPA_MODREF_H
 
 typedef modref_tree <alias_set_type> modref_records;
-typedef unsigned char eaf_flags_t;
+typedef unsigned short eaf_flags_t;
 
 /* Single function summary.  */
 
@@ -30,15 +30,43 @@ struct GTY(()) modref_summary
   /* Load and stores in function (transitively closed to all callees)  */
   modref_records *loads;
   modref_records *stores;
+  auto_vec<modref_access_node> GTY((skip)) kills;
   auto_vec<eaf_flags_t> GTY((skip)) arg_flags;
-  bool writes_errno;
+
+  eaf_flags_t retslot_flags;
+  eaf_flags_t static_chain_flags;
+
+  unsigned writes_errno : 1;
+  /* Side effects does not include memory loads and stores which are
+     expressed using loads, stores and calls_interposable fields.  */
+  unsigned side_effects : 1;
+  /* If true function can not be CSE optimized because it may behave
+     differently even if invoked with same inputs.  */
+  unsigned nondeterministic : 1;
+  /* IF true the function may read any reachable memory but not use
+     it for anything useful.  This may happen i.e. when interposing
+     function with optimized out conditional with unoptimized one.
+
+     In this situation the loads summary is not useful for DSE but
+     it is still useful for CSE.  */
+  unsigned calls_interposable : 1;
+
+  /* Flags coputed by finalize method.  */
+
+  /* global_memory_read is not set for functions calling functions
+     with !binds_to_current_def which, after interposition, may read global
+     memory but do nothing useful with it (except for crashing if some
+     stores are optimized out.  */
+  unsigned global_memory_read : 1;
+  unsigned global_memory_written : 1;
+  unsigned try_dse : 1;
+
 
   modref_summary ();
   ~modref_summary ();
   void dump (FILE *);
   bool useful_p (int ecf_flags, bool check_flags = true);
-  bool global_memory_read_p ();
-  bool global_memory_written_p ();
+  void finalize (tree);
 };
 
 modref_summary *get_modref_function_summary (cgraph_node *func);
@@ -46,14 +74,55 @@ void ipa_modref_c_finalize ();
 void ipa_merge_modref_summary_after_inlining (cgraph_edge *e);
 
 /* All flags that are implied by the ECF_CONST functions.  */
-static const int implicit_const_eaf_flags = EAF_DIRECT | EAF_NOCLOBBER | EAF_NOESCAPE
-				     | EAF_NODIRECTESCAPE | EAF_NOREAD;
+static const int implicit_const_eaf_flags
+   = EAF_NO_DIRECT_CLOBBER | EAF_NO_INDIRECT_CLOBBER
+    | EAF_NO_DIRECT_ESCAPE | EAF_NO_INDIRECT_ESCAPE
+    | EAF_NO_DIRECT_READ | EAF_NO_INDIRECT_READ
+    | EAF_NOT_RETURNED_INDIRECTLY;
+
 /* All flags that are implied by the ECF_PURE function.  */
-static const int implicit_pure_eaf_flags = EAF_NOCLOBBER | EAF_NOESCAPE
-				    | EAF_NODIRECTESCAPE;
+static const int implicit_pure_eaf_flags
+   = EAF_NO_DIRECT_CLOBBER | EAF_NO_INDIRECT_CLOBBER
+    | EAF_NO_DIRECT_ESCAPE | EAF_NO_INDIRECT_ESCAPE;
+
 /* All flags implied when we know we can ignore stores (i.e. when handling
    call to noreturn).  */
-static const int ignore_stores_eaf_flags = EAF_DIRECT | EAF_NOCLOBBER | EAF_NOESCAPE
-				    | EAF_NODIRECTESCAPE;
+static const int ignore_stores_eaf_flags
+   = EAF_NO_DIRECT_CLOBBER | EAF_NO_INDIRECT_CLOBBER
+    | EAF_NO_DIRECT_ESCAPE | EAF_NO_INDIRECT_ESCAPE;
+
+/* Return slot is write-only.  */
+static const int implicit_retslot_eaf_flags
+   = EAF_NO_DIRECT_READ | EAF_NO_INDIRECT_READ
+     | EAF_NO_INDIRECT_ESCAPE | EAF_NO_INDIRECT_CLOBBER
+     | EAF_NOT_RETURNED_INDIRECTLY;
+
+/* If function does not bind to current def (i.e. it is inline in comdat
+   section), the modref analysis may not match the behaviour of function
+   which will be later symbol interposed to.  All side effects must match
+   however it is possible that the other function body contains more loads
+   which may trap.
+   MODREF_FLAGS are flags determined by analysis of function body while
+   FLAGS are flags known otherwise (i.e. by fnspec, pure/const attributes
+   etc.)  */
+static inline int
+interposable_eaf_flags (int modref_flags, int flags)
+{
+  /* If parameter was previously unused, we know it is only read
+     and its value is not used.  */
+  if ((modref_flags & EAF_UNUSED) && !(flags & EAF_UNUSED))
+    {
+      modref_flags &= ~EAF_UNUSED;
+      modref_flags |= EAF_NO_DIRECT_ESCAPE | EAF_NO_INDIRECT_ESCAPE
+		      | EAF_NOT_RETURNED_DIRECTLY | EAF_NOT_RETURNED_INDIRECTLY
+		      | EAF_NO_DIRECT_CLOBBER | EAF_NO_INDIRECT_CLOBBER;
+    }
+  /* We can not deterine that value is not read at all.  */
+  if ((modref_flags & EAF_NO_DIRECT_READ) && !(flags & EAF_NO_DIRECT_READ))
+    modref_flags &= ~EAF_NO_DIRECT_READ;
+  if ((modref_flags & EAF_NO_INDIRECT_READ) && !(flags & EAF_NO_INDIRECT_READ))
+    modref_flags &= ~EAF_NO_INDIRECT_READ;
+  return modref_flags;
+}
 
 #endif
diff --git a/gcc/ipa-param-manipulation.c b/gcc/ipa-param-manipulation.c
index 26b02d7..cec1dba 100644
--- a/gcc/ipa-param-manipulation.c
+++ b/gcc/ipa-param-manipulation.c
@@ -43,6 +43,9 @@ along with GCC; see the file COPYING3.  If not see
 #include "alloc-pool.h"
 #include "symbol-summary.h"
 #include "symtab-clones.h"
+#include "tree-phinodes.h"
+#include "cfgexpand.h"
+#include "attribs.h"
 
 
 /* Actual prefixes of different newly synthetized parameters.  Keep in sync
@@ -276,14 +279,42 @@ fill_vector_of_new_param_types (vec<tree> *new_types, vec<tree> *otypes,
     }
 }
 
+/* Return false if given attribute should prevent type adjustments.  */
+
+bool
+ipa_param_adjustments::type_attribute_allowed_p (tree name)
+{
+  if ((is_attribute_p ("fn spec", name) && flag_ipa_modref)
+      || is_attribute_p ("access", name)
+      || is_attribute_p ("returns_nonnull", name)
+      || is_attribute_p ("assume_aligned", name)
+      || is_attribute_p ("nocf_check", name)
+      || is_attribute_p ("warn_unused_result", name))
+    return true;
+  return false;
+}
+
+/* Return true if attribute should be dropped if parameter changed.  */
+
+static bool
+drop_type_attribute_if_params_changed_p (tree name)
+{
+  if (is_attribute_p ("fn spec", name)
+      || is_attribute_p ("access", name))
+    return true;
+  return false;
+}
+
 /* Build and return a function type just like ORIG_TYPE but with parameter
    types given in NEW_PARAM_TYPES - which can be NULL if, but only if,
    ORIG_TYPE itself has NULL TREE_ARG_TYPEs.  If METHOD2FUNC is true, also make
-   it a FUNCTION_TYPE instead of FUNCTION_TYPE.  */
+   it a FUNCTION_TYPE instead of FUNCTION_TYPE.
+   If ARG_MODIFIED is true drop attributes that are no longer up to date.  */
 
 static tree
 build_adjusted_function_type (tree orig_type, vec<tree> *new_param_types,
-			      bool method2func, bool skip_return)
+			      bool method2func, bool skip_return,
+			      bool args_modified)
 {
   tree new_arg_types = NULL;
   if (TYPE_ARG_TYPES (orig_type))
@@ -332,6 +363,20 @@ build_adjusted_function_type (tree orig_type, vec<tree> *new_param_types,
       if (skip_return)
 	TREE_TYPE (new_type) = void_type_node;
     }
+  if (args_modified && TYPE_ATTRIBUTES (new_type))
+    {
+      tree t = TYPE_ATTRIBUTES (new_type);
+      tree *last = &TYPE_ATTRIBUTES (new_type);
+      TYPE_ATTRIBUTES (new_type) = NULL;
+      for (;t; t = TREE_CHAIN (t))
+	if (!drop_type_attribute_if_params_changed_p
+		(get_attribute_name (t)))
+	  {
+	    *last = copy_node (t);
+	    TREE_CHAIN (*last) = NULL;
+	    last = &TREE_CHAIN (*last);
+	  }
+    }
 
   return new_type;
 }
@@ -458,8 +503,22 @@ ipa_param_adjustments::build_new_function_type (tree old_type,
   else
     new_param_types_p = NULL;
 
+  /* Check if any params type cares about are modified.  In this case will
+     need to drop some type attributes.  */
+  bool modified = false;
+  size_t index = 0;
+  if (m_adj_params)
+    for (tree t = TYPE_ARG_TYPES (old_type);
+	 t && (int)index < m_always_copy_start && !modified;
+	 t = TREE_CHAIN (t), index++)
+      if (index >= m_adj_params->length ()
+	  || get_original_index (index) != (int)index)
+	modified = true;
+
+
   return build_adjusted_function_type (old_type, new_param_types_p,
-				       method2func_p (old_type), m_skip_return);
+				       method2func_p (old_type), m_skip_return,
+				       modified);
 }
 
 /* Build variant of function decl ORIG_DECL which has no return value if
@@ -829,9 +888,8 @@ ipa_param_adjustments::modify_call (cgraph_edge *cs,
 	      }
 	  if (ddecl == NULL)
 	    {
-	      ddecl = make_node (DEBUG_EXPR_DECL);
-	      DECL_ARTIFICIAL (ddecl) = 1;
-	      TREE_TYPE (ddecl) = TREE_TYPE (origin);
+	      ddecl = build_debug_expr_decl (TREE_TYPE (origin));
+	      /* FIXME: Is setting the mode really necessary? */
 	      SET_DECL_MODE (ddecl, DECL_MODE (origin));
 
 	      vec_safe_push (*debug_args, origin);
@@ -972,10 +1030,12 @@ ipa_param_body_adjustments::carry_over_param (tree t)
 
 /* Populate m_dead_stmts given that DEAD_PARAM is going to be removed without
    any replacement or splitting.  REPL is the replacement VAR_SECL to base any
-   remaining uses of a removed parameter on.  */
+   remaining uses of a removed parameter on.  Push all removed SSA names that
+   are used within debug statements to DEBUGSTACK.  */
 
 void
-ipa_param_body_adjustments::mark_dead_statements (tree dead_param)
+ipa_param_body_adjustments::mark_dead_statements (tree dead_param,
+						  vec<tree> *debugstack)
 {
   /* Current IPA analyses which remove unused parameters never remove a
      non-gimple register ones which have any use except as parameters in other
@@ -987,6 +1047,7 @@ ipa_param_body_adjustments::mark_dead_statements (tree dead_param)
     return;
 
   auto_vec<tree, 4> stack;
+  hash_set<tree> used_in_debug;
   m_dead_ssas.add (parm_ddef);
   stack.safe_push (parm_ddef);
   while (!stack.is_empty ())
@@ -1014,6 +1075,11 @@ ipa_param_body_adjustments::mark_dead_statements (tree dead_param)
 	    {
 	      m_dead_stmts.add (stmt);
 	      gcc_assert (gimple_debug_bind_p (stmt));
+	      if (!used_in_debug.contains (t))
+		{
+		  used_in_debug.add (t);
+		  debugstack->safe_push (t);
+		}
 	    }
 	  else if (gimple_code (stmt) == GIMPLE_PHI)
 	    {
@@ -1046,6 +1112,156 @@ ipa_param_body_adjustments::mark_dead_statements (tree dead_param)
 	    gcc_unreachable ();
 	}
     }
+
+  if (!MAY_HAVE_DEBUG_STMTS)
+    {
+      gcc_assert (debugstack->is_empty ());
+      return;
+    }
+
+  tree dp_ddecl = build_debug_expr_decl (TREE_TYPE (dead_param));
+  /* FIXME: Is setting the mode really necessary? */
+  SET_DECL_MODE (dp_ddecl, DECL_MODE (dead_param));
+  m_dead_ssa_debug_equiv.put (parm_ddef, dp_ddecl);
+}
+
+/* Callback to walk_tree.  If REMAP is an SSA_NAME that is present in hash_map
+   passed in DATA, replace it with unshared version of what it was mapped to.
+   If an SSA argument would be remapped to NULL, the whole operation needs to
+   abort which is signaled by returning error_mark_node.  */
+
+static tree
+replace_with_mapped_expr (tree *remap, int *walk_subtrees, void *data)
+{
+  if (TYPE_P (*remap))
+    {
+      *walk_subtrees = 0;
+      return 0;
+    }
+  if (TREE_CODE (*remap) != SSA_NAME)
+    return 0;
+
+  *walk_subtrees = 0;
+
+  hash_map<tree, tree> *equivs = (hash_map<tree, tree> *) data;
+  if (tree *p = equivs->get (*remap))
+    {
+      if (!*p)
+	return error_mark_node;
+      *remap = unshare_expr (*p);
+    }
+  return 0;
+}
+
+/* Replace all occurances of SSAs in m_dead_ssa_debug_equiv in t with what they
+   are mapped to.  */
+
+void
+ipa_param_body_adjustments::remap_with_debug_expressions (tree *t)
+{
+  /* If *t is an SSA_NAME which should have its debug statements reset, it is
+     mapped to NULL in the hash_map.
+
+     It is perhaps simpler to handle the SSA_NAME cases directly and only
+     invoke walk_tree on more complex expressions.  When
+     remap_with_debug_expressions is called from tree-inline.c, a to-be-reset
+     SSA_NAME can be an operand to such expressions and the entire debug
+     variable we are remapping should be reset.  This is signaled by walk_tree
+     returning error_mark_node and done by setting *t to NULL.  */
+  if (TREE_CODE (*t) == SSA_NAME)
+    {
+      if (tree *p = m_dead_ssa_debug_equiv.get (*t))
+	*t = *p;
+    }
+  else if (walk_tree (t, replace_with_mapped_expr,
+		      &m_dead_ssa_debug_equiv, NULL) == error_mark_node)
+    *t = NULL_TREE;
+}
+
+/* For an SSA_NAME DEAD_SSA which is about to be DCEd because it is based on a
+   useless parameter, prepare an expression that should represent it in
+   debug_binds in the cloned function and add a mapping from DEAD_SSA to
+   m_dead_ssa_debug_equiv.  That mapping is to NULL when the associated
+   debug_statement has to be reset instead.  In such case return false,
+   ottherwise return true.  If DEAD_SSA comes from a basic block which is not
+   about to be copied, ignore it and return true.  */
+
+bool
+ipa_param_body_adjustments::prepare_debug_expressions (tree dead_ssa)
+{
+  gcc_checking_assert (m_dead_ssas.contains (dead_ssa));
+  if (tree *d = m_dead_ssa_debug_equiv.get (dead_ssa))
+    return (*d != NULL_TREE);
+
+  gcc_assert (!SSA_NAME_IS_DEFAULT_DEF (dead_ssa));
+  gimple *def = SSA_NAME_DEF_STMT (dead_ssa);
+  if (m_id->blocks_to_copy
+      && !bitmap_bit_p (m_id->blocks_to_copy, gimple_bb (def)->index))
+    return true;
+
+  if (gimple_code (def) == GIMPLE_PHI)
+    {
+      /* In theory, we could ignore all SSAs coming from BBs not in
+	 m_id->blocks_to_copy but at the time of the writing this code that
+	 should never really be the case because only fnsplit uses that bitmap,
+	 so don't bother.  */
+      tree value = degenerate_phi_result (as_a <gphi *> (def));
+      if (!value
+	  || (m_dead_ssas.contains (value)
+	      && !prepare_debug_expressions (value)))
+	{
+	  m_dead_ssa_debug_equiv.put (dead_ssa, NULL_TREE);
+	  return false;
+	}
+
+      gcc_assert (TREE_CODE (value) == SSA_NAME);
+      tree *d = m_dead_ssa_debug_equiv.get (value);
+      m_dead_ssa_debug_equiv.put (dead_ssa, *d);
+      return true;
+    }
+
+  bool lost = false;
+  use_operand_p use_p;
+  ssa_op_iter oi;
+  FOR_EACH_PHI_OR_STMT_USE (use_p, def, oi, SSA_OP_USE)
+    {
+      tree use = USE_FROM_PTR (use_p);
+      if (m_dead_ssas.contains (use)
+	  && !prepare_debug_expressions (use))
+	{
+	  lost = true;
+	  break;
+	}
+    }
+
+  if (lost)
+    {
+      m_dead_ssa_debug_equiv.put (dead_ssa, NULL_TREE);
+      return false;
+    }
+
+  if (is_gimple_assign (def))
+    {
+      gcc_assert (!gimple_clobber_p (def));
+      if (gimple_assign_copy_p (def)
+	  && TREE_CODE (gimple_assign_rhs1 (def)) == SSA_NAME)
+	{
+	  tree *d = m_dead_ssa_debug_equiv.get (gimple_assign_rhs1 (def));
+	  m_dead_ssa_debug_equiv.put (dead_ssa, *d);
+	  return (*d != NULL_TREE);
+	}
+
+      tree val
+	= unshare_expr_without_location (gimple_assign_rhs_to_tree (def));
+      remap_with_debug_expressions (&val);
+
+      tree vexpr = build_debug_expr_decl (TREE_TYPE (val));
+      m_dead_stmt_debug_equiv.put (def, val);
+      m_dead_ssa_debug_equiv.put (dead_ssa, vexpr);
+      return true;
+    }
+  else
+    gcc_unreachable ();
 }
 
 /* Common initialization performed by all ipa_param_body_adjustments
@@ -1137,6 +1353,32 @@ ipa_param_body_adjustments::common_initialization (tree old_fndecl,
 	gcc_unreachable ();
     }
 
+  if (tree_map)
+    {
+      /* Do not treat parameters which were replaced with a constant as
+	 completely vanished.  */
+      auto_vec <int, 16> index_mapping;
+      bool need_remap = false;
+
+      if (m_id)
+	{
+	  clone_info *cinfo = clone_info::get (m_id->src_node);
+	  if (cinfo && cinfo->param_adjustments)
+	    {
+	      cinfo->param_adjustments->get_updated_indices (&index_mapping);
+	      need_remap = true;
+	    }
+	}
+
+      for (unsigned i = 0; i < tree_map->length (); i++)
+	{
+	  int parm_num = (*tree_map)[i]->parm_num;
+	  gcc_assert (parm_num >= 0);
+	  if (need_remap)
+	    parm_num = index_mapping[parm_num];
+	  kept[parm_num] = true;
+	}
+    }
 
   /* As part of body modifications, we will also have to replace remaining uses
      of remaining uses of removed PARM_DECLs (which do not however use the
@@ -1149,68 +1391,41 @@ ipa_param_body_adjustments::common_initialization (tree old_fndecl,
      replace_removed_params_ssa_names or perform_cfun_body_modifications when
      you construct with ID not equal to NULL.  */
 
+  auto_vec<tree, 8> ssas_to_process_debug;
   unsigned op_len = m_oparms.length ();
   for (unsigned i = 0; i < op_len; i++)
     if (!kept[i])
       {
 	if (m_id)
 	  {
-	    if (!m_id->decl_map->get (m_oparms[i]))
-	      {
-		tree var = copy_decl_to_var (m_oparms[i], m_id);
-		insert_decl_map (m_id, m_oparms[i], var);
-		/* Declare this new variable.  */
-		DECL_CHAIN (var) = *vars;
-		*vars = var;
-
-		/* If this is not a split but a real removal, init hash sets
-		   that will guide what not to copy to the new body.  */
-		if (!split[i])
-		  mark_dead_statements (m_oparms[i]);
-	      }
+	    gcc_assert (!m_id->decl_map->get (m_oparms[i]));
+	    tree var = copy_decl_to_var (m_oparms[i], m_id);
+	    insert_decl_map (m_id, m_oparms[i], var);
+	    /* Declare this new variable.  */
+	    DECL_CHAIN (var) = *vars;
+	    *vars = var;
+
+	    /* If this is not a split but a real removal, init hash sets
+	       that will guide what not to copy to the new body.  */
+	    if (!split[i])
+	      mark_dead_statements (m_oparms[i], &ssas_to_process_debug);
+	    if (MAY_HAVE_DEBUG_STMTS
+		&& is_gimple_reg (m_oparms[i]))
+	      m_reset_debug_decls.safe_push (m_oparms[i]);
 	  }
 	else
 	  {
 	    m_removed_decls.safe_push (m_oparms[i]);
 	    m_removed_map.put (m_oparms[i], m_removed_decls.length () - 1);
+	    if (MAY_HAVE_DEBUG_STMTS
+		&& !kept[i]
+		&& is_gimple_reg (m_oparms[i]))
+	      m_reset_debug_decls.safe_push (m_oparms[i]);
 	  }
       }
 
-  if (!MAY_HAVE_DEBUG_STMTS)
-    return;
-
-  /* Finally, when generating debug info, we fill vector m_reset_debug_decls
-    with removed parameters declarations.  We do this in order to re-map their
-    debug bind statements and create debug decls for them.  */
-
-  if (tree_map)
-    {
-      /* Do not output debuginfo for parameter declarations as if they vanished
-	 when they were in fact replaced by a constant.  */
-      auto_vec <int, 16> index_mapping;
-      bool need_remap = false;
-      clone_info *info = clone_info::get (m_id->src_node);
-
-      if (m_id && info && info->param_adjustments)
-	{
-	  ipa_param_adjustments *prev_adjustments = info->param_adjustments;
-	  prev_adjustments->get_updated_indices (&index_mapping);
-	  need_remap = true;
-	}
-
-      for (unsigned i = 0; i < tree_map->length (); i++)
-	{
-	  int parm_num = (*tree_map)[i]->parm_num;
-	  gcc_assert (parm_num >= 0);
-	  if (need_remap)
-	    parm_num = index_mapping[parm_num];
-	  kept[parm_num] = true;
-	}
-    }
-
-  for (unsigned i = 0; i < op_len; i++)
-    if (!kept[i] && is_gimple_reg (m_oparms[i]))
-      m_reset_debug_decls.safe_push (m_oparms[i]);
+  while (!ssas_to_process_debug.is_empty ())
+    prepare_debug_expressions (ssas_to_process_debug.pop ());
 }
 
 /* Constructor of ipa_param_body_adjustments from a simple list of
@@ -1224,9 +1439,9 @@ ipa_param_body_adjustments
 			      tree fndecl)
   : m_adj_params (adj_params), m_adjustments (NULL), m_reset_debug_decls (),
     m_split_modifications_p (false), m_dead_stmts (), m_dead_ssas (),
-    m_fndecl (fndecl), m_id (NULL), m_oparms (), m_new_decls (),
-    m_new_types (), m_replacements (), m_removed_decls (), m_removed_map (),
-    m_method2func (false)
+    m_dead_ssa_debug_equiv (), m_dead_stmt_debug_equiv (), m_fndecl (fndecl),
+    m_id (NULL), m_oparms (), m_new_decls (), m_new_types (), m_replacements (),
+    m_removed_decls (), m_removed_map (), m_method2func (false)
 {
   common_initialization (fndecl, NULL, NULL);
 }
@@ -1241,7 +1456,8 @@ ipa_param_body_adjustments
 			      tree fndecl)
   : m_adj_params (adjustments->m_adj_params), m_adjustments (adjustments),
     m_reset_debug_decls (), m_split_modifications_p (false), m_dead_stmts (),
-    m_dead_ssas (), m_fndecl (fndecl), m_id (NULL), m_oparms (), m_new_decls (),
+    m_dead_ssas (), m_dead_ssa_debug_equiv (), m_dead_stmt_debug_equiv (),
+    m_fndecl (fndecl), m_id (NULL), m_oparms (), m_new_decls (),
     m_new_types (), m_replacements (), m_removed_decls (), m_removed_map (),
     m_method2func (false)
 {
@@ -1264,8 +1480,9 @@ ipa_param_body_adjustments
 			      vec<ipa_replace_map *, va_gc> *tree_map)
   : m_adj_params (adjustments->m_adj_params), m_adjustments (adjustments),
     m_reset_debug_decls (), m_split_modifications_p (false), m_dead_stmts (),
-    m_dead_ssas (),m_fndecl (fndecl), m_id (id), m_oparms (), m_new_decls (),
-    m_new_types (), m_replacements (), m_removed_decls (), m_removed_map (),
+    m_dead_ssas (), m_dead_ssa_debug_equiv (), m_dead_stmt_debug_equiv (),
+    m_fndecl (fndecl), m_id (id), m_oparms (), m_new_decls (), m_new_types (),
+    m_replacements (), m_removed_decls (), m_removed_map (),
     m_method2func (false)
 {
   common_initialization (old_fndecl, vars, tree_map);
@@ -1305,12 +1522,23 @@ ipa_param_body_adjustments::modify_formal_parameters ()
   if (fndecl_built_in_p (m_fndecl))
     set_decl_built_in_function (m_fndecl, NOT_BUILT_IN, 0);
 
+  bool modified = false;
+  size_t index = 0;
+  if (m_adj_params)
+    for (tree t = TYPE_ARG_TYPES (orig_type);
+	 t && !modified;
+	 t = TREE_CHAIN (t), index++)
+      if (index >= m_adj_params->length ()
+	  || (*m_adj_params)[index].op != IPA_PARAM_OP_COPY
+	  || (*m_adj_params)[index].base_index != index)
+	modified = true;
+
   /* At this point, removing return value is only implemented when going
      through tree_function_versioning, not when modifying function body
      directly.  */
   gcc_assert (!m_adjustments || !m_adjustments->m_skip_return);
   tree new_type = build_adjusted_function_type (orig_type, &m_new_types,
-						m_method2func, false);
+						m_method2func, false, modified);
 
   TREE_TYPE (m_fndecl) = new_type;
   DECL_VIRTUAL_P (m_fndecl) = 0;
@@ -2055,11 +2283,10 @@ ipa_param_body_adjustments::reset_debug_stmts ()
 	    gcc_assert (is_gimple_debug (stmt));
 	    if (vexpr == NULL && gsip != NULL)
 	      {
-		vexpr = make_node (DEBUG_EXPR_DECL);
-		def_temp = gimple_build_debug_source_bind (vexpr, decl, NULL);
-		DECL_ARTIFICIAL (vexpr) = 1;
-		TREE_TYPE (vexpr) = TREE_TYPE (name);
+		vexpr = build_debug_expr_decl (TREE_TYPE (name));
+		/* FIXME: Is setting the mode really necessary? */
 		SET_DECL_MODE (vexpr, DECL_MODE (decl));
+		def_temp = gimple_build_debug_source_bind (vexpr, decl, NULL);
 		gsi_insert_before (gsip, def_temp, GSI_SAME_STMT);
 	      }
 	    if (vexpr)
diff --git a/gcc/ipa-param-manipulation.h b/gcc/ipa-param-manipulation.h
index afcbc09..5adf8a2 100644
--- a/gcc/ipa-param-manipulation.h
+++ b/gcc/ipa-param-manipulation.h
@@ -254,6 +254,7 @@ public:
   /* If true, make the function not return any value.  */
   bool m_skip_return;
 
+  static bool type_attribute_allowed_p (tree);
 private:
   ipa_param_adjustments () {}
 
@@ -328,6 +329,9 @@ public:
 			   gimple *orig_stmt);
   /* Return the new chain of parameters.  */
   tree get_new_param_chain ();
+  /* Replace all occurances of SSAs in m_dead_ssa_debug_equiv in t with what
+     they are mapped to.  */
+  void remap_with_debug_expressions (tree *t);
 
   /* Pointers to data structures defining how the function should be
      modified.  */
@@ -348,6 +352,12 @@ public:
   hash_set<gimple *> m_dead_stmts;
   hash_set<tree> m_dead_ssas;
 
+  /* Mapping from DCEd SSAs to what their potential debug_binds should be.  */
+  hash_map<tree, tree> m_dead_ssa_debug_equiv;
+  /* Mapping from DCEd statements to debug expressions that will be placed on
+     the RHS of debug statement that will replace this one.  */
+  hash_map<gimple *, tree> m_dead_stmt_debug_equiv;
+
 private:
   void common_initialization (tree old_fndecl, tree *vars,
 			      vec<ipa_replace_map *, va_gc> *tree_map);
@@ -361,7 +371,8 @@ private:
   bool modify_call_stmt (gcall **stmt_p, gimple *orig_stmt);
   bool modify_cfun_body ();
   void reset_debug_stmts ();
-  void mark_dead_statements (tree dead_param);
+  void mark_dead_statements (tree dead_param, vec<tree> *debugstack);
+  bool prepare_debug_expressions (tree dead_ssa);
 
   /* Declaration of the function that is being transformed.  */
 
diff --git a/gcc/ipa-predicate.c b/gcc/ipa-predicate.c
index e4b11ec..48318cc 100644
--- a/gcc/ipa-predicate.c
+++ b/gcc/ipa-predicate.c
@@ -69,7 +69,7 @@ expr_eval_ops_equal_p (expr_eval_ops ops1, expr_eval_ops ops2)
    sane.  */
 
 void
-predicate::add_clause (conditions conditions, clause_t new_clause)
+ipa_predicate::add_clause (conditions conditions, clause_t new_clause)
 {
   int i;
   int i2;
@@ -81,7 +81,7 @@ predicate::add_clause (conditions conditions, clause_t new_clause)
     return;
 
   /* False clause makes the whole predicate false.  Kill the other variants.  */
-  if (new_clause == (1 << predicate::false_condition))
+  if (new_clause == (1 << ipa_predicate::false_condition))
     {
       *this = false;
       return;
@@ -90,7 +90,7 @@ predicate::add_clause (conditions conditions, clause_t new_clause)
     return;
 
   /* No one should be silly enough to add false into nontrivial clauses.  */
-  gcc_checking_assert (!(new_clause & (1 << predicate::false_condition)));
+  gcc_checking_assert (!(new_clause & (1 << ipa_predicate::false_condition)));
 
   /* Look where to insert the new_clause.  At the same time prune out
      new_clauses of P that are implied by the new new_clause and thus
@@ -123,13 +123,13 @@ predicate::add_clause (conditions conditions, clause_t new_clause)
   /* Look for clauses that are obviously true.  I.e.
      op0 == 5 || op0 != 5.  */
   if (conditions)
-    for (c1 = predicate::first_dynamic_condition;
+    for (c1 = ipa_predicate::first_dynamic_condition;
 	 c1 < num_conditions; c1++)
       {
 	condition *cc1;
 	if (!(new_clause & (1 << c1)))
 	  continue;
-	cc1 = &(*conditions)[c1 - predicate::first_dynamic_condition];
+	cc1 = &(*conditions)[c1 - ipa_predicate::first_dynamic_condition];
 	/* We have no way to represent !changed and !is_not_constant
 	   and thus there is no point for looking for them.  */
 	if (cc1->code == changed || cc1->code == is_not_constant)
@@ -138,7 +138,7 @@ predicate::add_clause (conditions conditions, clause_t new_clause)
 	  if (new_clause & (1 << c2))
 	    {
 	      condition *cc2 =
-		&(*conditions)[c2 - predicate::first_dynamic_condition];
+		&(*conditions)[c2 - ipa_predicate::first_dynamic_condition];
 	      if (cc1->operand_num == cc2->operand_num
 		  && vrp_operand_equal_p (cc1->val, cc2->val)
 		  && cc2->code != is_not_constant
@@ -170,8 +170,8 @@ predicate::add_clause (conditions conditions, clause_t new_clause)
 
 /* Do THIS &= P.  */
 
-predicate &
-predicate::operator &= (const predicate &p)
+ipa_predicate &
+ipa_predicate::operator &= (const ipa_predicate &p)
 {
   /* Avoid busy work.  */
   if (p == false || *this == true)
@@ -184,13 +184,13 @@ predicate::operator &= (const predicate &p)
 
   int i;
 
-  /* See how far predicates match.  */
+  /* See how far ipa_predicates match.  */
   for (i = 0; m_clause[i] && m_clause[i] == p.m_clause[i]; i++)
     {
       gcc_checking_assert (i < max_clauses);
     }
 
-  /* Combine the predicates rest.  */
+  /* Combine the ipa_predicates rest.  */
   for (; p.m_clause[i]; i++)
     {
       gcc_checking_assert (i < max_clauses);
@@ -203,9 +203,9 @@ predicate::operator &= (const predicate &p)
 
 /* Return THIS | P2.  */
 
-predicate
-predicate::or_with (conditions conditions,
-	            const predicate &p) const
+ipa_predicate
+ipa_predicate::or_with (conditions conditions,
+			const ipa_predicate &p) const
 {
   /* Avoid busy work.  */
   if (p == false || *this == true || *this == p)
@@ -214,7 +214,7 @@ predicate::or_with (conditions conditions,
     return p;
 
   /* OK, combine the predicates.  */
-  predicate out = true;
+  ipa_predicate out = true;
 
   for (int i = 0; m_clause[i]; i++)
     for (int j = 0; p.m_clause[j]; j++)
@@ -230,7 +230,7 @@ predicate::or_with (conditions conditions,
    if predicate P is known to be false.  */
 
 bool
-predicate::evaluate (clause_t possible_truths) const
+ipa_predicate::evaluate (clause_t possible_truths) const
 {
   int i;
 
@@ -238,7 +238,7 @@ predicate::evaluate (clause_t possible_truths) const
   if (*this == true)
     return true;
 
-  gcc_assert (!(possible_truths & (1 << predicate::false_condition)));
+  gcc_assert (!(possible_truths & (1 << ipa_predicate::false_condition)));
 
   /* See if we can find clause we can disprove.  */
   for (i = 0; m_clause[i]; i++)
@@ -254,7 +254,7 @@ predicate::evaluate (clause_t possible_truths) const
    instruction will be recomputed per invocation of the inlined call.  */
 
 int
-predicate::probability (conditions conds,
+ipa_predicate::probability (conditions conds,
 	                clause_t possible_truths,
 	                vec<inline_param_summary> inline_param_summary) const
 {
@@ -268,7 +268,7 @@ predicate::probability (conditions conds,
   if (*this == false)
     return 0;
 
-  gcc_assert (!(possible_truths & (1 << predicate::false_condition)));
+  gcc_assert (!(possible_truths & (1 << ipa_predicate::false_condition)));
 
   /* See if we can find clause we can disprove.  */
   for (i = 0; m_clause[i]; i++)
@@ -285,11 +285,11 @@ predicate::probability (conditions conds,
 	  for (i2 = 0; i2 < num_conditions; i2++)
 	    if ((m_clause[i] & possible_truths) & (1 << i2))
 	      {
-		if (i2 >= predicate::first_dynamic_condition)
+		if (i2 >= ipa_predicate::first_dynamic_condition)
 		  {
 		    condition *c =
-		      &(*conds)[i2 - predicate::first_dynamic_condition];
-		    if (c->code == predicate::changed
+		      &(*conds)[i2 - ipa_predicate::first_dynamic_condition];
+		    if (c->code == ipa_predicate::changed
 			&& (c->operand_num <
 			    (int) inline_param_summary.length ()))
 		      {
@@ -318,13 +318,13 @@ void
 dump_condition (FILE *f, conditions conditions, int cond)
 {
   condition *c;
-  if (cond == predicate::false_condition)
+  if (cond == ipa_predicate::false_condition)
     fprintf (f, "false");
-  else if (cond == predicate::not_inlined_condition)
+  else if (cond == ipa_predicate::not_inlined_condition)
     fprintf (f, "not inlined");
   else
     {
-      c = &(*conditions)[cond - predicate::first_dynamic_condition];
+      c = &(*conditions)[cond - ipa_predicate::first_dynamic_condition];
       fprintf (f, "op%i", c->operand_num);
       if (c->agg_contents)
 	fprintf (f, "[%soffset: " HOST_WIDE_INT_PRINT_DEC "]",
@@ -406,12 +406,12 @@ dump_condition (FILE *f, conditions conditions, int cond)
 	  fprintf (f, ")");
 	}
 
-      if (c->code == predicate::is_not_constant)
+      if (c->code == ipa_predicate::is_not_constant)
 	{
 	  fprintf (f, " not constant");
 	  return;
 	}
-      if (c->code == predicate::changed)
+      if (c->code == ipa_predicate::changed)
 	{
 	  fprintf (f, " changed");
 	  return;
@@ -432,7 +432,7 @@ dump_clause (FILE *f, conditions conds, clause_t clause)
   fprintf (f, "(");
   if (!clause)
     fprintf (f, "true");
-  for (i = 0; i < predicate::num_conditions; i++)
+  for (i = 0; i < ipa_predicate::num_conditions; i++)
     if (clause & (1 << i))
       {
 	if (found)
@@ -445,10 +445,10 @@ dump_clause (FILE *f, conditions conds, clause_t clause)
 
 
 /* Dump THIS to F.  CONDS a vector of conditions used when evaluating
-   predicates.  When NL is true new line is output at the end of dump.  */
+   ipa_predicates.  When NL is true new line is output at the end of dump.  */
 
 void
-predicate::dump (FILE *f, conditions conds, bool nl) const
+ipa_predicate::dump (FILE *f, conditions conds, bool nl) const
 {
   int i;
   if (*this == true)
@@ -466,7 +466,7 @@ predicate::dump (FILE *f, conditions conds, bool nl) const
 
 
 void
-predicate::debug (conditions conds) const
+ipa_predicate::debug (conditions conds) const
 {
   dump (stderr, conds);
 }
@@ -476,11 +476,11 @@ predicate::debug (conditions conds) const
    POSSIBLE_TRUTHS is clause of possible truths in the duplicated node,
    INFO is inline summary of the duplicated node.  */
 
-predicate
-predicate::remap_after_duplication (clause_t possible_truths)
+ipa_predicate
+ipa_predicate::remap_after_duplication (clause_t possible_truths)
 {
   int j;
-  predicate out = true;
+  ipa_predicate out = true;
   for (j = 0; m_clause[j]; j++)
     if (!(possible_truths & m_clause[j]))
       return false;
@@ -503,26 +503,26 @@ predicate::remap_after_duplication (clause_t possible_truths)
    because they might not be preserved (and should be considered offset zero
    for other purposes).  */
 
-predicate
-predicate::remap_after_inlining (class ipa_fn_summary *info,
+ipa_predicate
+ipa_predicate::remap_after_inlining (class ipa_fn_summary *info,
 				 class ipa_node_params *params_summary,
 				 class ipa_fn_summary *callee_info,
 				 const vec<int> &operand_map,
 				 const vec<HOST_WIDE_INT> &offset_map,
 				 clause_t possible_truths,
-				 const predicate &toplev_predicate)
+				 const ipa_predicate &toplev_predicate)
 {
   int i;
-  predicate out = true;
+  ipa_predicate out = true;
 
-  /* True predicate is easy.  */
+  /* True ipa_predicate is easy.  */
   if (*this == true)
     return toplev_predicate;
   for (i = 0; m_clause[i]; i++)
     {
       clause_t clause = m_clause[i];
       int cond;
-      predicate clause_predicate = false;
+      ipa_predicate clause_predicate = false;
 
       gcc_assert (i < max_clauses);
 
@@ -530,16 +530,15 @@ predicate::remap_after_inlining (class ipa_fn_summary *info,
 	/* Do we have condition we can't disprove?   */
 	if (clause & possible_truths & (1 << cond))
 	  {
-	    predicate cond_predicate;
+	    ipa_predicate cond_predicate;
 	    /* Work out if the condition can translate to predicate in the
 	       inlined function.  */
-	    if (cond >= predicate::first_dynamic_condition)
+	    if (cond >= ipa_predicate::first_dynamic_condition)
 	      {
 		struct condition *c;
 
-		c = &(*callee_info->conds)[cond
-					   -
-					   predicate::first_dynamic_condition];
+		int index = cond - ipa_predicate::first_dynamic_condition;
+		c = &(*callee_info->conds)[index];
 		/* See if we can remap condition operand to caller's operand.
 		   Otherwise give up.  */
 		if (!operand_map.exists ()
@@ -576,7 +575,7 @@ predicate::remap_after_inlining (class ipa_fn_summary *info,
 	    /* Fixed conditions remains same, construct single
 	       condition predicate.  */
 	    else
-	      cond_predicate = predicate::predicate_testing_cond (cond);
+	      cond_predicate = ipa_predicate::predicate_testing_cond (cond);
 	    clause_predicate = clause_predicate.or_with (info->conds,
 					                 cond_predicate);
 	  }
@@ -590,7 +589,7 @@ predicate::remap_after_inlining (class ipa_fn_summary *info,
 /* Read predicate from IB.  */
 
 void
-predicate::stream_in (class lto_input_block *ib)
+ipa_predicate::stream_in (class lto_input_block *ib)
 {
   clause_t clause;
   int k = 0;
@@ -611,7 +610,7 @@ predicate::stream_in (class lto_input_block *ib)
 /* Write predicate P to OB.  */
 
 void
-predicate::stream_out (struct output_block *ob)
+ipa_predicate::stream_out (struct output_block *ob)
 {
   int j;
   for (j = 0; m_clause[j]; j++)
@@ -629,7 +628,7 @@ predicate::stream_out (struct output_block *ob)
    aggregate it is.  It can be NULL, which means this not a load from an
    aggregate.  */
 
-predicate
+ipa_predicate
 add_condition (class ipa_fn_summary *summary,
 	       class ipa_node_params *params_summary,
 	       int operand_num,
@@ -669,10 +668,10 @@ add_condition (class ipa_fn_summary *summary,
 	  && c->agg_contents == agg_contents
 	  && expr_eval_ops_equal_p (c->param_ops, param_ops)
 	  && (!agg_contents || (c->offset == offset && c->by_ref == by_ref)))
-	return predicate::predicate_testing_cond (i);
+	return ipa_predicate::predicate_testing_cond (i);
     }
   /* Too many conditions.  Give up and return constant true.  */
-  if (i == predicate::num_conditions - predicate::first_dynamic_condition)
+  if (i == ipa_predicate::num_conditions - ipa_predicate::first_dynamic_condition)
     return true;
 
   new_cond.operand_num = operand_num;
@@ -694,5 +693,5 @@ add_condition (class ipa_fn_summary *summary,
 
   vec_safe_push (summary->conds, new_cond);
 
-  return predicate::predicate_testing_cond (i);
+  return ipa_predicate::predicate_testing_cond (i);
 }
diff --git a/gcc/ipa-predicate.h b/gcc/ipa-predicate.h
index ac52b54..190dced 100644
--- a/gcc/ipa-predicate.h
+++ b/gcc/ipa-predicate.h
@@ -111,7 +111,7 @@ typedef vec<condition, va_gc> *conditions;
    is not.  */
 
 typedef uint32_t clause_t;
-class predicate
+class ipa_predicate
 {
 public:
   enum predicate_conditions
@@ -138,7 +138,7 @@ public:
 
 
   /* Initialize predicate either to true of false depending on P.  */
-  inline predicate (bool p = true)
+  inline ipa_predicate (bool p = true)
     {
       if (p)
         /* True predicate.  */
@@ -149,42 +149,42 @@ public:
     }
 
   /* Sanity check that we do not mix pointers to predicates with predicates.  */
-  inline predicate (predicate *)
+  inline ipa_predicate (ipa_predicate *)
     {
       gcc_unreachable ();
     }
 
   /* Return predicate testing condition I.  */
-  static inline predicate predicate_testing_cond (int i)
+  static inline ipa_predicate predicate_testing_cond (int i)
     {
-      class predicate p;
+      ipa_predicate p;
       p.set_to_cond (i + first_dynamic_condition);
       return p;
     }
 
   /* Return predicate testing that function was not inlined.  */
-  static predicate not_inlined (void)
+  static ipa_predicate not_inlined (void)
     {
-      class predicate p;
+      ipa_predicate p;
       p.set_to_cond (not_inlined_condition);
       return p;
     }
 
-  /* Compute logical and of predicates.  */
-  predicate & operator &= (const predicate &);
-  inline predicate operator &(const predicate &p) const
+  /* Compute logical and of ipa_predicates.  */
+  ipa_predicate & operator &= (const ipa_predicate &);
+  inline ipa_predicate operator &(const ipa_predicate &p) const
     {
-      predicate ret = *this;
+      ipa_predicate ret = *this;
       ret &= p;
       return ret;
     }
 
-  /* Compute logical or of predicates.  This is not operator because
+  /* Compute logical or of ipa_predicates.  This is not operator because
      extra parameter CONDITIONS is needed  */
-  predicate or_with (conditions, const predicate &) const;
+  ipa_predicate or_with (conditions, const ipa_predicate &) const;
 
-  /* Return true if predicates are known to be equal.  */
-  inline bool operator==(const predicate &p2) const
+  /* Return true if ipa_predicates are known to be equal.  */
+  inline bool operator==(const ipa_predicate &p2) const
     {
       int i;
       for (i = 0; m_clause[i]; i++)
@@ -215,7 +215,7 @@ public:
       return false;
     }
 
-  inline bool operator!=(const predicate &p2) const
+  inline bool operator!=(const ipa_predicate &p2) const
     {
       return !(*this == p2);
     }
@@ -236,18 +236,19 @@ public:
   void dump (FILE *f, conditions, bool nl=true) const;
   void DEBUG_FUNCTION debug (conditions) const;
 
-  /* Return predicate equal to THIS after duplication.  */
-  predicate remap_after_duplication (clause_t);
+  /* Return ipa_predicate equal to THIS after duplication.  */
+  ipa_predicate remap_after_duplication (clause_t);
 
-  /* Return predicate equal to THIS after inlining.  */
-  predicate remap_after_inlining (class ipa_fn_summary *,
-		  		  class ipa_node_params *params_summary,
-			          class ipa_fn_summary *,
-				  const vec<int> &, const vec<HOST_WIDE_INT> &,
-				  clause_t, const predicate &);
+  /* Return ipa_predicate equal to THIS after inlining.  */
+  ipa_predicate remap_after_inlining (class ipa_fn_summary *,
+				      ipa_node_params *params_summary,
+				      ipa_fn_summary *,
+				      const vec<int> &,
+				      const vec<HOST_WIDE_INT> &,
+				      clause_t, const ipa_predicate &);
 
-  void stream_in (class lto_input_block *);
-  void stream_out (struct output_block *);
+  void stream_in (lto_input_block *);
+  void stream_out (output_block *);
 
 private:
   static const int max_clauses = 8;
@@ -264,9 +265,9 @@ private:
 };
 
 void dump_condition (FILE *f, conditions conditions, int cond);
-predicate add_condition (class ipa_fn_summary *summary,
-			 class ipa_node_params *params_summary,
-	       		 int operand_num,
-			 tree type, struct agg_position_info *aggpos,
-			 enum tree_code code, tree val,
-			 expr_eval_ops param_ops = NULL);
+ipa_predicate add_condition (ipa_fn_summary *summary,
+			     ipa_node_params *params_summary,
+			     int operand_num,
+			     tree type, struct agg_position_info *aggpos,
+			     enum tree_code code, tree val,
+			     expr_eval_ops param_ops = NULL);
diff --git a/gcc/ipa-prop.c b/gcc/ipa-prop.c
index 443f21c..e85df09 100644
--- a/gcc/ipa-prop.c
+++ b/gcc/ipa-prop.c
@@ -1421,8 +1421,6 @@ compute_complex_assign_jump_func (struct ipa_func_body_info *fbi,
   if (TREE_CODE (op1) != ADDR_EXPR)
     return;
   op1 = TREE_OPERAND (op1, 0);
-  if (TREE_CODE (TREE_TYPE (op1)) != RECORD_TYPE)
-    return;
   base = get_ref_base_and_extent_hwi (op1, &offset, &size, &reverse);
   offset_int mem_offset;
   if (!base
diff --git a/gcc/ipa-pure-const.c b/gcc/ipa-pure-const.c
index a84a4eb..a332940 100644
--- a/gcc/ipa-pure-const.c
+++ b/gcc/ipa-pure-const.c
@@ -62,6 +62,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "ipa-prop.h"
 #include "ipa-fnsummary.h"
 #include "symtab-thunks.h"
+#include "dbgcnt.h"
 
 /* Lattice values for const and pure functions.  Everything starts out
    being const, then may drop to pure and then neither depending on
@@ -275,7 +276,7 @@ warn_function_noreturn (tree decl)
   static hash_set<tree> *warned_about;
   if (!lang_hooks.missing_noreturn_ok_p (decl)
       && targetm.warn_func_return (decl))
-    warned_about 
+    warned_about
       = suggest_attribute (OPT_Wsuggest_attribute_noreturn, original_decl,
 			   true, warned_about, "noreturn");
 }
@@ -286,7 +287,7 @@ warn_function_cold (tree decl)
   tree original_decl = decl;
 
   static hash_set<tree> *warned_about;
-  warned_about 
+  warned_about
     = suggest_attribute (OPT_Wsuggest_attribute_cold, original_decl,
 			 true, warned_about, "cold");
 }
@@ -506,11 +507,10 @@ worse_state (enum pure_const_state_e *state, bool *looping,
   *looping = MAX (*looping, looping2);
 }
 
-/* Recognize special cases of builtins that are by themselves not pure or const
+/* Recognize special cases of builtins that are by themselves not const
    but function using them is.  */
-static bool
-special_builtin_state (enum pure_const_state_e *state, bool *looping,
-		       tree callee)
+bool
+builtin_safe_for_const_function_p (bool *looping, tree callee)
 {
   if (DECL_BUILT_IN_CLASS (callee) == BUILT_IN_NORMAL)
     switch (DECL_FUNCTION_CODE (callee))
@@ -529,12 +529,12 @@ special_builtin_state (enum pure_const_state_e *state, bool *looping,
       case BUILT_IN_APPLY_ARGS:
       case BUILT_IN_ASAN_BEFORE_DYNAMIC_INIT:
       case BUILT_IN_ASAN_AFTER_DYNAMIC_INIT:
+      case BUILT_IN_DWARF_CFA:
+      case BUILT_IN_RETURN_ADDRESS:
 	*looping = false;
-	*state = IPA_CONST;
 	return true;
       case BUILT_IN_PREFETCH:
 	*looping = true;
-	*state = IPA_CONST;
 	return true;
       default:
 	break;
@@ -592,17 +592,16 @@ check_call (funct_state local, gcall *call, bool ipa)
      graph.  */
   if (callee_t)
     {
-      enum pure_const_state_e call_state;
       bool call_looping;
 
       if (gimple_call_builtin_p (call, BUILT_IN_NORMAL)
 	  && !nonfreeing_call_p (call))
 	local->can_free = true;
 
-      if (special_builtin_state (&call_state, &call_looping, callee_t))
+      if (builtin_safe_for_const_function_p (&call_looping, callee_t))
 	{
 	  worse_state (&local->pure_const_state, &local->looping,
-		       call_state, call_looping,
+		       IPA_CONST, call_looping,
 		       NULL, NULL);
 	  return;
 	}
@@ -1005,6 +1004,51 @@ malloc_candidate_p (function *fun, bool ipa)
 
 #undef DUMP_AND_RETURN
 
+/* Return true if function is known to be finite.  */
+
+bool
+finite_function_p ()
+{
+  /* Const functions cannot have back edges (an
+     indication of possible infinite loop side
+     effect.  */
+  bool finite = true;
+  if (mark_dfs_back_edges ())
+    {
+      /* Preheaders are needed for SCEV to work.
+	 Simple latches and recorded exits improve chances that loop will
+	 proved to be finite in testcases such as in loop-15.c
+	 and loop-24.c  */
+      loop_optimizer_init (LOOPS_HAVE_PREHEADERS
+			   | LOOPS_HAVE_SIMPLE_LATCHES
+			   | LOOPS_HAVE_RECORDED_EXITS);
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	flow_loops_dump (dump_file, NULL, 0);
+      if (mark_irreducible_loops ())
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "    has irreducible loops\n");
+	  finite = false;
+	}
+      else
+	{
+	  scev_initialize ();
+	  for (auto loop : loops_list (cfun, 0))
+	    if (!finite_loop_p (loop))
+	      {
+		if (dump_file)
+		  fprintf (dump_file, "    cannot prove finiteness of "
+			   "loop %i\n", loop->num);
+		finite =false;
+		break;
+	      }
+	  scev_finalize ();
+	}
+      loop_optimizer_finalize ();
+    }
+  return finite;
+}
+
 /* This is the main routine for finding the reference patterns for
    global variables within a function FN.  */
 
@@ -1063,45 +1107,10 @@ analyze_function (struct cgraph_node *fn, bool ipa)
     }
 
 end:
-  if (l->pure_const_state != IPA_NEITHER)
-    {
-      /* Const functions cannot have back edges (an
-	 indication of possible infinite loop side
-	 effect.  */
-      if (mark_dfs_back_edges ())
-        {
-	  /* Preheaders are needed for SCEV to work.
-	     Simple latches and recorded exits improve chances that loop will
-	     proved to be finite in testcases such as in loop-15.c
-	     and loop-24.c  */
-	  loop_optimizer_init (LOOPS_HAVE_PREHEADERS
-			       | LOOPS_HAVE_SIMPLE_LATCHES
-			       | LOOPS_HAVE_RECORDED_EXITS);
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    flow_loops_dump (dump_file, NULL, 0);
-	  if (mark_irreducible_loops ())
-	    {
-	      if (dump_file)
-	        fprintf (dump_file, "    has irreducible loops\n");
-	      l->looping = true;
-	    }
-	  else
-	    {
-	      scev_initialize ();
-	      for (auto loop : loops_list (cfun, 0))
-		if (!finite_loop_p (loop))
-		  {
-		    if (dump_file)
-		      fprintf (dump_file, "    cannot prove finiteness of "
-			       "loop %i\n", loop->num);
-		    l->looping =true;
-		    break;
-		  }
-	      scev_finalize ();
-	    }
-          loop_optimizer_finalize ();
-	}
-    }
+  if (l->pure_const_state != IPA_NEITHER
+      && !l->looping
+      && !finite_function_p ())
+    l->looping = true;
 
   if (dump_file && (dump_flags & TDF_DETAILS))
     fprintf (dump_file, "    checking previously known:");
@@ -1420,6 +1429,109 @@ ignore_edge_for_pure_const (struct cgraph_edge *e)
 			  flag_ipa_pure_const));
 }
 
+/* Return true if function should be skipped for local pure const analysis.  */
+
+static bool
+skip_function_for_local_pure_const (struct cgraph_node *node)
+{
+  /* Because we do not schedule pass_fixup_cfg over whole program after early
+     optimizations we must not promote functions that are called by already
+     processed functions.  */
+
+  if (function_called_by_processed_nodes_p ())
+    {
+      if (dump_file)
+	fprintf (dump_file, "Function called in recursive cycle; ignoring\n");
+      return true;
+    }
+  /* Save some work and do not analyze functions which are interposable and
+     do not have any non-interposable aliases.  */
+  if (node->get_availability () <= AVAIL_INTERPOSABLE
+      && !flag_lto
+      && !node->has_aliases_p ())
+    {
+      if (dump_file)
+	fprintf (dump_file,
+		 "Function is interposable; not analyzing.\n");
+      return true;
+    }
+  return false;
+}
+
+/* Make function const and output warning.  If LOCAL is true,
+   return true if anything changed.  Otherwise return true if
+   we may have introduced removale ctors.  */
+
+bool
+ipa_make_function_const (struct cgraph_node *node, bool looping, bool local)
+{
+  bool cdtor = false;
+
+  if (TREE_READONLY (node->decl)
+      && (looping || !DECL_LOOPING_CONST_OR_PURE_P (node->decl)))
+    return false;
+  warn_function_const (node->decl, !looping);
+  if (local && skip_function_for_local_pure_const (node))
+    return false;
+  if (dump_file)
+    fprintf (dump_file, "Function found to be %sconst: %s\n",
+	     looping ? "looping " : "",
+	     node->dump_name ());
+  if (!local && !looping)
+    cdtor = node->call_for_symbol_and_aliases (cdtor_p, NULL, true);
+  if (!dbg_cnt (ipa_attr))
+    return false;
+  if (node->set_const_flag (true, looping))
+    {
+      if (dump_file)
+	fprintf (dump_file,
+		 "Declaration updated to be %sconst: %s\n",
+		 looping ? "looping " : "",
+		 node->dump_name ());
+      if (local)
+	return true;
+      return cdtor;
+    }
+  return false;
+}
+
+/* Make function const and output warning.  If LOCAL is true,
+   return true if anything changed.  Otherwise return true if
+   we may have introduced removale ctors.  */
+
+bool
+ipa_make_function_pure (struct cgraph_node *node, bool looping, bool local)
+{
+  bool cdtor = false;
+
+  if (DECL_PURE_P (node->decl)
+      && (looping || !DECL_LOOPING_CONST_OR_PURE_P (node->decl)))
+    return false;
+  warn_function_pure (node->decl, !looping);
+  if (local && skip_function_for_local_pure_const (node))
+    return false;
+  if (dump_file)
+    fprintf (dump_file, "Function found to be %spure: %s\n",
+	     looping ? "looping " : "",
+	     node->dump_name ());
+  if (!local && !looping)
+    cdtor = node->call_for_symbol_and_aliases (cdtor_p, NULL, true);
+  if (!dbg_cnt (ipa_attr))
+    return false;
+  if (node->set_pure_flag (true, looping))
+    {
+      if (dump_file)
+	fprintf (dump_file,
+		 "Declaration updated to be %spure: %s\n",
+		 looping ? "looping " : "",
+		 node->dump_name ());
+      if (local)
+	return true;
+      return cdtor;
+    }
+  return false;
+}
+
 /* Produce transitive closure over the callgraph and compute pure/const
    attributes.  */
 
@@ -1434,7 +1546,6 @@ propagate_pure_const (void)
   int i;
   struct ipa_dfs_info * w_info;
   bool remove_p = false;
-  bool has_cdtor;
 
   order_pos = ipa_reduced_postorder (order, true,
 				     ignore_edge_for_pure_const);
@@ -1505,6 +1616,9 @@ propagate_pure_const (void)
 	      enum pure_const_state_e edge_state = IPA_CONST;
 	      bool edge_looping = false;
 
+	      if (e->recursive_p ())
+		looping = true;
+
 	      if (dump_file && (dump_flags & TDF_DETAILS))
 		{
 		  fprintf (dump_file, "    Call to %s",
@@ -1537,9 +1651,9 @@ propagate_pure_const (void)
 		      edge_looping = y_l->looping;
 		    }
 		}
-	      else if (special_builtin_state (&edge_state, &edge_looping,
-					      y->decl))
-		;
+	      else if (builtin_safe_for_const_function_p (&edge_looping,
+							   y->decl))
+		edge_state = IPA_CONST;
 	      else
 		state_from_flags (&edge_state, &edge_looping,
 				  flags_from_decl_or_type (y->decl),
@@ -1688,55 +1802,11 @@ propagate_pure_const (void)
 	    switch (this_state)
 	      {
 	      case IPA_CONST:
-		if (!TREE_READONLY (w->decl))
-		  {
-		    warn_function_const (w->decl, !this_looping);
-		    if (dump_file)
-		      fprintf (dump_file, "Function found to be %sconst: %s\n",
-			       this_looping ? "looping " : "",
-			       w->dump_name ());
-		  }
-		/* Turning constructor or destructor to non-looping const/pure
-		   enables us to possibly remove the function completely.  */
-		if (this_looping)
-		  has_cdtor = false;
-		else
-		  has_cdtor = w->call_for_symbol_and_aliases (cdtor_p,
-							      NULL, true);
-		if (w->set_const_flag (true, this_looping))
-		  {
-		    if (dump_file)
-		      fprintf (dump_file,
-			       "Declaration updated to be %sconst: %s\n",
-			       this_looping ? "looping " : "",
-			       w->dump_name ());
-		    remove_p |= has_cdtor;
-		  }
+		remove_p |= ipa_make_function_const (w, this_looping, false);
 		break;
 
 	      case IPA_PURE:
-		if (!DECL_PURE_P (w->decl))
-		  {
-		    warn_function_pure (w->decl, !this_looping);
-		    if (dump_file)
-		      fprintf (dump_file, "Function found to be %spure: %s\n",
-			       this_looping ? "looping " : "",
-			       w->dump_name ());
-		  }
-		if (this_looping)
-		  has_cdtor = false;
-		else
-		  has_cdtor = w->call_for_symbol_and_aliases (cdtor_p,
-							      NULL, true);
-		if (w->set_pure_flag (true, this_looping))
-		  {
-		    if (dump_file)
-		      fprintf (dump_file,
-			       "Declaration updated to be %spure: %s\n",
-			       this_looping ? "looping " : "",
-			       w->dump_name ());
-		    remove_p |= has_cdtor;
-		  }
+		remove_p |= ipa_make_function_pure (w, this_looping, false);
 		break;
 
 	      default:
@@ -2035,34 +2105,6 @@ make_pass_ipa_pure_const (gcc::context *ctxt)
   return new pass_ipa_pure_const (ctxt);
 }
 
-/* Return true if function should be skipped for local pure const analysis.  */
-
-static bool
-skip_function_for_local_pure_const (struct cgraph_node *node)
-{
-  /* Because we do not schedule pass_fixup_cfg over whole program after early
-     optimizations we must not promote functions that are called by already
-     processed functions.  */
-
-  if (function_called_by_processed_nodes_p ())
-    {
-      if (dump_file)
-        fprintf (dump_file, "Function called in recursive cycle; ignoring\n");
-      return true;
-    }
-  /* Save some work and do not analyze functions which are interposable and
-     do not have any non-interposable aliases.  */
-  if (node->get_availability () <= AVAIL_INTERPOSABLE
-      && !node->has_aliases_p ())
-    {
-      if (dump_file)
-        fprintf (dump_file,
-		 "Function is interposable; not analyzing.\n");
-      return true;
-    }
-  return false;
-}
-
 /* Simple local pass for pure const discovery reusing the analysis from
    ipa_pure_const.   This pass is effective when executed together with
    other optimization passes in early optimization pass queue.  */
@@ -2124,65 +2166,22 @@ pass_local_pure_const::execute (function *fun)
 		 current_function_name ());
 
       /* Update declaration and reduce profile to executed once.  */
-      TREE_THIS_VOLATILE (current_function_decl) = 1;
+      if (cgraph_node::get (current_function_decl)->set_noreturn_flag (true))
+	changed = true;
       if (node->frequency > NODE_FREQUENCY_EXECUTED_ONCE)
 	node->frequency = NODE_FREQUENCY_EXECUTED_ONCE;
-
-      changed = true;
     }
 
   switch (l->pure_const_state)
     {
     case IPA_CONST:
-      if (!TREE_READONLY (current_function_decl))
-	{
-	  warn_function_const (current_function_decl, !l->looping);
-	  if (dump_file)
-	    fprintf (dump_file, "Function found to be %sconst: %s\n",
-		     l->looping ? "looping " : "",
-		     current_function_name ());
-	}
-      else if (DECL_LOOPING_CONST_OR_PURE_P (current_function_decl)
-	       && !l->looping)
-	{
-	  if (dump_file)
-	    fprintf (dump_file, "Function found to be non-looping: %s\n",
-		     current_function_name ());
-	}
-      if (!skip && node->set_const_flag (true, l->looping))
-	{
-	  if (dump_file)
-	    fprintf (dump_file, "Declaration updated to be %sconst: %s\n",
-		     l->looping ? "looping " : "",
-		     current_function_name ());
-	  changed = true;
-	}
+      changed |= ipa_make_function_const
+		   (cgraph_node::get (current_function_decl), l->looping, true);
       break;
 
     case IPA_PURE:
-      if (!DECL_PURE_P (current_function_decl))
-	{
-	  warn_function_pure (current_function_decl, !l->looping);
-	  if (dump_file)
-	    fprintf (dump_file, "Function found to be %spure: %s\n",
-		     l->looping ? "looping " : "",
-		     current_function_name ());
-	}
-      else if (DECL_LOOPING_CONST_OR_PURE_P (current_function_decl)
-	       && !l->looping)
-	{
-	  if (dump_file)
-	    fprintf (dump_file, "Function found to be non-looping: %s\n",
-		     current_function_name ());
-	}
-      if (!skip && node->set_pure_flag (true, l->looping))
-	{
-	  if (dump_file)
-	    fprintf (dump_file, "Declaration updated to be %spure: %s\n",
-		     l->looping ? "looping " : "",
-		     current_function_name ());
-	  changed = true;
-	}
+      changed |= ipa_make_function_pure
+		   (cgraph_node::get (current_function_decl), l->looping, true);
       break;
 
     default:
diff --git a/gcc/ipa-sra.c b/gcc/ipa-sra.c
index 8803659..cb0e305 100644
--- a/gcc/ipa-sra.c
+++ b/gcc/ipa-sra.c
@@ -85,6 +85,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-streamer.h"
 #include "internal-fn.h"
 #include "symtab-clones.h"
+#include "attribs.h"
 
 static void ipa_sra_summarize_function (cgraph_node *);
 
@@ -616,13 +617,6 @@ ipa_sra_preliminary_function_checks (cgraph_node *node)
       return false;
     }
 
-  if (TYPE_ATTRIBUTES (TREE_TYPE (node->decl)))
-    {
-      if (dump_file)
-	fprintf (dump_file, "Function type has attributes. \n");
-      return false;
-    }
-
   if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
     {
       if (dump_file)
diff --git a/gcc/ipa-utils.h b/gcc/ipa-utils.h
index 3cfaf2d..e2440a1 100644
--- a/gcc/ipa-utils.h
+++ b/gcc/ipa-utils.h
@@ -47,6 +47,12 @@ void ipa_merge_profiles (struct cgraph_node *dst,
 			 struct cgraph_node *src, bool preserve_body = false);
 bool recursive_call_p (tree, tree);
 
+/* In ipa-pure-const.c  */
+bool finite_function_p ();
+bool builtin_safe_for_const_function_p (bool *, tree);
+bool ipa_make_function_const (cgraph_node *, bool, bool);
+bool ipa_make_function_pure (cgraph_node *, bool, bool);
+
 /* In ipa-profile.c  */
 bool ipa_propagate_frequency (struct cgraph_node *node);
 
diff --git a/gcc/ipa.c b/gcc/ipa.c
index 4f62ac1..325b658 100644
--- a/gcc/ipa.c
+++ b/gcc/ipa.c
@@ -837,7 +837,7 @@ ipa_discover_variable_flags (void)
    FINAL specify whether the externally visible name for collect2 should
    be produced. */
 
-static void
+static tree
 cgraph_build_static_cdtor_1 (char which, tree body, int priority, bool final,
 			     tree optimization,
 			     tree target)
@@ -916,6 +916,7 @@ cgraph_build_static_cdtor_1 (char which, tree body, int priority, bool final,
 
   set_cfun (NULL);
   current_function_decl = NULL;
+  return decl;
 }
 
 /* Generate and emit a static constructor or destructor.  WHICH must
@@ -1022,6 +1023,124 @@ build_cdtor (bool ctor_p, const vec<tree> &cdtors)
     }
 }
 
+/* Helper functions for build_cxa_dtor_registrations ().
+   Build a decl for __cxa_atexit ().  */
+
+static tree
+build_cxa_atexit_decl ()
+{
+  /* The parameter to "__cxa_atexit" is "void (*)(void *)".  */
+  tree fn_type = build_function_type_list (void_type_node,
+					   ptr_type_node, NULL_TREE);
+  tree fn_ptr_type = build_pointer_type (fn_type);
+  /* The declaration for `__cxa_atexit' is:
+     int __cxa_atexit (void (*)(void *), void *, void *).  */
+  const char *name = "__cxa_atexit";
+  tree cxa_name = get_identifier (name);
+  fn_type = build_function_type_list (integer_type_node, fn_ptr_type,
+				      ptr_type_node, ptr_type_node, NULL_TREE);
+  tree atexit_fndecl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL,
+				   cxa_name, fn_type);
+  SET_DECL_ASSEMBLER_NAME (atexit_fndecl, cxa_name);
+  DECL_VISIBILITY (atexit_fndecl) = VISIBILITY_DEFAULT;
+  DECL_VISIBILITY_SPECIFIED (atexit_fndecl) = true;
+  set_call_expr_flags (atexit_fndecl, ECF_LEAF | ECF_NOTHROW);
+  TREE_PUBLIC (atexit_fndecl) = true;
+  DECL_EXTERNAL (atexit_fndecl) = true;
+  DECL_ARTIFICIAL (atexit_fndecl) = true;
+  return atexit_fndecl;
+}
+
+/* Build a decl for __dso_handle.  */
+
+static tree
+build_dso_handle_decl ()
+{
+  /* Declare the __dso_handle variable.  */
+  tree dso_handle_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
+				     get_identifier ("__dso_handle"),
+				     ptr_type_node);
+  TREE_PUBLIC (dso_handle_decl) = true;
+  DECL_EXTERNAL (dso_handle_decl) = true;
+  DECL_ARTIFICIAL (dso_handle_decl) = true;
+#ifdef HAVE_GAS_HIDDEN
+  if (dso_handle_decl != error_mark_node)
+    {
+      DECL_VISIBILITY (dso_handle_decl) = VISIBILITY_HIDDEN;
+      DECL_VISIBILITY_SPECIFIED (dso_handle_decl) = true;
+    }
+#endif
+  return dso_handle_decl;
+}
+
+/*  This builds one or more constructor functions that register DTORs with
+    __cxa_atexit ().  Within a priority level, DTORs are registered in TU
+    order - which means that they will run in reverse TU order from cxa_atexit.
+    This is the same behavior as using a .fini / .mod_term_funcs section.
+    As the functions are built, they are appended to the CTORs vector.  */
+
+static void
+build_cxa_dtor_registrations (const vec<tree> &dtors, vec<tree> *ctors)
+{
+  size_t i,j;
+  size_t len = dtors.length ();
+
+  location_t sav_loc = input_location;
+  input_location = UNKNOWN_LOCATION;
+
+  tree atexit_fndecl = build_cxa_atexit_decl ();
+  tree dso_handle_decl = build_dso_handle_decl ();
+
+  /* We want &__dso_handle.  */
+  tree dso_ptr = build1_loc (UNKNOWN_LOCATION, ADDR_EXPR,
+			     ptr_type_node, dso_handle_decl);
+
+  i = 0;
+  while (i < len)
+    {
+      priority_type priority = 0;
+      tree body = NULL_TREE;
+      j = i;
+      do
+	{
+	  priority_type p;
+	  tree fn = dtors[j];
+	  p = DECL_FINI_PRIORITY (fn);
+	  if (j == i)
+	    priority = p;
+	  else if (p != priority)
+	    break;
+	  j++;
+	}
+      while (j < len);
+
+      /* Find the next batch of destructors with the same initialization
+	 priority.  */
+      for (;i < j; i++)
+	{
+	  tree fn = dtors[i];
+	  DECL_STATIC_DESTRUCTOR (fn) = 0;
+	  tree dtor_ptr = build1_loc (UNKNOWN_LOCATION, ADDR_EXPR,
+				      ptr_type_node, fn);
+	  tree call_cxa_atexit
+	    = build_call_expr_loc (UNKNOWN_LOCATION, atexit_fndecl, 3,
+				   dtor_ptr, null_pointer_node, dso_ptr);
+	  TREE_SIDE_EFFECTS (call_cxa_atexit) = 1;
+	  append_to_statement_list (call_cxa_atexit, &body);
+	}
+
+      gcc_assert (body != NULL_TREE);
+      /* Generate a function to register the DTORs at this priority.  */
+      tree new_ctor
+	= cgraph_build_static_cdtor_1 ('I', body, priority, true,
+				       DECL_FUNCTION_SPECIFIC_OPTIMIZATION (dtors[0]),
+				       DECL_FUNCTION_SPECIFIC_TARGET (dtors[0]));
+      /* Add this to the list of ctors.  */
+      ctors->safe_push (new_ctor);
+    }
+  input_location = sav_loc;
+}
+
 /* Comparison function for qsort.  P1 and P2 are actually of type
    "tree *" and point to static constructors.  DECL_INIT_PRIORITY is
    used to determine the sort order.  */
@@ -1071,7 +1190,46 @@ compare_dtor (const void *p1, const void *p2)
   else if (priority1 > priority2)
     return 1;
   else
-    /* Ensure a stable sort.  */
+    /* Ensure a stable sort - into TU order.  */
+    return DECL_UID (f1) - DECL_UID (f2);
+}
+
+/* Comparison function for qsort.  P1 and P2 are of type "tree *" and point to
+   a pair of static constructors or destructors.  We first sort on the basis of
+   priority and then into TU order (on the strict assumption that DECL_UIDs are
+   ordered in the same way as the original functions).  ???: this seems quite
+   fragile. */
+
+static int
+compare_cdtor_tu_order (const void *p1, const void *p2)
+{
+  tree f1;
+  tree f2;
+  int priority1;
+  int priority2;
+
+  f1 = *(const tree *)p1;
+  f2 = *(const tree *)p2;
+  /* We process the DTORs first, and then remove their flag, so this order
+     allows for functions that are declared as both CTOR and DTOR.  */
+  if (DECL_STATIC_DESTRUCTOR (f1))
+    {
+      gcc_checking_assert (DECL_STATIC_DESTRUCTOR (f2));
+      priority1 = DECL_FINI_PRIORITY (f1);
+      priority2 = DECL_FINI_PRIORITY (f2);
+    }
+  else
+    {
+      priority1 = DECL_INIT_PRIORITY (f1);
+      priority2 = DECL_INIT_PRIORITY (f2);
+    }
+
+  if (priority1 < priority2)
+    return -1;
+  else if (priority1 > priority2)
+    return 1;
+  else
+    /* For equal priority, sort into the order of definition in the TU.  */
     return DECL_UID (f1) - DECL_UID (f2);
 }
 
@@ -1097,6 +1255,37 @@ build_cdtor_fns (vec<tree> *ctors, vec<tree> *dtors)
     }
 }
 
+/* Generate new CTORs to register static destructors with __cxa_atexit and add
+   them to the existing list of CTORs; we then process the revised CTORs list.
+
+   We sort the DTORs into priority and then TU order, this means that they are
+   registered in that order with __cxa_atexit () and therefore will be run in
+   the reverse order.
+
+   Likewise, CTORs are sorted into priority and then TU order, which means that
+   they will run in that order.
+
+   This matches the behavior of using init/fini or mod_init_func/mod_term_func
+   sections.  */
+
+static void
+build_cxa_atexit_fns (vec<tree> *ctors, vec<tree> *dtors)
+{
+  if (!dtors->is_empty ())
+    {
+      gcc_assert (targetm.dtors_from_cxa_atexit);
+      dtors->qsort (compare_cdtor_tu_order);
+      build_cxa_dtor_registrations (*dtors, ctors);
+    }
+
+  if (!ctors->is_empty ())
+    {
+      gcc_assert (targetm.dtors_from_cxa_atexit);
+      ctors->qsort (compare_cdtor_tu_order);
+      build_cdtor (/*ctor_p=*/true, *ctors);
+    }
+}
+
 /* Look for constructors and destructors and produce function calling them.
    This is needed for targets not supporting ctors or dtors, but we perform the
    transformation also at linktime to merge possibly numerous
@@ -1115,7 +1304,10 @@ ipa_cdtor_merge (void)
     if (DECL_STATIC_CONSTRUCTOR (node->decl)
 	|| DECL_STATIC_DESTRUCTOR (node->decl))
        record_cdtor_fn (node, &ctors, &dtors);
-  build_cdtor_fns (&ctors, &dtors);
+  if (targetm.dtors_from_cxa_atexit)
+    build_cxa_atexit_fns (&ctors, &dtors);
+  else
+    build_cdtor_fns (&ctors, &dtors);
   return 0;
 }
 
@@ -1162,7 +1354,7 @@ pass_ipa_cdtor_merge::gate (function *)
   /* Perform the pass when we have no ctors/dtors support
      or at LTO time to merge multiple constructors into single
      function.  */
-  return !targetm.have_ctors_dtors || in_lto_p;
+  return !targetm.have_ctors_dtors || in_lto_p || targetm.dtors_from_cxa_atexit;
 }
 
 } // anon namespace
diff --git a/gcc/jit/ChangeLog b/gcc/jit/ChangeLog
index 334fd39..d7f7cc5 100644
--- a/gcc/jit/ChangeLog
+++ b/gcc/jit/ChangeLog
@@ -1,3 +1,11 @@
+2021-11-12  David Malcolm  <dmalcolm@redhat.com>
+
+	PR jit/103199
+	* docs/examples/tut04-toyvm/toyvm.c (toyvm_function_compile):
+	Increase size of buffer.
+	* docs/examples/tut04-toyvm/toyvm.cc
+	(compilation_state::create_function): Likewise.
+
 2021-09-10  Petter Tomner  <tomner@kth.se>
 
 	* jit-playback.c: Moved global var processing to after loc handling.
diff --git a/gcc/jit/docs/examples/tut04-toyvm/toyvm.c b/gcc/jit/docs/examples/tut04-toyvm/toyvm.c
index 8ea716e..63418f9 100644
--- a/gcc/jit/docs/examples/tut04-toyvm/toyvm.c
+++ b/gcc/jit/docs/examples/tut04-toyvm/toyvm.c
@@ -561,7 +561,7 @@ toyvm_function_compile (toyvm_function *fn)
   /* Create a block per operation.  */
   for (pc = 0; pc < fn->fn_num_ops; pc++)
     {
-      char buf[16];
+      char buf[100];
       sprintf (buf, "instr%i", pc);
       state.op_blocks[pc] = gcc_jit_function_new_block (state.fn, buf);
     }
diff --git a/gcc/jit/docs/examples/tut04-toyvm/toyvm.cc b/gcc/jit/docs/examples/tut04-toyvm/toyvm.cc
index 7e95501..81c8045 100644
--- a/gcc/jit/docs/examples/tut04-toyvm/toyvm.cc
+++ b/gcc/jit/docs/examples/tut04-toyvm/toyvm.cc
@@ -633,7 +633,7 @@ compilation_state::create_function (const char *funcname)
   /* Create a block per operation.  */
   for (int pc = 0; pc < toyvmfn.fn_num_ops; pc++)
     {
-      char buf[16];
+      char buf[100];
       sprintf (buf, "instr%i", pc);
       op_blocks[pc] = fn.new_block (buf);
     }
diff --git a/gcc/langhooks-def.h b/gcc/langhooks-def.h
index 02b4681..5f17620 100644
--- a/gcc/langhooks-def.h
+++ b/gcc/langhooks-def.h
@@ -56,6 +56,8 @@ extern void lhd_overwrite_decl_assembler_name (tree decl, tree name);
 extern bool lhd_warn_unused_global_decl (const_tree);
 extern tree lhd_simulate_enum_decl (location_t, const char *,
 				    vec<string_int_pair> *);
+extern tree lhd_simulate_record_decl (location_t, const char *,
+				      array_slice<const tree>);
 extern tree lhd_type_for_size (unsigned precision, int unsignedp);
 extern void lhd_incomplete_type_error (location_t, const_tree, const_tree);
 extern tree lhd_type_promotes_to (tree);
@@ -183,6 +185,7 @@ extern tree lhd_unit_size_without_reusable_padding (tree);
 
 #define LANG_HOOKS_MAKE_TYPE lhd_make_node
 #define LANG_HOOKS_SIMULATE_ENUM_DECL	lhd_simulate_enum_decl
+#define LANG_HOOKS_SIMULATE_RECORD_DECL	lhd_simulate_record_decl
 #define LANG_HOOKS_CLASSIFY_RECORD	NULL
 #define LANG_HOOKS_TYPE_FOR_SIZE	lhd_type_for_size
 #define LANG_HOOKS_INCOMPLETE_TYPE_ERROR lhd_incomplete_type_error
@@ -217,6 +220,7 @@ extern tree lhd_unit_size_without_reusable_padding (tree);
 #define LANG_HOOKS_FOR_TYPES_INITIALIZER { \
   LANG_HOOKS_MAKE_TYPE, \
   LANG_HOOKS_SIMULATE_ENUM_DECL, \
+  LANG_HOOKS_SIMULATE_RECORD_DECL, \
   LANG_HOOKS_CLASSIFY_RECORD, \
   LANG_HOOKS_TYPE_FOR_MODE, \
   LANG_HOOKS_TYPE_FOR_SIZE, \
diff --git a/gcc/langhooks.c b/gcc/langhooks.c
index 48c7237..49613b4 100644
--- a/gcc/langhooks.c
+++ b/gcc/langhooks.c
@@ -516,6 +516,25 @@ lhd_simulate_enum_decl (location_t loc, const char *name,
   return enumtype;
 }
 
+/* Default implementation of LANG_HOOKS_SIMULATE_RECORD_DECL.
+   Just create a normal RECORD_TYPE and a TYPE_DECL for it.  */
+tree
+lhd_simulate_record_decl (location_t loc, const char *name,
+			  array_slice<const tree> fields)
+{
+  for (unsigned int i = 1; i < fields.size (); ++i)
+    /* Reversed by finish_builtin_struct.  */
+    DECL_CHAIN (fields[i]) = fields[i - 1];
+
+  tree type = lang_hooks.types.make_type (RECORD_TYPE);
+  finish_builtin_struct (type, name, fields.back (), NULL_TREE);
+
+  tree decl = build_decl (loc, TYPE_DECL, get_identifier (name), type);
+  lang_hooks.decls.pushdecl (decl);
+
+  return type;
+}
+
 /* Default implementation of LANG_HOOKS_TYPE_FOR_SIZE.
    Return an integer type with PRECISION bits of precision,
    that is unsigned if UNSIGNEDP is nonzero, otherwise signed.  */
diff --git a/gcc/langhooks.h b/gcc/langhooks.h
index ffd3e0b..3e89134 100644
--- a/gcc/langhooks.h
+++ b/gcc/langhooks.h
@@ -68,6 +68,16 @@ struct lang_hooks_for_types
      them all with the given source location.  */
   tree (*simulate_enum_decl) (location_t, const char *, vec<string_int_pair> *);
 
+  /* Do the equivalent of:
+
+       typedef struct NAME { FIELDS; } NAME;
+
+     associating it with location LOC.  Return the associated RECORD_TYPE.
+
+     FIELDS is a list of FIELD_DECLs, in layout order.  */
+  tree (*simulate_record_decl) (location_t loc, const char *name,
+				array_slice<const tree> fields);
+
   /* Return what kind of RECORD_TYPE this is, mainly for purposes of
      debug information.  If not defined, record types are assumed to
      be structures.  */
diff --git a/gcc/loop-unroll.c b/gcc/loop-unroll.c
index 2b31faf..a06bdfb 100644
--- a/gcc/loop-unroll.c
+++ b/gcc/loop-unroll.c
@@ -520,14 +520,11 @@ unroll_loop_constant_iterations (class loop *loop)
       if (exit_mod)
 	{
 	  opt_info_start_duplication (opt_info);
-          ok = duplicate_loop_to_header_edge (loop, loop_preheader_edge (loop),
-					      exit_mod,
-					      wont_exit, desc->out_edge,
-					      &remove_edges,
-					      DLTHE_FLAG_UPDATE_FREQ
-					      | (opt_info && exit_mod > 1
-						 ? DLTHE_RECORD_COPY_NUMBER
-						   : 0));
+	  ok = duplicate_loop_body_to_header_edge (
+	    loop, loop_preheader_edge (loop), exit_mod, wont_exit,
+	    desc->out_edge, &remove_edges,
+	    DLTHE_FLAG_UPDATE_FREQ
+	      | (opt_info && exit_mod > 1 ? DLTHE_RECORD_COPY_NUMBER : 0));
 	  gcc_assert (ok);
 
           if (opt_info && exit_mod > 1)
@@ -569,14 +566,11 @@ unroll_loop_constant_iterations (class loop *loop)
 	    bitmap_clear_bit (wont_exit, 1);
 
           opt_info_start_duplication (opt_info);
-	  ok = duplicate_loop_to_header_edge (loop, loop_preheader_edge (loop),
-					      exit_mod + 1,
-					      wont_exit, desc->out_edge,
-					      &remove_edges,
-					      DLTHE_FLAG_UPDATE_FREQ
-					      | (opt_info && exit_mod > 0
-						 ? DLTHE_RECORD_COPY_NUMBER
-						   : 0));
+	  ok = duplicate_loop_body_to_header_edge (
+	    loop, loop_preheader_edge (loop), exit_mod + 1, wont_exit,
+	    desc->out_edge, &remove_edges,
+	    DLTHE_FLAG_UPDATE_FREQ
+	      | (opt_info && exit_mod > 0 ? DLTHE_RECORD_COPY_NUMBER : 0));
 	  gcc_assert (ok);
 
           if (opt_info && exit_mod > 0)
@@ -606,14 +600,10 @@ unroll_loop_constant_iterations (class loop *loop)
   /* Now unroll the loop.  */
 
   opt_info_start_duplication (opt_info);
-  ok = duplicate_loop_to_header_edge (loop, loop_latch_edge (loop),
-				      max_unroll,
-				      wont_exit, desc->out_edge,
-				      &remove_edges,
-				      DLTHE_FLAG_UPDATE_FREQ
-				      | (opt_info
-					 ? DLTHE_RECORD_COPY_NUMBER
-					   : 0));
+  ok = duplicate_loop_body_to_header_edge (
+    loop, loop_latch_edge (loop), max_unroll, wont_exit, desc->out_edge,
+    &remove_edges,
+    DLTHE_FLAG_UPDATE_FREQ | (opt_info ? DLTHE_RECORD_COPY_NUMBER : 0));
   gcc_assert (ok);
 
   if (opt_info)
@@ -975,10 +965,10 @@ unroll_loop_runtime_iterations (class loop *loop)
       if (!desc->noloop_assumptions)
 	bitmap_set_bit (wont_exit, 1);
       ezc_swtch = loop_preheader_edge (loop)->src;
-      ok = duplicate_loop_to_header_edge (loop, loop_preheader_edge (loop),
-					  1, wont_exit, desc->out_edge,
-					  &remove_edges,
-					  DLTHE_FLAG_UPDATE_FREQ);
+      ok = duplicate_loop_body_to_header_edge (loop, loop_preheader_edge (loop),
+					       1, wont_exit, desc->out_edge,
+					       &remove_edges,
+					       DLTHE_FLAG_UPDATE_FREQ);
       gcc_assert (ok);
     }
 
@@ -997,10 +987,10 @@ unroll_loop_runtime_iterations (class loop *loop)
       bitmap_clear (wont_exit);
       if (i != n_peel - 1 || !last_may_exit)
 	bitmap_set_bit (wont_exit, 1);
-      ok = duplicate_loop_to_header_edge (loop, loop_preheader_edge (loop),
-					  1, wont_exit, desc->out_edge,
-					  &remove_edges,
-					  DLTHE_FLAG_UPDATE_FREQ);
+      ok = duplicate_loop_body_to_header_edge (loop, loop_preheader_edge (loop),
+					       1, wont_exit, desc->out_edge,
+					       &remove_edges,
+					       DLTHE_FLAG_UPDATE_FREQ);
       gcc_assert (ok);
 
       /* Create item for switch.  */
@@ -1061,14 +1051,10 @@ unroll_loop_runtime_iterations (class loop *loop)
   bitmap_clear_bit (wont_exit, may_exit_copy);
   opt_info_start_duplication (opt_info);
 
-  ok = duplicate_loop_to_header_edge (loop, loop_latch_edge (loop),
-				      max_unroll,
-				      wont_exit, desc->out_edge,
-				      &remove_edges,
-				      DLTHE_FLAG_UPDATE_FREQ
-				      | (opt_info
-					 ? DLTHE_RECORD_COPY_NUMBER
-					   : 0));
+  ok = duplicate_loop_body_to_header_edge (
+    loop, loop_latch_edge (loop), max_unroll, wont_exit, desc->out_edge,
+    &remove_edges,
+    DLTHE_FLAG_UPDATE_FREQ | (opt_info ? DLTHE_RECORD_COPY_NUMBER : 0));
   gcc_assert (ok);
 
   if (opt_info)
@@ -1255,13 +1241,9 @@ unroll_loop_stupid (class loop *loop)
   bitmap_clear (wont_exit);
   opt_info_start_duplication (opt_info);
 
-  ok = duplicate_loop_to_header_edge (loop, loop_latch_edge (loop),
-				      nunroll, wont_exit,
-				      NULL, NULL,
-				      DLTHE_FLAG_UPDATE_FREQ
-				      | (opt_info
-					 ? DLTHE_RECORD_COPY_NUMBER
-					   : 0));
+  ok = duplicate_loop_body_to_header_edge (
+    loop, loop_latch_edge (loop), nunroll, wont_exit, NULL, NULL,
+    DLTHE_FLAG_UPDATE_FREQ | (opt_info ? DLTHE_RECORD_COPY_NUMBER : 0));
   gcc_assert (ok);
 
   if (opt_info)
@@ -2019,7 +2001,7 @@ apply_opt_in_copies (struct opt_info *opt_info,
       orig_bb = get_bb_original (bb);
 
       /* bb->aux holds position in copy sequence initialized by
-	 duplicate_loop_to_header_edge.  */
+	 duplicate_loop_body_to_header_edge.  */
       delta = determine_split_iv_delta ((size_t)bb->aux, n_copies,
 					unrolling);
       bb->aux = 0;
diff --git a/gcc/lower-subreg.c b/gcc/lower-subreg.c
index 2107826..f0dc63f 100644
--- a/gcc/lower-subreg.c
+++ b/gcc/lower-subreg.c
@@ -383,8 +383,10 @@ simple_move (rtx_insn *insn, bool speed_p)
      non-integer mode for which there is no integer mode of the same
      size.  */
   mode = GET_MODE (SET_DEST (set));
+  scalar_int_mode int_mode;
   if (!SCALAR_INT_MODE_P (mode)
-      && !int_mode_for_size (GET_MODE_BITSIZE (mode), 0).exists ())
+      && (!int_mode_for_size (GET_MODE_BITSIZE (mode), 0).exists (&int_mode)
+	  || !targetm.modes_tieable_p (mode, int_mode)))
     return NULL_RTX;
 
   /* Reject PARTIAL_INT modes.  They are used for processor specific
diff --git a/gcc/lra-constraints.c b/gcc/lra-constraints.c
index 8f75125..0195b4f 100644
--- a/gcc/lra-constraints.c
+++ b/gcc/lra-constraints.c
@@ -1102,7 +1102,7 @@ match_reload (signed char out, signed char *ins, signed char *outs,
 	  for (i = 0; outs[i] >= 0; i++)
 	    {
 	      rtx other_out_rtx = *curr_id->operand_loc[outs[i]];
-	      if (REG_P (other_out_rtx)
+	      if (outs[i] != out && REG_P (other_out_rtx)
 		  && (regno_val_use_in (REGNO (in_rtx), other_out_rtx)
 		      != NULL_RTX))
 		{
@@ -4382,7 +4382,10 @@ curr_insn_transform (bool check_only_p)
       }
 
   n_outputs = 0;
-  outputs[0] = -1;
+  for (i = 0; i < n_operands; i++)
+    if (curr_static_id->operand[i].type == OP_OUT)
+      outputs[n_outputs++] = i;
+  outputs[n_outputs] = -1;
   for (i = 0; i < n_operands; i++)
     {
       int regno;
@@ -4457,8 +4460,6 @@ curr_insn_transform (bool check_only_p)
 		     lra-lives.c.  */
 		  match_reload (i, goal_alt_matched[i], outputs, goal_alt[i], &before,
 				&after, TRUE);
-		  outputs[n_outputs++] = i;
-		  outputs[n_outputs] = -1;
 		}
 	      continue;
 	    }
@@ -4636,14 +4637,6 @@ curr_insn_transform (bool check_only_p)
 	   process_alt_operands decides that it is possible.  */
 	gcc_unreachable ();
 
-      /* Memorise processed outputs so that output remaining to be processed
-	 can avoid using the same register value (see match_reload).  */
-      if (curr_static_id->operand[i].type == OP_OUT)
-	{
-	  outputs[n_outputs++] = i;
-	  outputs[n_outputs] = -1;
-	}
-
       if (optional_p)
 	{
 	  rtx reg = op;
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index 7b9e488..54f642d 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -224,7 +224,7 @@ merge_flto_options (vec<cl_decoded_option> &decoded_options,
    ontop of DECODED_OPTIONS.  */
 
 static void
-merge_and_complain (vec<cl_decoded_option> decoded_options,
+merge_and_complain (vec<cl_decoded_option> &decoded_options,
 		    vec<cl_decoded_option> fdecoded_options,
 		    vec<cl_decoded_option> decoded_cl_options)
 {
diff --git a/gcc/lto/ChangeLog b/gcc/lto/ChangeLog
index 1aea3f8..38350b3 100644
--- a/gcc/lto/ChangeLog
+++ b/gcc/lto/ChangeLog
@@ -1,3 +1,15 @@
+2021-11-06  Jan Hubicka  <hubicka@ucw.cz>
+
+	PR ipa/103070
+	PR ipa/103058
+	* lto-partition.c (must_not_rename): Update comment.
+	(promote_symbol): Set resolution to LDPR_PREVAILING_DEF_IRONLY.
+
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Make-lang.in: Add dummy lto.install-dvi target.
+
 2021-09-27  Martin Liska  <mliska@suse.cz>
 
 	* lto-lang.c (lto_init_options_struct): Set also
diff --git a/gcc/lto/Make-lang.in b/gcc/lto/Make-lang.in
index aefebed..40d2f83 100644
--- a/gcc/lto/Make-lang.in
+++ b/gcc/lto/Make-lang.in
@@ -60,6 +60,7 @@ lto.install-common: installdirs
 lto.install-man:
 lto.install-info:
 lto.dvi:
+lto.install-dvi:
 lto.pdf:
 lto.install-pdf:
 lto.html:
diff --git a/gcc/lto/lto-partition.c b/gcc/lto/lto-partition.c
index 15761ac..bee4021 100644
--- a/gcc/lto/lto-partition.c
+++ b/gcc/lto/lto-partition.c
@@ -852,7 +852,9 @@ must_not_rename (symtab_node *node, const char *name)
   /* Avoid mangling of already mangled clones. 
      ???  should have a flag whether a symbol has a 'private' name already,
      since we produce some symbols like that i.e. for global constructors
-     that are not really clones.  */
+     that are not really clones.
+     ???  it is what unique_name means.  We only need to set it when doing
+     private symbols.  */
   if (node->unique_name)
     {
       if (dump_file)
@@ -995,6 +997,10 @@ promote_symbol (symtab_node *node)
      defined by the non-LTO part.  */
   privatize_symbol_name (node);
   TREE_PUBLIC (node->decl) = 1;
+  /* After privatization the node should not conflict with any other symbol,
+     so it is prevailing.  This is important to keep binds_to_current_def_p
+     to work across partitions.  */
+  node->resolution = LDPR_PREVAILING_DEF_IRONLY;
   DECL_VISIBILITY (node->decl) = VISIBILITY_HIDDEN;
   DECL_VISIBILITY_SPECIFIED (node->decl) = true;
   if (dump_file)
diff --git a/gcc/match.pd b/gcc/match.pd
index 5bed2e1..758322d 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -37,7 +37,8 @@ along with GCC; see the file COPYING3.  If not see
    integer_pow2p
    uniform_integer_cst_p
    HONOR_NANS
-   uniform_vector_p)
+   uniform_vector_p
+   expand_vec_cmp_expr_p)
 
 /* Operator lists.  */
 (define_operator_list tcc_comparison
@@ -104,6 +105,39 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 (define_operator_list COND_TERNARY
   IFN_COND_FMA IFN_COND_FMS IFN_COND_FNMA IFN_COND_FNMS)
 
+/* __atomic_fetch_or_*, __atomic_fetch_xor_*, __atomic_xor_fetch_*  */
+(define_operator_list ATOMIC_FETCH_OR_XOR_N
+  BUILT_IN_ATOMIC_FETCH_OR_1 BUILT_IN_ATOMIC_FETCH_OR_2
+  BUILT_IN_ATOMIC_FETCH_OR_4 BUILT_IN_ATOMIC_FETCH_OR_8
+  BUILT_IN_ATOMIC_FETCH_OR_16
+  BUILT_IN_ATOMIC_FETCH_XOR_1 BUILT_IN_ATOMIC_FETCH_XOR_2
+  BUILT_IN_ATOMIC_FETCH_XOR_4 BUILT_IN_ATOMIC_FETCH_XOR_8
+  BUILT_IN_ATOMIC_FETCH_XOR_16
+  BUILT_IN_ATOMIC_XOR_FETCH_1 BUILT_IN_ATOMIC_XOR_FETCH_2
+  BUILT_IN_ATOMIC_XOR_FETCH_4 BUILT_IN_ATOMIC_XOR_FETCH_8
+  BUILT_IN_ATOMIC_XOR_FETCH_16)
+/* __sync_fetch_and_or_*, __sync_fetch_and_xor_*, __sync_xor_and_fetch_*  */
+(define_operator_list SYNC_FETCH_OR_XOR_N
+  BUILT_IN_SYNC_FETCH_AND_OR_1 BUILT_IN_SYNC_FETCH_AND_OR_2
+  BUILT_IN_SYNC_FETCH_AND_OR_4 BUILT_IN_SYNC_FETCH_AND_OR_8
+  BUILT_IN_SYNC_FETCH_AND_OR_16
+  BUILT_IN_SYNC_FETCH_AND_XOR_1 BUILT_IN_SYNC_FETCH_AND_XOR_2
+  BUILT_IN_SYNC_FETCH_AND_XOR_4 BUILT_IN_SYNC_FETCH_AND_XOR_8
+  BUILT_IN_SYNC_FETCH_AND_XOR_16
+  BUILT_IN_SYNC_XOR_AND_FETCH_1 BUILT_IN_SYNC_XOR_AND_FETCH_2
+  BUILT_IN_SYNC_XOR_AND_FETCH_4 BUILT_IN_SYNC_XOR_AND_FETCH_8
+  BUILT_IN_SYNC_XOR_AND_FETCH_16)
+/* __atomic_fetch_and_*.  */
+(define_operator_list ATOMIC_FETCH_AND_N
+  BUILT_IN_ATOMIC_FETCH_AND_1 BUILT_IN_ATOMIC_FETCH_AND_2
+  BUILT_IN_ATOMIC_FETCH_AND_4 BUILT_IN_ATOMIC_FETCH_AND_8
+  BUILT_IN_ATOMIC_FETCH_AND_16)
+/* __sync_fetch_and_and_*.  */
+(define_operator_list SYNC_FETCH_AND_AND_N
+  BUILT_IN_SYNC_FETCH_AND_AND_1 BUILT_IN_SYNC_FETCH_AND_AND_2
+  BUILT_IN_SYNC_FETCH_AND_AND_4 BUILT_IN_SYNC_FETCH_AND_AND_8
+  BUILT_IN_SYNC_FETCH_AND_AND_16)
+
 /* With nop_convert? combine convert? and view_convert? in one pattern
    plus conditionalize on tree_nop_conversion_p conversions.  */
 (match (nop_convert @0)
@@ -831,6 +865,39 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
     { tree utype = unsigned_type_for (type); }
     (convert (rshift (lshift (convert:utype @0) @2) @3))))))
 
+/* Fold (-x >> C) into -(x > 0) where C = precision(type) - 1.  */
+(for cst (INTEGER_CST VECTOR_CST)
+ (simplify
+  (rshift (negate:s @0) cst@1)
+   (if (!TYPE_UNSIGNED (type)
+        && TYPE_OVERFLOW_UNDEFINED (type))
+    (with { tree stype = TREE_TYPE (@1);
+	    tree bt = truth_type_for (type);
+	    tree zeros = build_zero_cst (type);
+	    tree cst = NULL_TREE; }
+     (switch
+      /* Handle scalar case.  */
+      (if (INTEGRAL_TYPE_P (type)
+	   /* If we apply the rule to the scalar type before vectorization
+	      we will enforce the result of the comparison being a bool
+	      which will require an extra AND on the result that will be
+	      indistinguishable from when the user did actually want 0
+	      or 1 as the result so it can't be removed.  */
+	   && canonicalize_math_after_vectorization_p ()
+	   && wi::eq_p (wi::to_wide (@1), TYPE_PRECISION (type) - 1))
+       (negate (convert (gt @0 { zeros; }))))
+      /* Handle vector case.  */
+      (if (VECTOR_INTEGER_TYPE_P (type)
+	   /* First check whether the target has the same mode for vector
+	      comparison results as it's operands do.  */
+	   && TYPE_MODE (bt) == TYPE_MODE (type)
+	   /* Then check to see if the target is able to expand the comparison
+	      with the given type later on, otherwise we may ICE.  */
+	   && expand_vec_cmp_expr_p (type, bt, GT_EXPR)
+	   && (cst = uniform_integer_cst_p (@1)) != NULL
+	   && wi::eq_p (wi::to_wide (cst), element_precision (type) - 1))
+       (view_convert (gt:bt @0 { zeros; }))))))))
+
 /* Fold (C1/X)*C2 into (C1*C2)/X.  */
 (simplify
  (mult (rdiv@3 REAL_CST@0 @1) REAL_CST@2)
@@ -1411,21 +1478,21 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
  (absu tree_expr_nonnegative_p@0)
  (convert @0))
 
-/* Simplify (-(X < 0) | 1) * X into abs (X).  */
+/* Simplify (-(X < 0) | 1) * X into abs (X) or absu(X).  */
 (simplify
- (mult:c (bit_ior (negate (convert? (lt @0 integer_zerop))) integer_onep) @0)
- (if (INTEGRAL_TYPE_P (type) && !TYPE_UNSIGNED (type))
-  (abs @0)))
-
-/* Similarly (-(X < 0) | 1U) * X into absu (X).  */
-(simplify
- (mult:c (bit_ior (nop_convert (negate (convert? (lt @0 integer_zerop))))
-		  integer_onep) (nop_convert @0))
+ (mult:c (nop_convert1?
+	  (bit_ior (nop_convert2? (negate (convert? (lt @0 integer_zerop))))
+		    integer_onep))
+	 (nop_convert3? @0))
  (if (INTEGRAL_TYPE_P (type)
-      && TYPE_UNSIGNED (type)
       && INTEGRAL_TYPE_P (TREE_TYPE (@0))
       && !TYPE_UNSIGNED (TREE_TYPE (@0)))
-  (absu @0)))
+  (if (TYPE_UNSIGNED (type))
+   (absu @0)
+   (abs @0)
+  )
+ )
+)
 
 /* A few cases of fold-const.c negate_expr_p predicate.  */
 (match negate_expr_p
@@ -3009,6 +3076,30 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
     @0
     @2)))
 
+/* Simplify min (&var[off0], &var[off1]) etc. depending on whether
+   the addresses are known to be less, equal or greater.  */
+(for minmax (min max)
+     cmp (lt gt)
+ (simplify
+  (minmax (convert1?@2 addr@0) (convert2?@3 addr@1))
+  (with
+   {
+     poly_int64 off0, off1;
+     tree base0, base1;
+     int equal = address_compare (cmp, TREE_TYPE (@2), @0, @1, base0, base1,
+				  off0, off1, GENERIC);
+   }
+   (if (equal == 1)
+    (if (minmax == MIN_EXPR)
+     (if (known_le (off0, off1))
+      @2
+      (if (known_gt (off0, off1))
+       @3))
+     (if (known_ge (off0, off1))
+      @2
+      (if (known_lt (off0, off1))
+       @3)))))))
+
 /* (convert (minmax ((convert (x) c)))) -> minmax (x c) if x is promoted
    and the outer convert demotes the expression back to x's type.  */
 (for minmax (min max)
@@ -3383,6 +3474,17 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 	     (bit_and (convert (shift:shift_type (convert @3) @1)) { newmaskt; })
 	     (bit_and @4 { newmaskt; })))))))))))))
 
+/* ((1 << n) & M) != 0  -> n == log2 (M) */
+(for cmp (ne eq)
+       icmp (eq ne)
+ (simplify
+  (cmp
+   (bit_and
+    (nop_convert? (lshift integer_onep @0)) integer_pow2p@1) integer_zerop)
+  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)))
+   (icmp @0 { wide_int_to_tree (TREE_TYPE (@0),
+				wi::exact_log2 (wi::to_wide (@1))); }))))
+
 /* Fold (X {&,^,|} C2) << C1 into (X << C1) {&,^,|} (C2 << C1)
    (X {&,^,|} C2) >> C1 into (X >> C1) & (C2 >> C1).  */
 (for shift (lshift rshift)
@@ -3907,6 +4009,76 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
   (vec_cond @0 (op! @3 @1) (op! @3 @2))))
 #endif
 
+#if GIMPLE
+(match (nop_atomic_bit_test_and_p @0 @1 @4)
+ (bit_and (nop_convert?@4 (ATOMIC_FETCH_OR_XOR_N @2 INTEGER_CST@0 @3))
+	   INTEGER_CST@1)
+ (with {
+	 int ibit = tree_log2 (@0);
+	 int ibit2 = tree_log2 (@1);
+       }
+  (if (ibit == ibit2
+      && ibit >= 0))))
+
+(match (nop_atomic_bit_test_and_p @0 @1 @3)
+ (bit_and (nop_convert?@3 (SYNC_FETCH_OR_XOR_N @2 INTEGER_CST@0))
+	  INTEGER_CST@1)
+ (with {
+	 int ibit = tree_log2 (@0);
+	 int ibit2 = tree_log2 (@1);
+       }
+  (if (ibit == ibit2
+      && ibit >= 0))))
+
+(match (nop_atomic_bit_test_and_p @0 @0 @4)
+ (bit_and:c
+  (nop_convert?@4
+   (ATOMIC_FETCH_OR_XOR_N @2 (nop_convert? (lshift@0 integer_onep@5 @6)) @3))
+  @0))
+
+(match (nop_atomic_bit_test_and_p @0 @0 @4)
+ (bit_and:c
+  (nop_convert?@4
+   (SYNC_FETCH_OR_XOR_N @2 (nop_convert? (lshift@0 integer_onep@3 @5))))
+  @0))
+
+(match (nop_atomic_bit_test_and_p @0 @1 @3)
+ (bit_and@4 (nop_convert?@3 (ATOMIC_FETCH_AND_N @2 INTEGER_CST@0 @5))
+	    INTEGER_CST@1)
+ (with {
+	 int ibit = wi::exact_log2 (wi::zext (wi::bit_not (wi::to_wide (@0)),
+					      TYPE_PRECISION(type)));
+	 int ibit2 = tree_log2 (@1);
+       }
+  (if (ibit == ibit2
+      && ibit >= 0))))
+
+(match (nop_atomic_bit_test_and_p @0 @1 @3)
+ (bit_and@4
+  (nop_convert?@3 (SYNC_FETCH_AND_AND_N @2 INTEGER_CST@0))
+  INTEGER_CST@1)
+ (with {
+	 int ibit = wi::exact_log2 (wi::zext (wi::bit_not (wi::to_wide (@0)),
+					      TYPE_PRECISION(type)));
+	 int ibit2 = tree_log2 (@1);
+       }
+  (if (ibit == ibit2
+      && ibit >= 0))))
+
+(match (nop_atomic_bit_test_and_p @0 @0 @3)
+ (bit_and:c
+  (nop_convert?@3
+   (ATOMIC_FETCH_AND_N @2 (nop_convert? (bit_not (lshift@0 integer_onep@6 @7))) @5))
+   @0))
+
+(match (nop_atomic_bit_test_and_p @0 @0 @3)
+ (bit_and:c
+  (nop_convert?@3
+   (SYNC_FETCH_AND_AND_N @2 (nop_convert? (bit_not (lshift@0 integer_onep@6 @7)))))
+   @0))
+
+#endif
+
 /* (v ? w : 0) ? a : b is just (v & w) ? a : b
    Currently disabled after pass lvec because ARM understands
    VEC_COND_EXPR<v==w,-1,0> but not a plain v==w fed to BIT_IOR_EXPR.  */
@@ -3987,8 +4159,7 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
      (if (integer_onep (@2))
       (convert (bit_xor (convert:boolean_type_node @0) { booltrue; } )))
      /* a ? powerof2cst : 0 -> (!a) << (log2(powerof2cst)) */
-     (if (INTEGRAL_TYPE_P (type) &&  integer_pow2p (@2)
-         && TYPE_PRECISION (type) != 1)
+     (if (INTEGRAL_TYPE_P (type) &&  integer_pow2p (@2))
       (with {
 	tree shift = build_int_cst (integer_type_node, tree_log2 (@2));
        }
@@ -5291,132 +5462,30 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
   (with
    {
      poly_int64 off0, off1;
-     tree base0 = get_addr_base_and_unit_offset (TREE_OPERAND (@0, 0), &off0);
-     tree base1 = get_addr_base_and_unit_offset (TREE_OPERAND (@1, 0), &off1);
-     if (base0 && TREE_CODE (base0) == MEM_REF)
-       {
-	 off0 += mem_ref_offset (base0).force_shwi ();
-         base0 = TREE_OPERAND (base0, 0);
-       }
-     if (base1 && TREE_CODE (base1) == MEM_REF)
-       {
-	 off1 += mem_ref_offset (base1).force_shwi ();
-         base1 = TREE_OPERAND (base1, 0);
-       }
+     tree base0, base1;
+     int equal = address_compare (cmp, TREE_TYPE (@2), @0, @1, base0, base1,
+				  off0, off1, GENERIC);
    }
-   (if (base0 && base1)
-    (with
-     {
-       int equal = 2;
-       /* Punt in GENERIC on variables with value expressions;
-	  the value expressions might point to fields/elements
-	  of other vars etc.  */
-       if (GENERIC
-	   && ((VAR_P (base0) && DECL_HAS_VALUE_EXPR_P (base0))
-	       || (VAR_P (base1) && DECL_HAS_VALUE_EXPR_P (base1))))
-	 ;
-       else if (decl_in_symtab_p (base0)
-		&& decl_in_symtab_p (base1))
-         equal = symtab_node::get_create (base0)
-	           ->equal_address_to (symtab_node::get_create (base1));
-       else if ((DECL_P (base0)
-		 || TREE_CODE (base0) == SSA_NAME
-		 || TREE_CODE (base0) == STRING_CST)
-		&& (DECL_P (base1)
-		    || TREE_CODE (base1) == SSA_NAME
-		    || TREE_CODE (base1) == STRING_CST))
-         equal = (base0 == base1);
-       if (equal == 0)
-	 {
-	   HOST_WIDE_INT ioff0 = -1, ioff1 = -1;
-	   off0.is_constant (&ioff0);
-	   off1.is_constant (&ioff1);
-	   if ((DECL_P (base0) && TREE_CODE (base1) == STRING_CST)
-	       || (TREE_CODE (base0) == STRING_CST && DECL_P (base1))
-	       || (TREE_CODE (base0) == STRING_CST
-		   && TREE_CODE (base1) == STRING_CST
-		   && ioff0 >= 0 && ioff1 >= 0
-		   && ioff0 < TREE_STRING_LENGTH (base0)
-		   && ioff1 < TREE_STRING_LENGTH (base1)
-		   /* This is a too conservative test that the STRING_CSTs
-		      will not end up being string-merged.  */
-		   && strncmp (TREE_STRING_POINTER (base0) + ioff0,
-			       TREE_STRING_POINTER (base1) + ioff1,
-			       MIN (TREE_STRING_LENGTH (base0) - ioff0,
-				    TREE_STRING_LENGTH (base1) - ioff1)) != 0))
-	     ;
-	   else if (!DECL_P (base0) || !DECL_P (base1))
-	     equal = 2;
-	   else if (cmp != EQ_EXPR && cmp != NE_EXPR)
-	     equal = 2;
-	   /* If this is a pointer comparison, ignore for now even
-	      valid equalities where one pointer is the offset zero
-	      of one object and the other to one past end of another one.  */
-	   else if (!INTEGRAL_TYPE_P (TREE_TYPE (@2)))
-	     ;
-	   /* Assume that automatic variables can't be adjacent to global
-	      variables.  */
-	   else if (is_global_var (base0) != is_global_var (base1))
-	     ;
-	   else
-	     {
-	       tree sz0 = DECL_SIZE_UNIT (base0);
-	       tree sz1 = DECL_SIZE_UNIT (base1);
-	       /* If sizes are unknown, e.g. VLA or not representable,
-		  punt.  */
-	       if (!tree_fits_poly_int64_p (sz0)
-		   || !tree_fits_poly_int64_p (sz1))
-		 equal = 2;
-	       else
-		 {
-		   poly_int64 size0 = tree_to_poly_int64 (sz0);
-		   poly_int64 size1 = tree_to_poly_int64 (sz1);
-		   /* If one offset is pointing (or could be) to the beginning
-		      of one object and the other is pointing to one past the
-		      last byte of the other object, punt.  */
-		   if (maybe_eq (off0, 0) && maybe_eq (off1, size1))
-		     equal = 2;
-		   else if (maybe_eq (off1, 0) && maybe_eq (off0, size0))
-		     equal = 2;
-		   /* If both offsets are the same, there are some cases
-		      we know that are ok.  Either if we know they aren't
-		      zero, or if we know both sizes are no zero.  */
-		   if (equal == 2
-		       && known_eq (off0, off1)
-		       && (known_ne (off0, 0)
-			   || (known_ne (size0, 0) && known_ne (size1, 0))))
-		     equal = 0;
-		 }
-	     }
-	 }
-     }
-     (if (equal == 1
-	  && (cmp == EQ_EXPR || cmp == NE_EXPR
-	      /* If the offsets are equal we can ignore overflow.  */
-	      || known_eq (off0, off1)
-	      || TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (@0))
-		 /* Or if we compare using pointers to decls or strings.  */
-	      || (POINTER_TYPE_P (TREE_TYPE (@2))
-		  && (DECL_P (base0) || TREE_CODE (base0) == STRING_CST))))
-      (switch
-       (if (cmp == EQ_EXPR && (known_eq (off0, off1) || known_ne (off0, off1)))
-	{ constant_boolean_node (known_eq (off0, off1), type); })
-       (if (cmp == NE_EXPR && (known_eq (off0, off1) || known_ne (off0, off1)))
-	{ constant_boolean_node (known_ne (off0, off1), type); })
-       (if (cmp == LT_EXPR && (known_lt (off0, off1) || known_ge (off0, off1)))
-	{ constant_boolean_node (known_lt (off0, off1), type); })
-       (if (cmp == LE_EXPR && (known_le (off0, off1) || known_gt (off0, off1)))
-	{ constant_boolean_node (known_le (off0, off1), type); })
-       (if (cmp == GE_EXPR && (known_ge (off0, off1) || known_lt (off0, off1)))
-	{ constant_boolean_node (known_ge (off0, off1), type); })
-       (if (cmp == GT_EXPR && (known_gt (off0, off1) || known_le (off0, off1)))
-	{ constant_boolean_node (known_gt (off0, off1), type); }))
-      (if (equal == 0)
-	(switch
-	 (if (cmp == EQ_EXPR)
-	  { constant_boolean_node (false, type); })
-	 (if (cmp == NE_EXPR)
-	  { constant_boolean_node (true, type); })))))))))
+   (if (equal == 1)
+    (switch
+     (if (cmp == EQ_EXPR && (known_eq (off0, off1) || known_ne (off0, off1)))
+      { constant_boolean_node (known_eq (off0, off1), type); })
+     (if (cmp == NE_EXPR && (known_eq (off0, off1) || known_ne (off0, off1)))
+      { constant_boolean_node (known_ne (off0, off1), type); })
+     (if (cmp == LT_EXPR && (known_lt (off0, off1) || known_ge (off0, off1)))
+      { constant_boolean_node (known_lt (off0, off1), type); })
+     (if (cmp == LE_EXPR && (known_le (off0, off1) || known_gt (off0, off1)))
+      { constant_boolean_node (known_le (off0, off1), type); })
+     (if (cmp == GE_EXPR && (known_ge (off0, off1) || known_lt (off0, off1)))
+      { constant_boolean_node (known_ge (off0, off1), type); })
+     (if (cmp == GT_EXPR && (known_gt (off0, off1) || known_le (off0, off1)))
+      { constant_boolean_node (known_gt (off0, off1), type); }))
+    (if (equal == 0)
+     (switch
+      (if (cmp == EQ_EXPR)
+       { constant_boolean_node (false, type); })
+      (if (cmp == NE_EXPR)
+       { constant_boolean_node (true, type); })))))))
 
 /* Simplify pointer equality compares using PTA.  */
 (for neeq (ne eq)
@@ -6228,14 +6297,16 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 	    BUILT_IN_ROUNDEVENL BUILT_IN_ROUNDEVEN BUILT_IN_ROUNDEVENF
 	    BUILT_IN_ROUNDL BUILT_IN_ROUND BUILT_IN_ROUNDF
 	    BUILT_IN_NEARBYINTL BUILT_IN_NEARBYINT BUILT_IN_NEARBYINTF
-	    BUILT_IN_RINTL BUILT_IN_RINT BUILT_IN_RINTF)
+	    BUILT_IN_RINTL BUILT_IN_RINT BUILT_IN_RINTF
+	    BUILT_IN_SQRTL BUILT_IN_SQRT BUILT_IN_SQRTF)
      tos (IFN_TRUNC IFN_TRUNC IFN_TRUNC
 	  IFN_FLOOR IFN_FLOOR IFN_FLOOR
 	  IFN_CEIL IFN_CEIL IFN_CEIL
 	  IFN_ROUNDEVEN IFN_ROUNDEVEN IFN_ROUNDEVEN
 	  IFN_ROUND IFN_ROUND IFN_ROUND
 	  IFN_NEARBYINT IFN_NEARBYINT IFN_NEARBYINT
-	  IFN_RINT IFN_RINT IFN_RINT)
+	  IFN_RINT IFN_RINT IFN_RINT
+	  IFN_SQRT IFN_SQRT IFN_SQRT)
  /* (_Float16) round ((doube) x) -> __built_in_roundf16 (x), etc.,
     if x is a _Float16.  */
  (simplify
@@ -6245,6 +6316,47 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 	  && direct_internal_fn_supported_p (as_internal_fn (tos),
 					     type, OPTIMIZE_FOR_BOTH))
        (tos @0))))
+
+/* Simplify (trunc)copysign ((extend)x, (extend)y) to copysignf (x, y),
+   x,y is float value, similar for _Float16/double.  */
+(for copysigns (COPYSIGN_ALL)
+ (simplify
+  (convert (copysigns (convert@2 @0) (convert @1)))
+   (if (optimize
+       && !HONOR_SNANS (@2)
+       && types_match (type, TREE_TYPE (@0))
+       && types_match (type, TREE_TYPE (@1))
+       && TYPE_PRECISION (type) < TYPE_PRECISION (TREE_TYPE (@2))
+       && direct_internal_fn_supported_p (IFN_COPYSIGN,
+					  type, OPTIMIZE_FOR_BOTH))
+    (IFN_COPYSIGN @0 @1))))
+
+(for froms (BUILT_IN_FMAF BUILT_IN_FMA BUILT_IN_FMAL)
+     tos (IFN_FMA IFN_FMA IFN_FMA)
+ (simplify
+  (convert (froms (convert@3 @0) (convert @1) (convert @2)))
+   (if (flag_unsafe_math_optimizations
+       && optimize
+       && FLOAT_TYPE_P (type)
+       && FLOAT_TYPE_P (TREE_TYPE (@3))
+       && types_match (type, TREE_TYPE (@0))
+       && types_match (type, TREE_TYPE (@1))
+       && types_match (type, TREE_TYPE (@2))
+       && TYPE_PRECISION (type) < TYPE_PRECISION (TREE_TYPE (@3))
+       && direct_internal_fn_supported_p (as_internal_fn (tos),
+					  type, OPTIMIZE_FOR_BOTH))
+    (tos @0 @1 @2))))
+
+(for maxmin (max min)
+ (simplify
+  (convert (maxmin (convert@2 @0) (convert @1)))
+   (if (optimize
+       && FLOAT_TYPE_P (type)
+       && FLOAT_TYPE_P (TREE_TYPE (@2))
+       && types_match (type, TREE_TYPE (@0))
+       && types_match (type, TREE_TYPE (@1))
+       && element_precision (type) < element_precision (TREE_TYPE (@2)))
+    (maxmin @0 @1))))
 #endif
 
 (for froms (XFLOORL XCEILL XROUNDL XRINTL)
diff --git a/gcc/objc/ChangeLog b/gcc/objc/ChangeLog
index 853ce09..51b2103 100644
--- a/gcc/objc/ChangeLog
+++ b/gcc/objc/ChangeLog
@@ -1,3 +1,8 @@
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Make-lang.in: Add dummy objc.install-dvi target.
+
 2021-10-08  Martin Liska  <mliska@suse.cz>
 
 	* objc-next-runtime-abi-01.c: Add missing include.
diff --git a/gcc/objc/Make-lang.in b/gcc/objc/Make-lang.in
index 8215283..d3f99c8 100644
--- a/gcc/objc/Make-lang.in
+++ b/gcc/objc/Make-lang.in
@@ -91,6 +91,7 @@ objc.rest.encap:
 objc.info:
 objc.install-info:
 objc.dvi:
+objc.install-dvi:
 objc.pdf:
 objc.install-pdf:
 objc.html:
diff --git a/gcc/objcp/ChangeLog b/gcc/objcp/ChangeLog
index c5d402c..c83b831 100644
--- a/gcc/objcp/ChangeLog
+++ b/gcc/objcp/ChangeLog
@@ -1,3 +1,8 @@
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Make-lang.in: Add dummy objc++.install-dvi target.
+
 2021-10-11  Eric Gallager  <egallager@gcc.gnu.org>
 
 	PR objc++/56604
diff --git a/gcc/objcp/Make-lang.in b/gcc/objcp/Make-lang.in
index d7cafe8..1806386 100644
--- a/gcc/objcp/Make-lang.in
+++ b/gcc/objcp/Make-lang.in
@@ -124,6 +124,7 @@ obj-c++.rest.encap:
 obj-c++.info:
 obj-c++.install-info:
 obj-c++.dvi:
+obj-c++.install-dvi:
 obj-c++.pdf:
 obj-c++.install-pdf:
 obj-c++.html:
diff --git a/gcc/omp-builtins.def b/gcc/omp-builtins.def
index 4520dc0..2950813 100644
--- a/gcc/omp-builtins.def
+++ b/gcc/omp-builtins.def
@@ -442,8 +442,8 @@ DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TARGET_UPDATE, "GOMP_target_update_ext",
 DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA,
 		  "GOMP_target_enter_exit_data",
 		  BT_FN_VOID_INT_SIZE_PTR_PTR_PTR_UINT_PTR, ATTR_NOTHROW_LIST)
-DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TEAMS, "GOMP_teams",
-		  BT_FN_VOID_UINT_UINT, ATTR_NOTHROW_LIST)
+DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TEAMS4, "GOMP_teams4",
+		  BT_FN_BOOL_UINT_UINT_UINT_BOOL, ATTR_NOTHROW_LIST)
 DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TEAMS_REG, "GOMP_teams_reg",
 		  BT_FN_VOID_OMPFN_PTR_UINT_UINT_UINT, ATTR_NOTHROW_LIST)
 DEF_GOMP_BUILTIN (BUILT_IN_GOMP_TASKGROUP_REDUCTION_REGISTER,
diff --git a/gcc/omp-expand.c b/gcc/omp-expand.c
index 9715ef5..c5fa5a0 100644
--- a/gcc/omp-expand.c
+++ b/gcc/omp-expand.c
@@ -938,7 +938,7 @@ expand_teams_call (basic_block bb, gomp_teams *entry_stmt)
     num_teams = build_int_cst (unsigned_type_node, 0);
   else
     {
-      num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams);
+      num_teams = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams);
       num_teams = fold_convert (unsigned_type_node, num_teams);
     }
   tree thread_limit = omp_find_clause (clauses, OMP_CLAUSE_THREAD_LIMIT);
@@ -1208,6 +1208,28 @@ expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from,
     }
 }
 
+/* Prepend or append LHS CODE RHS condition before or after *GSI_P.  */
+
+static gcond *
+expand_omp_build_cond (gimple_stmt_iterator *gsi_p, enum tree_code code,
+		       tree lhs, tree rhs, bool after = false)
+{
+  gcond *cond_stmt = gimple_build_cond (code, lhs, rhs, NULL_TREE, NULL_TREE);
+  if (after)
+    gsi_insert_after (gsi_p, cond_stmt, GSI_CONTINUE_LINKING);
+  else
+    gsi_insert_before (gsi_p, cond_stmt, GSI_SAME_STMT);
+  if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), expand_omp_regimplify_p,
+		 NULL, NULL)
+      || walk_tree (gimple_cond_rhs_ptr (cond_stmt), expand_omp_regimplify_p,
+		    NULL, NULL))
+    {
+      gimple_stmt_iterator gsi = gsi_for_stmt (cond_stmt);
+      gimple_regimplify_operands (cond_stmt, &gsi);
+    }
+  return cond_stmt;
+}
+
 /* Expand the OpenMP parallel or task directive starting at REGION.  */
 
 static void
@@ -1868,17 +1890,8 @@ expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 	  n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2));
 	  n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE,
 					 true, GSI_SAME_STMT);
-	  cond_stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
-					 NULL_TREE, NULL_TREE);
-	  gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT);
-	  if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
-			 expand_omp_regimplify_p, NULL, NULL)
-	      || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
-			    expand_omp_regimplify_p, NULL, NULL))
-	    {
-	      *gsi = gsi_for_stmt (cond_stmt);
-	      gimple_regimplify_operands (cond_stmt, gsi);
-	    }
+	  cond_stmt = expand_omp_build_cond (gsi, fd->loops[i].cond_code,
+					     n1, n2);
 	  e = split_block (entry_bb, cond_stmt);
 	  basic_block &zero_iter_bb
 	    = i < fd->collapse ? zero_iter1_bb : zero_iter2_bb;
@@ -1975,6 +1988,7 @@ expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 	  break;
       if (i == fd->last_nonrect
 	  && fd->loops[i].outer == fd->last_nonrect - fd->first_nonrect
+	  && !POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v))
 	  && !TYPE_UNSIGNED (TREE_TYPE (fd->loops[i].v)))
 	{
 	  int o = fd->first_nonrect;
@@ -2074,18 +2088,16 @@ expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 	  n2e = force_gimple_operand_gsi (&gsi2, n2e, true, NULL_TREE,
 					  true, GSI_SAME_STMT);
 	  gcond *cond_stmt
-	    = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
-				 NULL_TREE, NULL_TREE);
-	  gsi_insert_before (&gsi2, cond_stmt, GSI_SAME_STMT);
+	    = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
+				     n1, n2);
 	  e = split_block (bb1, cond_stmt);
 	  e->flags = EDGE_TRUE_VALUE;
 	  e->probability = profile_probability::likely ().guessed ();
 	  basic_block bb2 = e->dest;
 	  gsi2 = gsi_after_labels (bb2);
 
-	  cond_stmt = gimple_build_cond (fd->loops[i].cond_code, n1e, n2e,
-					 NULL_TREE, NULL_TREE);
-	  gsi_insert_before (&gsi2, cond_stmt, GSI_SAME_STMT);
+	  cond_stmt = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
+					     n1e, n2e);
 	  e = split_block (bb2, cond_stmt);
 	  e->flags = EDGE_TRUE_VALUE;
 	  e->probability = profile_probability::likely ().guessed ();
@@ -2136,9 +2148,8 @@ expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 	  e->probability = profile_probability::unlikely ().guessed ();
 
 	  gsi2 = gsi_after_labels (bb3);
-	  cond_stmt = gimple_build_cond (fd->loops[i].cond_code, n1e, n2e,
-					 NULL_TREE, NULL_TREE);
-	  gsi_insert_before (&gsi2, cond_stmt, GSI_SAME_STMT);
+	  cond_stmt = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
+					     n1e, n2e);
 	  e = split_block (bb3, cond_stmt);
 	  e->flags = EDGE_TRUE_VALUE;
 	  e->probability = profile_probability::likely ().guessed ();
@@ -2192,9 +2203,8 @@ expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 					     true, GSI_SAME_STMT);
 	      expand_omp_build_assign (&gsi2, j ? n2o : n1o, tem);
 
-	      cond_stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
-					     NULL_TREE, NULL_TREE);
-	      gsi_insert_before (&gsi2, cond_stmt, GSI_SAME_STMT);
+	      cond_stmt = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
+						 n1, n2);
 	      e = split_block (gsi_bb (gsi2), cond_stmt);
 	      e->flags = j ? EDGE_TRUE_VALUE : EDGE_FALSE_VALUE;
 	      e->probability = profile_probability::unlikely ().guessed ();
@@ -2250,15 +2260,22 @@ expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 	      gsi2 = gsi_after_labels (cur_bb);
 	      tree n1, n2;
 	      t = fold_convert (itype, unshare_expr (fd->loops[i].n1));
-	      if (fd->loops[i].m1)
+	      if (fd->loops[i].m1 == NULL_TREE)
+		n1 = t;
+	      else if (POINTER_TYPE_P (itype))
+		{
+		  gcc_assert (integer_onep (fd->loops[i].m1));
+		  t = fold_convert (sizetype,
+				    unshare_expr (fd->loops[i].n1));
+		  n1 = fold_build_pointer_plus (vs[i - fd->loops[i].outer], t);
+		}
+	      else
 		{
 		  n1 = fold_convert (itype, unshare_expr (fd->loops[i].m1));
 		  n1 = fold_build2 (MULT_EXPR, itype,
 				    vs[i - fd->loops[i].outer], n1);
 		  n1 = fold_build2 (PLUS_EXPR, itype, n1, t);
 		}
-	      else
-		n1 = t;
 	      n1 = force_gimple_operand_gsi (&gsi2, n1, true, NULL_TREE,
 					     true, GSI_SAME_STMT);
 	      if (i < fd->last_nonrect)
@@ -2267,23 +2284,31 @@ expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 		  expand_omp_build_assign (&gsi2, vs[i], n1);
 		}
 	      t = fold_convert (itype, unshare_expr (fd->loops[i].n2));
-	      if (fd->loops[i].m2)
+	      if (fd->loops[i].m2 == NULL_TREE)
+		n2 = t;
+	      else if (POINTER_TYPE_P (itype))
+		{
+		  gcc_assert (integer_onep (fd->loops[i].m2));
+		  t = fold_convert (sizetype,
+				    unshare_expr (fd->loops[i].n2));
+		  n2 = fold_build_pointer_plus (vs[i - fd->loops[i].outer], t);
+		}
+	      else
 		{
 		  n2 = fold_convert (itype, unshare_expr (fd->loops[i].m2));
 		  n2 = fold_build2 (MULT_EXPR, itype,
 				    vs[i - fd->loops[i].outer], n2);
 		  n2 = fold_build2 (PLUS_EXPR, itype, n2, t);
 		}
-	      else
-		n2 = t;
 	      n2 = force_gimple_operand_gsi (&gsi2, n2, true, NULL_TREE,
 					     true, GSI_SAME_STMT);
+	      if (POINTER_TYPE_P (itype))
+		itype = signed_type_for (itype);
 	      if (i == fd->last_nonrect)
 		{
 		  gcond *cond_stmt
-		    = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
-					 NULL_TREE, NULL_TREE);
-		  gsi_insert_before (&gsi2, cond_stmt, GSI_SAME_STMT);
+		    = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
+					     n1, n2);
 		  e = split_block (cur_bb, cond_stmt);
 		  e->flags = EDGE_TRUE_VALUE;
 		  ne = make_edge (cur_bb, next_bb, EDGE_FALSE_VALUE);
@@ -2295,8 +2320,10 @@ expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 					     ? -1 : 1));
 		  t = fold_build2 (PLUS_EXPR, itype,
 				   fold_convert (itype, fd->loops[i].step), t);
-		  t = fold_build2 (PLUS_EXPR, itype, t, n2);
-		  t = fold_build2 (MINUS_EXPR, itype, t, n1);
+		  t = fold_build2 (PLUS_EXPR, itype, t,
+				   fold_convert (itype, n2));
+		  t = fold_build2 (MINUS_EXPR, itype, t,
+				   fold_convert (itype, n1));
 		  tree step = fold_convert (itype, fd->loops[i].step);
 		  if (TYPE_UNSIGNED (itype)
 		      && fd->loops[i].cond_code == GT_EXPR)
@@ -2323,7 +2350,11 @@ expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 	      gsi2 = gsi_after_labels (e->dest);
 	      tree step = fold_convert (itype,
 					unshare_expr (fd->loops[i].step));
-	      t = fold_build2 (PLUS_EXPR, itype, vs[i], step);
+	      if (POINTER_TYPE_P (TREE_TYPE (vs[i])))
+		t = fold_build_pointer_plus (vs[i],
+					     fold_convert (sizetype, step));
+	      else
+		t = fold_build2 (PLUS_EXPR, itype, vs[i], step);
 	      t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
 					    true, GSI_SAME_STMT);
 	      expand_omp_build_assign (&gsi2, vs[i], t);
@@ -2331,10 +2362,7 @@ expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 	      ne = split_block (e->dest, last_stmt (e->dest));
 	      gsi2 = gsi_after_labels (ne->dest);
 
-	      gcond *cond_stmt
-		= gimple_build_cond (fd->loops[i].cond_code, vs[i], n2,
-				     NULL_TREE, NULL_TREE);
-	      gsi_insert_before (&gsi2, cond_stmt, GSI_SAME_STMT);
+	      expand_omp_build_cond (&gsi2, fd->loops[i].cond_code, vs[i], n2);
 	      edge e3, e4;
 	      if (next_bb == entry_bb)
 		{
@@ -2535,10 +2563,8 @@ expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 	      tree first_inner_iterations = fd->first_inner_iterations;
 	      tree factor = fd->factor;
 	      gcond *cond_stmt
-		= gimple_build_cond (NE_EXPR, factor,
-				     build_zero_cst (TREE_TYPE (factor)),
-				     NULL_TREE, NULL_TREE);
-	      gsi_insert_after (gsi, cond_stmt, GSI_CONTINUE_LINKING);
+		= expand_omp_build_cond (gsi, NE_EXPR, factor,
+					 build_zero_cst (TREE_TYPE (factor)));
 	      edge e = split_block (gsi_bb (*gsi), cond_stmt);
 	      basic_block bb0 = e->src;
 	      e->flags = EDGE_TRUE_VALUE;
@@ -2761,17 +2787,22 @@ expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 			     && !fd->loops[j].non_rect_referenced);
 	      gsi2 = gsi_after_labels (cur_bb);
 	      t = fold_convert (itype, unshare_expr (fd->loops[j].n1));
-	      if (fd->loops[j].m1)
+	      if (fd->loops[j].m1 == NULL_TREE)
+		n1 = rect_p ? build_zero_cst (type) : t;
+	      else if (POINTER_TYPE_P (itype))
+		{
+		  gcc_assert (integer_onep (fd->loops[j].m1));
+		  t = fold_convert (sizetype,
+				    unshare_expr (fd->loops[j].n1));
+		  n1 = fold_build_pointer_plus (vs[j - fd->loops[j].outer], t);
+		}
+	      else
 		{
 		  n1 = fold_convert (itype, unshare_expr (fd->loops[j].m1));
 		  n1 = fold_build2 (MULT_EXPR, itype,
 				    vs[j - fd->loops[j].outer], n1);
 		  n1 = fold_build2 (PLUS_EXPR, itype, n1, t);
 		}
-	      else if (rect_p)
-		n1 = build_zero_cst (type);
-	      else
-		n1 = t;
 	      n1 = force_gimple_operand_gsi (&gsi2, n1, true, NULL_TREE,
 					     true, GSI_SAME_STMT);
 	      if (j < fd->last_nonrect)
@@ -2780,25 +2811,31 @@ expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 		  expand_omp_build_assign (&gsi2, vs[j], n1);
 		}
 	      t = fold_convert (itype, unshare_expr (fd->loops[j].n2));
-	      if (fd->loops[j].m2)
+	      if (fd->loops[j].m2 == NULL_TREE)
+		n2 = rect_p ? counts[j] : t;
+	      else if (POINTER_TYPE_P (itype))
+		{
+		  gcc_assert (integer_onep (fd->loops[j].m2));
+		  t = fold_convert (sizetype,
+				    unshare_expr (fd->loops[j].n2));
+		  n2 = fold_build_pointer_plus (vs[j - fd->loops[j].outer], t);
+		}
+	      else
 		{
 		  n2 = fold_convert (itype, unshare_expr (fd->loops[j].m2));
 		  n2 = fold_build2 (MULT_EXPR, itype,
 				    vs[j - fd->loops[j].outer], n2);
 		  n2 = fold_build2 (PLUS_EXPR, itype, n2, t);
 		}
-	      else if (rect_p)
-		n2 = counts[j];
-	      else
-		n2 = t;
 	      n2 = force_gimple_operand_gsi (&gsi2, n2, true, NULL_TREE,
 					     true, GSI_SAME_STMT);
+	      if (POINTER_TYPE_P (itype))
+		itype = signed_type_for (itype);
 	      if (j == fd->last_nonrect)
 		{
 		  gcond *cond_stmt
-		    = gimple_build_cond (fd->loops[j].cond_code, n1, n2,
-					 NULL_TREE, NULL_TREE);
-		  gsi_insert_before (&gsi2, cond_stmt, GSI_SAME_STMT);
+		    = expand_omp_build_cond (&gsi2, fd->loops[i].cond_code,
+					     n1, n2);
 		  e = split_block (cur_bb, cond_stmt);
 		  e->flags = EDGE_TRUE_VALUE;
 		  edge ne = make_edge (cur_bb, next_bb, EDGE_FALSE_VALUE);
@@ -2810,8 +2847,10 @@ expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 					     ? -1 : 1));
 		  t = fold_build2 (PLUS_EXPR, itype,
 				   fold_convert (itype, fd->loops[j].step), t);
-		  t = fold_build2 (PLUS_EXPR, itype, t, n2);
-		  t = fold_build2 (MINUS_EXPR, itype, t, n1);
+		  t = fold_build2 (PLUS_EXPR, itype, t,
+				   fold_convert (itype, n2));
+		  t = fold_build2 (MINUS_EXPR, itype, t,
+				   fold_convert (itype, n1));
 		  tree step = fold_convert (itype, fd->loops[j].step);
 		  if (TYPE_UNSIGNED (itype)
 		      && fd->loops[j].cond_code == GT_EXPR)
@@ -2853,7 +2892,11 @@ expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 		{
 		  tree step
 		    = fold_convert (itype, unshare_expr (fd->loops[j].step));
-		  t = fold_build2 (PLUS_EXPR, itype, vs[j], step);
+		  if (POINTER_TYPE_P (vtype))
+		    t = fold_build_pointer_plus (vs[j], fold_convert (sizetype,
+								      step));
+		  else
+		    t = fold_build2 (PLUS_EXPR, itype, vs[j], step);
 		}
 	      t = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
 					    true, GSI_SAME_STMT);
@@ -2899,7 +2942,10 @@ expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 	    }
 	  for (int j = fd->last_nonrect; j >= fd->first_nonrect; j--)
 	    {
-	      tree itype = TREE_TYPE (fd->loops[j].v);
+	      tree vtype = TREE_TYPE (fd->loops[j].v);
+	      tree itype = vtype;
+	      if (POINTER_TYPE_P (itype))
+		itype = signed_type_for (itype);
 	      bool rect_p = (fd->loops[j].m1 == NULL_TREE
 			     && fd->loops[j].m2 == NULL_TREE
 			     && !fd->loops[j].non_rect_referenced);
@@ -2910,7 +2956,11 @@ expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 		  tree t2
 		    = fold_convert (itype, unshare_expr (fd->loops[j].step));
 		  t = fold_build2 (MULT_EXPR, itype, t, t2);
-		  t = fold_build2 (PLUS_EXPR, itype, n1, t);
+		  if (POINTER_TYPE_P (vtype))
+		    t = fold_build_pointer_plus (n1,
+						 fold_convert (sizetype, t));
+		  else
+		    t = fold_build2 (PLUS_EXPR, itype, n1, t);
 		}
 	      else if (rect_p)
 		{
@@ -2918,7 +2968,8 @@ expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 		  t = fold_build2 (MULT_EXPR, itype, t,
 				   fold_convert (itype, fd->loops[j].step));
 		  if (POINTER_TYPE_P (vtype))
-		    t = fold_build_pointer_plus (fd->loops[j].n1, t);
+		    t = fold_build_pointer_plus (fd->loops[j].n1,
+						 fold_convert (sizetype, t));
 		  else
 		    t = fold_build2 (PLUS_EXPR, itype, fd->loops[j].n1, t);
 		}
@@ -2978,12 +3029,23 @@ expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
 	{
 	  tree itype = TREE_TYPE (fd->loops[i].v);
 
-	  tree t = fold_convert (itype, unshare_expr (fd->loops[i].m2));
-	  t = fold_build2 (MULT_EXPR, itype,
-			   fd->loops[i - fd->loops[i].outer].v, t);
-	  t = fold_build2 (PLUS_EXPR, itype, t,
-			   fold_convert (itype,
-					 unshare_expr (fd->loops[i].n2)));
+	  tree t;
+	  if (POINTER_TYPE_P (itype))
+	    {
+	      gcc_assert (integer_onep (fd->loops[i].m2));
+	      t = fold_convert (sizetype, unshare_expr (fd->loops[i].n2));
+	      t = fold_build_pointer_plus (fd->loops[i - fd->loops[i].outer].v,
+					   t);
+	    }
+	  else
+	    {
+	      t = fold_convert (itype, unshare_expr (fd->loops[i].m2));
+	      t = fold_build2 (MULT_EXPR, itype,
+			       fd->loops[i - fd->loops[i].outer].v, t);
+	      t = fold_build2 (PLUS_EXPR, itype, t,
+			       fold_convert (itype,
+					     unshare_expr (fd->loops[i].n2)));
+	    }
 	  nonrect_bounds[i] = create_tmp_reg (itype, ".bound");
 	  t = force_gimple_operand_gsi (gsi, t, false,
 					NULL_TREE, false,
@@ -3065,10 +3127,16 @@ extract_omp_for_update_vars (struct omp_for_data *fd, tree *nonrect_bounds,
 	      t = l->n1;
 	      if (l->m1)
 		{
-		  tree t2
-		    = fold_build2 (MULT_EXPR, TREE_TYPE (t),
-				   fd->loops[i + 1 - l->outer].v, l->m1);
-		  t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t2, t);
+		  if (POINTER_TYPE_P (TREE_TYPE (l->v)))
+		    t = fold_build_pointer_plus (fd->loops[i + 1 - l->outer].v,
+						 fold_convert (sizetype, t));
+		  else
+		    {
+		      tree t2
+			= fold_build2 (MULT_EXPR, TREE_TYPE (t),
+				       fd->loops[i + 1 - l->outer].v, l->m1);
+		      t = fold_build2 (PLUS_EXPR, TREE_TYPE (t), t2, t);
+		    }
 		}
 	      t = force_gimple_operand_gsi (&gsi, t,
 					    DECL_P (l->v)
@@ -3116,9 +3184,17 @@ extract_omp_for_update_vars (struct omp_for_data *fd, tree *nonrect_bounds,
 		  }
 		if (l->m1)
 		  {
-		    t = fold_build2 (MULT_EXPR, TREE_TYPE (l->m1), l->m1,
-				     fd->loops[i].v);
-		    t = fold_build2 (PLUS_EXPR, TREE_TYPE (l->v), t, l->n1);
+		    if (POINTER_TYPE_P (TREE_TYPE (l->v)))
+		      t = fold_build_pointer_plus (fd->loops[i].v,
+						   fold_convert (sizetype,
+								 l->n1));
+		    else
+		      {
+			t = fold_build2 (MULT_EXPR, TREE_TYPE (l->m1), l->m1,
+					 fd->loops[i].v);
+			t = fold_build2 (PLUS_EXPR, TREE_TYPE (l->v),
+					 t, l->n1);
+		      }
 		    n1 = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
 						   false,
 						   GSI_CONTINUE_LINKING);
@@ -3132,10 +3208,18 @@ extract_omp_for_update_vars (struct omp_for_data *fd, tree *nonrect_bounds,
 						 GSI_CONTINUE_LINKING);
 		if (l->m2)
 		  {
-		    t = fold_build2 (MULT_EXPR, TREE_TYPE (l->m2), l->m2,
-				     fd->loops[i].v);
-		    t = fold_build2 (PLUS_EXPR, TREE_TYPE (nonrect_bounds[j]),
-				     t, unshare_expr (l->n2));
+		    if (POINTER_TYPE_P (TREE_TYPE (l->v)))
+		      t = fold_build_pointer_plus (fd->loops[i].v,
+						   fold_convert (sizetype,
+								 l->n2));
+		    else
+		      {
+			t = fold_build2 (MULT_EXPR, TREE_TYPE (l->m2), l->m2,
+					 fd->loops[i].v);
+			t = fold_build2 (PLUS_EXPR,
+					 TREE_TYPE (nonrect_bounds[j]),
+					 t, unshare_expr (l->n2));
+		      }
 		    n2 = force_gimple_operand_gsi (&gsi2, t, true, NULL_TREE,
 						   false,
 						   GSI_CONTINUE_LINKING);
@@ -4827,17 +4911,8 @@ expand_omp_for_static_nochunk (struct omp_region *region,
       n2 = fold_convert (type, unshare_expr (fd->loop.n2));
       n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
 				     true, GSI_SAME_STMT);
-      gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
-					    NULL_TREE, NULL_TREE);
-      gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
-      if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
-		     expand_omp_regimplify_p, NULL, NULL)
-	  || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
-			expand_omp_regimplify_p, NULL, NULL))
-	{
-	  gsi = gsi_for_stmt (cond_stmt);
-	  gimple_regimplify_operands (cond_stmt, &gsi);
-	}
+      gcond *cond_stmt = expand_omp_build_cond (&gsi, fd->loop.cond_code,
+						n1, n2);
       ep = split_block (entry_bb, cond_stmt);
       ep->flags = EDGE_TRUE_VALUE;
       entry_bb = ep->dest;
@@ -5631,17 +5706,8 @@ expand_omp_for_static_chunk (struct omp_region *region,
       n2 = fold_convert (type, unshare_expr (fd->loop.n2));
       n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
 				     true, GSI_SAME_STMT);
-      gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
-						 NULL_TREE, NULL_TREE);
-      gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
-      if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
-		     expand_omp_regimplify_p, NULL, NULL)
-	  || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
-			expand_omp_regimplify_p, NULL, NULL))
-	{
-	  gsi = gsi_for_stmt (cond_stmt);
-	  gimple_regimplify_operands (cond_stmt, &gsi);
-	}
+      gcond *cond_stmt = expand_omp_build_cond (&gsi, fd->loop.cond_code,
+						n1, n2);
       se = split_block (entry_bb, cond_stmt);
       se->flags = EDGE_TRUE_VALUE;
       entry_bb = se->dest;
@@ -9543,7 +9609,7 @@ get_target_arguments (gimple_stmt_iterator *gsi, gomp_target *tgt_stmt)
   tree clauses = gimple_omp_target_clauses (tgt_stmt);
   tree t, c = omp_find_clause (clauses, OMP_CLAUSE_NUM_TEAMS);
   if (c)
-    t = OMP_CLAUSE_NUM_TEAMS_EXPR (c);
+    t = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (c);
   else
     t = integer_minus_one_node;
   push_target_argument_according_to_value (gsi, GOMP_TARGET_ARG_DEVICE_ALL,
diff --git a/gcc/omp-low.c b/gcc/omp-low.c
index 15e4424..63a47f6 100644
--- a/gcc/omp-low.c
+++ b/gcc/omp-low.c
@@ -3942,7 +3942,8 @@ omp_runtime_api_call (const_tree fndecl)
       "target_memcpy",
       "target_memcpy_rect",
       NULL,
-      /* Now omp_* calls that are available as omp_* and omp_*_.  */
+      /* Now omp_* calls that are available as omp_* and omp_*_; however, the
+	 DECL_NAME is always omp_* without tailing underscore.  */
       "capture_affinity",
       "destroy_allocator",
       "destroy_lock",
@@ -3994,7 +3995,8 @@ omp_runtime_api_call (const_tree fndecl)
       "unset_lock",
       "unset_nest_lock",
       NULL,
-      /* And finally calls available as omp_*, omp_*_ and omp_*_8_.  */
+      /* And finally calls available as omp_*, omp_*_ and omp_*_8_; however,
+	 as DECL_NAME only omp_* and omp_*_8 appear.  */
       "display_env",
       "get_ancestor_thread_num",
       "init_allocator",
@@ -4024,11 +4026,7 @@ omp_runtime_api_call (const_tree fndecl)
       size_t len = strlen (omp_runtime_apis[i]);
       if (strncmp (name + 4, omp_runtime_apis[i], len) == 0
 	  && (name[4 + len] == '\0'
-	      || (mode > 0
-		  && name[4 + len] == '_'
-		  && (name[4 + len + 1] == '\0'
-		      || (mode > 1
-			  && strcmp (name + 4 + len + 1, "8_") == 0)))))
+	      || (mode > 1 && strcmp (name + 4 + len, "_8") == 0)))
 	return true;
     }
   return false;
@@ -4095,9 +4093,26 @@ scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
 			    "OpenMP runtime API call %qD in a region with "
 			    "%<order(concurrent)%> clause", fndecl);
 		}
+	      if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
+		  && omp_runtime_api_call (fndecl)
+		  && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
+		       != strlen ("omp_get_num_teams"))
+		      || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
+				 "omp_get_num_teams") != 0)
+		  && ((IDENTIFIER_LENGTH (DECL_NAME (fndecl))
+		       != strlen ("omp_get_team_num"))
+		      || strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)),
+				 "omp_get_team_num") != 0))
+		{
+		  remove = true;
+		  error_at (gimple_location (stmt),
+			    "OpenMP runtime API call %qD strictly nested in a "
+			    "%<teams%> region", fndecl);
+		}
 	      if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
 		  && (gimple_omp_target_kind (ctx->stmt)
-		      == GF_OMP_TARGET_KIND_REGION))
+		      == GF_OMP_TARGET_KIND_REGION)
+		  && omp_runtime_api_call (fndecl))
 		{
 		  tree tgt_clauses = gimple_omp_target_clauses (ctx->stmt);
 		  tree c = omp_find_clause (tgt_clauses, OMP_CLAUSE_DEVICE);
@@ -13153,6 +13168,19 @@ lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
 		    else if (integer_nonzerop (s))
 		      tkind_zero = tkind;
 		  }
+		if (tkind_zero == tkind
+		    && OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (c)
+		    && (((tkind & GOMP_MAP_FLAG_SPECIAL_BITS)
+			 & ~GOMP_MAP_IMPLICIT)
+			== 0))
+		  {
+		    /* If this is an implicit map, and the GOMP_MAP_IMPLICIT
+		       bits are not interfered by other special bit encodings,
+		       then turn the GOMP_IMPLICIT_BIT flag on for the runtime
+		       to see.  */
+		    tkind |= GOMP_MAP_IMPLICIT;
+		    tkind_zero = tkind;
+		  }
 		break;
 	      case OMP_CLAUSE_FIRSTPRIVATE:
 		gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
@@ -13887,14 +13915,24 @@ lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
 
   tree num_teams = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
 				    OMP_CLAUSE_NUM_TEAMS);
+  tree num_teams_lower = NULL_TREE;
   if (num_teams == NULL_TREE)
     num_teams = build_int_cst (unsigned_type_node, 0);
   else
     {
-      num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams);
+      num_teams_lower = OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (num_teams);
+      if (num_teams_lower)
+	{
+	  num_teams_lower = fold_convert (unsigned_type_node, num_teams_lower);
+	  gimplify_expr (&num_teams_lower, &bind_body, NULL, is_gimple_val,
+			 fb_rvalue);
+	}
+      num_teams = OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (num_teams);
       num_teams = fold_convert (unsigned_type_node, num_teams);
       gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
     }
+  if (num_teams_lower == NULL_TREE)
+    num_teams_lower = num_teams;
   tree thread_limit = omp_find_clause (gimple_omp_teams_clauses (teams_stmt),
 				       OMP_CLAUSE_THREAD_LIMIT);
   if (thread_limit == NULL_TREE)
@@ -13906,6 +13944,30 @@ lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
       gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
 		     fb_rvalue);
     }
+  location_t loc = gimple_location (teams_stmt);
+  tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS4);
+  tree rettype = TREE_TYPE (TREE_TYPE (decl));
+  tree first = create_tmp_var (rettype);
+  gimple_seq_add_stmt (&bind_body,
+		       gimple_build_assign (first, build_one_cst (rettype)));
+  tree llabel = create_artificial_label (loc);
+  gimple_seq_add_stmt (&bind_body, gimple_build_label (llabel));
+  gimple *call
+    = gimple_build_call (decl, 4, num_teams_lower, num_teams, thread_limit,
+			 first);
+  gimple_set_location (call, loc);
+  tree temp = create_tmp_var (rettype);
+  gimple_call_set_lhs (call, temp);
+  gimple_seq_add_stmt (&bind_body, call);
+
+  tree tlabel = create_artificial_label (loc);
+  tree flabel = create_artificial_label (loc);
+  gimple *cond = gimple_build_cond (NE_EXPR, temp, build_zero_cst (rettype),
+				    tlabel, flabel);
+  gimple_seq_add_stmt (&bind_body, cond);
+  gimple_seq_add_stmt (&bind_body, gimple_build_label (tlabel));
+  gimple_seq_add_stmt (&bind_body,
+		       gimple_build_assign (first, build_zero_cst (rettype)));
 
   lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
 			   &bind_body, &dlist, ctx, NULL);
@@ -13914,17 +13976,13 @@ lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
 			   NULL, ctx);
   gimple_seq_add_stmt (&bind_body, teams_stmt);
 
-  location_t loc = gimple_location (teams_stmt);
-  tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS);
-  gimple *call = gimple_build_call (decl, 2, num_teams, thread_limit);
-  gimple_set_location (call, loc);
-  gimple_seq_add_stmt (&bind_body, call);
-
   gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
   gimple_omp_set_body (teams_stmt, NULL);
   gimple_seq_add_seq (&bind_body, olist);
   gimple_seq_add_seq (&bind_body, dlist);
   gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
+  gimple_seq_add_stmt (&bind_body, gimple_build_goto (llabel));
+  gimple_seq_add_stmt (&bind_body, gimple_build_label (flabel));
   gimple_bind_set_body (bind, bind_body);
 
   pop_gimplify_context (bind);
diff --git a/gcc/omp-simd-clone.c b/gcc/omp-simd-clone.c
index b772b7f..4d43a86 100644
--- a/gcc/omp-simd-clone.c
+++ b/gcc/omp-simd-clone.c
@@ -910,11 +910,8 @@ ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
       gimple *stmt;
       if (is_gimple_debug (info->stmt))
 	{
-	  tree vexpr = make_node (DEBUG_EXPR_DECL);
+	  tree vexpr = build_debug_expr_decl (TREE_TYPE (repl));
 	  stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
-	  DECL_ARTIFICIAL (vexpr) = 1;
-	  TREE_TYPE (vexpr) = TREE_TYPE (repl);
-	  SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl)));
 	  repl = vexpr;
 	}
       else
diff --git a/gcc/opt-functions.awk b/gcc/opt-functions.awk
index be92550..9bc8560 100644
--- a/gcc/opt-functions.awk
+++ b/gcc/opt-functions.awk
@@ -356,7 +356,7 @@ function search_var_name(name, opt_numbers, opts, flags, n_opts)
     return ""
 }
 
-function integer_range_info(range_option, init, option)
+function integer_range_info(range_option, init, option, uinteger_used)
 {
     if (range_option != "") {
 	ival = init + 0;
@@ -364,6 +364,8 @@ function integer_range_info(range_option, init, option)
 	end = nth_arg(1, range_option) + 0;
 	if (init != "" && init != "-1" && (ival < start || ival > end))
 	  print "#error initial value " init " of '" option "' must be in range [" start "," end "]"
+	if (uinteger_used && start < 0)
+	  print "#error '" option"': negative IntegerRange (" start ", " end ") cannot be combined with UInteger"
 	return start ", " end
     }
     else
diff --git a/gcc/optc-gen.awk b/gcc/optc-gen.awk
index 77e598e..ebc1a02 100644
--- a/gcc/optc-gen.awk
+++ b/gcc/optc-gen.awk
@@ -422,7 +422,7 @@ for (i = 0; i < n_opts; i++) {
 		       cl_flags, cl_bit_fields)
 	printf("    %s, %s, %s }%s\n", var_ref(opts[i], flags[i]),
 	       var_set(flags[i]), integer_range_info(opt_args("IntegerRange", flags[i]),
-		    opt_args("Init", flags[i]), opts[i]), comma)
+		    opt_args("Init", flags[i]), opts[i], flag_set_p("UInteger", flags[i])), comma)
 
 	# Bump up the informational option index.
 	++optindex
diff --git a/gcc/opts.c b/gcc/opts.c
index 65fe192..175b463 100644
--- a/gcc/opts.c
+++ b/gcc/opts.c
@@ -1349,6 +1349,53 @@ finish_options (struct gcc_options *opts, struct gcc_options *opts_set,
     SET_OPTION_IF_UNSET (opts, opts_set, flag_vect_cost_model,
 			 VECT_COST_MODEL_CHEAP);
 
+  /* One could use EnabledBy, but it would lead to a circular dependency.  */
+  if (!OPTION_SET_P (flag_var_tracking_uninit))
+     flag_var_tracking_uninit = flag_var_tracking;
+
+  if (!OPTION_SET_P (flag_var_tracking_assignments))
+    flag_var_tracking_assignments
+      = (flag_var_tracking
+	 && !(flag_selective_scheduling || flag_selective_scheduling2));
+
+  if (flag_var_tracking_assignments_toggle)
+    flag_var_tracking_assignments = !flag_var_tracking_assignments;
+
+  if (flag_var_tracking_assignments && !flag_var_tracking)
+    flag_var_tracking = flag_var_tracking_assignments = -1;
+
+  if (flag_var_tracking_assignments
+      && (flag_selective_scheduling || flag_selective_scheduling2))
+    warning_at (loc, 0,
+		"var-tracking-assignments changes selective scheduling");
+
+  if (flag_syntax_only)
+    {
+      write_symbols = NO_DEBUG;
+      profile_flag = 0;
+    }
+
+  if (flag_gtoggle)
+    {
+      /* Make sure to process -gtoggle only once.  */
+      flag_gtoggle = false;
+      if (debug_info_level == DINFO_LEVEL_NONE)
+	{
+	  debug_info_level = DINFO_LEVEL_NORMAL;
+
+	  if (write_symbols == NO_DEBUG)
+	    write_symbols = PREFERRED_DEBUGGING_TYPE;
+	}
+      else
+	debug_info_level = DINFO_LEVEL_NONE;
+    }
+
+  if (!OPTION_SET_P (debug_nonbind_markers_p))
+    debug_nonbind_markers_p
+      = (optimize
+	 && debug_info_level >= DINFO_LEVEL_NORMAL
+	 && dwarf_debuginfo_p ()
+	 && !(flag_selective_scheduling || flag_selective_scheduling2));
 }
 
 #define LEFT_COLUMN	27
@@ -2564,6 +2611,26 @@ common_handle_option (struct gcc_options *opts,
       /* Currently handled in a prescan.  */
       break;
 
+    case OPT_Wattributes_:
+      if (lang_mask == CL_DRIVER)
+	break;
+
+      if (value)
+	{
+	  error_at (loc, "arguments ignored for %<-Wattributes=%>; use "
+		    "%<-Wno-attributes=%> instead");
+	  break;
+	}
+      else if (arg[strlen (arg) - 1] == ',')
+	{
+	  error_at (loc, "trailing %<,%> in arguments for "
+		    "%<-Wno-attributes=%>");
+	  break;
+	}
+
+      add_comma_separated_to_vector (&opts->x_flag_ignored_attributes, arg);
+      break;
+
     case OPT_Werror:
       dc->warning_as_error_requested = value;
       break;
@@ -2689,6 +2756,10 @@ common_handle_option (struct gcc_options *opts,
       dc->column_origin = value;
       break;
 
+    case OPT_fdiagnostics_escape_format_:
+      dc->escape_format = (enum diagnostics_escape_format)value;
+      break;
+
     case OPT_fdiagnostics_show_cwe:
       dc->show_cwe = value;
       break;
diff --git a/gcc/params.opt b/gcc/params.opt
index 83b3db6..e725c99 100644
--- a/gcc/params.opt
+++ b/gcc/params.opt
@@ -136,7 +136,7 @@ Maximum number of outgoing edges in a switch before EVRP will not process it.
 
 -param=evrp-mode=
 Common Joined Var(param_evrp_mode) Enum(evrp_mode) Init(EVRP_MODE_RVRP_ONLY) Param Optimization
---param=evrp-mode=[legacy|ranger|legacy-first|ranger-first|trace|gori|cache|tracegori|debug] Specifies the mode Early VRP should operate in.
+--param=evrp-mode=[legacy|ranger|legacy-first|ranger-first] Specifies the mode Early VRP should operate in.
 
 Enum
 Name(evrp_mode) Type(enum evrp_mode) UnknownError(unknown evrp mode %qs)
@@ -153,26 +153,6 @@ Enum(evrp_mode) String(legacy-first) Value(EVRP_MODE_EVRP_FIRST)
 EnumValue
 Enum(evrp_mode) String(ranger-first) Value(EVRP_MODE_RVRP_FIRST)
 
-EnumValue
-Enum(evrp_mode) String(trace) Value(EVRP_MODE_TRACE)
-
-EnumValue
-Enum(evrp_mode) String(cache) Value(EVRP_MODE_CACHE)
-
-EnumValue
-Enum(evrp_mode) String(gori) Value(EVRP_MODE_GORI)
-
-EnumValue
-Enum(evrp_mode) String(tracegori) Value(EVRP_MODE_TRACE_GORI)
-
-EnumValue
-Enum(evrp_mode) String(debug) Value(EVRP_MODE_DEBUG)
-
--param=ranger-logical-depth=
-Common Joined UInteger Var(param_ranger_logical_depth) Init(6) IntegerRange(1, 999) Param Optimization
-Maximum depth of logical expression evaluation ranger will look through when
-evaluating outgoing edge ranges.
-
 -param=fsm-scale-path-blocks=
 Common Joined UInteger Var(param_fsm_scale_path_blocks) Init(3) IntegerRange(1, 10) Param Optimization
 Scale factor to apply to the number of blocks in a threading path when comparing to the number of (scaled) statements.
@@ -293,6 +273,10 @@ The size of translation unit that IPA-CP pass considers large.
 Common Joined UInteger Var(param_ipa_cp_value_list_size) Init(8) Param Optimization
 Maximum size of a list of values associated with each parameter for interprocedural constant propagation.
 
+-param=ipa-cp-profile-count-base=
+Common Joined UInteger Var(param_ipa_cp_profile_count_base) Init(10) IntegerRange(0, 100) Param Optimization
+When using profile feedback, use the edge at this percentage position in frequncy histogram as the bases for IPA-CP heuristics.
+
 -param=ipa-jump-function-lookups=
 Common Joined UInteger Var(param_ipa_jump_function_lookups) Init(8) Param Optimization
 Maximum number of statements visited during jump function offset discovery.
@@ -418,7 +402,7 @@ Common Joined UInteger Var(param_lim_expensive) Init(20) Param Optimization
 The minimum cost of an expensive expression in the loop invariant motion.
 
 -param=logical-op-non-short-circuit=
-Common Joined UInteger Var(param_logical_op_non_short_circuit) Init(-1) IntegerRange(-1, 1) Param
+Common Joined UInteger Var(param_logical_op_non_short_circuit) Init(-1) IntegerRange(0, 1) Param
 True if a non-short-circuit operation is optimal.
 
 -param=loop-block-tile-size=
@@ -893,6 +877,36 @@ The minimum constant stride beyond which we should use prefetch hints for.
 Common Joined UInteger Var(param_profile_func_internal_id) IntegerRange(0, 1) Param
 Use internal function id in profile lookup.
 
+-param=ranger-debug=
+Common Joined Var(param_ranger_debug) Enum(ranger_debug) Init(RANGER_DEBUG_NONE) Param Optimization
+--param=ranger-debug=[none|trace|gori|cache|tracegori|all] Specifies the output mode for debugging ranger.
+
+Enum
+Name(ranger_debug) Type(enum ranger_debug) UnknownError(unknown ranger debug mode %qs)
+
+EnumValue
+Enum(ranger_debug) String(none) Value(RANGER_DEBUG_NONE)
+
+EnumValue
+Enum(ranger_debug) String(trace) Value(RANGER_DEBUG_TRACE)
+
+EnumValue
+Enum(ranger_debug) String(cache) Value(RANGER_DEBUG_TRACE_CACHE)
+
+EnumValue
+Enum(ranger_debug) String(gori) Value(RANGER_DEBUG_GORI)
+
+EnumValue
+Enum(ranger_debug) String(tracegori) Value(RANGER_DEBUG_TRACE_GORI)
+
+EnumValue
+Enum(ranger_debug) String(all) Value(RANGER_DEBUG_ALL)
+
+-param=ranger-logical-depth=
+Common Joined UInteger Var(param_ranger_logical_depth) Init(6) IntegerRange(1, 999) Param Optimization
+Maximum depth of logical expression evaluation ranger will look through when
+evaluating outgoing edge ranges.
+
 -param=rpo-vn-max-loop-depth=
 Common Joined UInteger Var(param_rpo_vn_max_loop_depth) Init(7) IntegerRange(2, 65536) Param Optimization
 Maximum depth of a loop nest to fully value-number optimistically.
@@ -1022,7 +1036,7 @@ Common Joined UInteger Var(param_modref_max_tests) Init(64) Param Optimization
 Maximum number of tests performed by modref query.
 
 -param=modref-max-depth=
-Common Joined UInteger Var(param_modref_max_depth) Init(256) Param Optimization
+Common Joined UInteger Var(param_modref_max_depth) Init(256) IntegerRange(1, 65536) Param Optimization
 Maximum depth of DFS walk used by modref escape analysis.
 
 -param=modref-max-escape-points=
@@ -1030,9 +1044,22 @@ Common Joined UInteger Var(param_modref_max_escape_points) Init(256) Param Optim
 Maximum number of escape points tracked by modref per SSA-name.
 
 -param=modref-max-adjustments=
-Common Joined UInteger Var(param_modref_max_adjustments) Init(8) IntegerRange (0, 254) Param Optimization
+Common Joined UInteger Var(param_modref_max_adjustments) Init(8) IntegerRange(0, 254) Param Optimization
 Maximum number of times a given range is adjusted during the dataflow.
 
+-param=threader-debug=
+Common Joined Var(param_threader_debug) Enum(threader_debug) Init(THREADER_DEBUG_NONE) Param Optimization
+--param=threader-debug=[none|all] Enables verbose dumping of the threader solver.
+
+Enum
+Name(threader_debug) Type(enum threader_debug) UnknownError(unknown threader debug mode %qs)
+
+EnumValue
+Enum(threader_debug) String(none) Value(THREADER_DEBUG_NONE)
+
+EnumValue
+Enum(threader_debug) String(all) Value(THREADER_DEBUG_ALL)
+
 -param=tm-max-aggregate-size=
 Common Joined UInteger Var(param_tm_max_aggregate_size) Init(9) Param Optimization
 Size in bytes after which thread-local aggregates should be instrumented with the logging functions instead of save/restore pairs.
@@ -1118,7 +1145,7 @@ Common Joined UInteger Var(param_vect_epilogues_nomask) Init(1) IntegerRange(0,
 Enable loop epilogue vectorization using smaller vector size.
 
 -param=vect-max-peeling-for-alignment=
-Common Joined UInteger Var(param_vect_max_peeling_for_alignment) Init(-1) IntegerRange(-1, 64) Param Optimization
+Common Joined UInteger Var(param_vect_max_peeling_for_alignment) Init(-1) IntegerRange(0, 64) Param Optimization
 Maximum number of loop peels to enhance alignment of data references in a loop.
 
 -param=vect-max-version-for-alias-checks=
@@ -1137,4 +1164,21 @@ Controls how loop vectorizer uses partial vectors.  0 means never, 1 means only
 Common Joined UInteger Var(param_vect_inner_loop_cost_factor) Init(50) IntegerRange(1, 10000) Param Optimization
 The maximum factor which the loop vectorizer applies to the cost of statements in an inner loop relative to the loop being vectorized.
 
+-param=vrp1-mode=
+Common Joined Var(param_vrp1_mode) Enum(vrp_mode) Init(VRP_MODE_VRP) Param Optimization
+--param=vrp1-mode=[vrp|ranger] Specifies the mode VRP1 should operate in.
+
+-param=vrp2-mode=
+Common Joined Var(param_vrp2_mode) Enum(vrp_mode) Init(VRP_MODE_RANGER) Param Optimization
+--param=vrp2-mode=[vrp|ranger] Specifies the mode VRP2 should operate in.
+
+Enum
+Name(vrp_mode) Type(enum vrp_mode) UnknownError(unknown vrp mode %qs)
+
+EnumValue
+Enum(vrp_mode) String(vrp) Value(VRP_MODE_VRP)
+
+EnumValue
+Enum(vrp_mode) String(ranger) Value(VRP_MODE_RANGER)
+
 ; This comment is to ensure we retain the blank line above.
diff --git a/gcc/passes.def b/gcc/passes.def
index 4c54176..b583d17 100644
--- a/gcc/passes.def
+++ b/gcc/passes.def
@@ -81,7 +81,7 @@ along with GCC; see the file COPYING3.  If not see
 	  /* After CCP we rewrite no longer addressed locals into SSA
 	     form if possible.  */
 	  NEXT_PASS (pass_forwprop);
-          NEXT_PASS (pass_early_thread_jumps);
+          NEXT_PASS (pass_early_thread_jumps, /*first=*/true);
 	  NEXT_PASS (pass_sra_early);
 	  /* pass_build_ealias is a dummy pass that ensures that we
 	     execute TODO_rebuild_alias at this point.  */
@@ -92,13 +92,13 @@ along with GCC; see the file COPYING3.  If not see
           NEXT_PASS (pass_dse);
 	  NEXT_PASS (pass_cd_dce, false /* update_address_taken_p */);
 	  NEXT_PASS (pass_phiopt, true /* early_p */);
-	  NEXT_PASS (pass_modref);
 	  NEXT_PASS (pass_tail_recursion);
 	  NEXT_PASS (pass_if_to_switch);
 	  NEXT_PASS (pass_convert_switch);
 	  NEXT_PASS (pass_cleanup_eh);
 	  NEXT_PASS (pass_profile);
 	  NEXT_PASS (pass_local_pure_const);
+	  NEXT_PASS (pass_modref);
 	  /* Split functions creates parts that are not run through
 	     early optimizations again.  It is thus good idea to do this
 	      late.  */
@@ -210,9 +210,8 @@ along with GCC; see the file COPYING3.  If not see
       NEXT_PASS (pass_return_slot);
       NEXT_PASS (pass_fre, true /* may_iterate */);
       NEXT_PASS (pass_merge_phi);
-      NEXT_PASS (pass_thread_jumps);
+      NEXT_PASS (pass_thread_jumps_full, /*first=*/true);
       NEXT_PASS (pass_vrp, true /* warn_array_bounds_p */);
-      NEXT_PASS (pass_vrp_threader);
       NEXT_PASS (pass_dse);
       NEXT_PASS (pass_dce);
       /* pass_stdarg is always run and at this point we execute
@@ -234,7 +233,7 @@ along with GCC; see the file COPYING3.  If not see
 	 propagations have already run, but before some more dead code
 	 is removed, and this place fits nicely.  Remember this when
 	 trying to move or duplicate pass_dominator somewhere earlier.  */
-      NEXT_PASS (pass_thread_jumps);
+      NEXT_PASS (pass_thread_jumps, /*first=*/true);
       NEXT_PASS (pass_dominator, true /* may_peel_loop_headers_p */);
       /* Threading can leave many const/copy propagations in the IL.
 	 Clean them up.  Failure to do so well can lead to false
@@ -333,12 +332,11 @@ along with GCC; see the file COPYING3.  If not see
       NEXT_PASS (pass_fre, false /* may_iterate */);
       /* After late FRE we rewrite no longer addressed locals into SSA
          form if possible.  */
-      NEXT_PASS (pass_thread_jumps);
+      NEXT_PASS (pass_thread_jumps, /*first=*/false);
       NEXT_PASS (pass_dominator, false /* may_peel_loop_headers_p */);
       NEXT_PASS (pass_strlen);
-      NEXT_PASS (pass_thread_jumps);
+      NEXT_PASS (pass_thread_jumps_full, /*first=*/false);
       NEXT_PASS (pass_vrp, false /* warn_array_bounds_p */);
-      NEXT_PASS (pass_vrp_threader);
       /* Run CCP to compute alignment and nonzero bits.  */
       NEXT_PASS (pass_ccp, true /* nonzero_p */);
       NEXT_PASS (pass_warn_restrict);
@@ -362,9 +360,11 @@ along with GCC; see the file COPYING3.  If not see
          number of false positives from it.  */
       NEXT_PASS (pass_split_crit_edges);
       NEXT_PASS (pass_late_warn_uninitialized);
-      NEXT_PASS (pass_uncprop);
       NEXT_PASS (pass_local_pure_const);
       NEXT_PASS (pass_modref);
+      /* uncprop replaces constants by SSA names.  This makes analysis harder
+	 and thus it should be run last.  */
+      NEXT_PASS (pass_uncprop);
   POP_INSERT_PASSES ()
   NEXT_PASS (pass_all_optimizations_g);
   PUSH_INSERT_PASSES_WITHIN (pass_all_optimizations_g)
@@ -395,9 +395,11 @@ along with GCC; see the file COPYING3.  If not see
          number of false positives from it.  */
       NEXT_PASS (pass_split_crit_edges);
       NEXT_PASS (pass_late_warn_uninitialized);
-      NEXT_PASS (pass_uncprop);
       NEXT_PASS (pass_local_pure_const);
       NEXT_PASS (pass_modref);
+      /* uncprop replaces constants by SSA names.  This makes analysis harder
+	 and thus it should be run last.  */
+      NEXT_PASS (pass_uncprop);
   POP_INSERT_PASSES ()
   NEXT_PASS (pass_tm_init);
   PUSH_INSERT_PASSES_WITHIN (pass_tm_init)
@@ -419,6 +421,8 @@ along with GCC; see the file COPYING3.  If not see
   NEXT_PASS (pass_lower_resx);
   NEXT_PASS (pass_nrv);
   NEXT_PASS (pass_gimple_isel);
+  NEXT_PASS (pass_harden_conditional_branches);
+  NEXT_PASS (pass_harden_compares);
   NEXT_PASS (pass_cleanup_cfg_post_optimizing);
   NEXT_PASS (pass_warn_function_noreturn);
   NEXT_PASS (pass_warn_access);
diff --git a/gcc/plugin.h b/gcc/plugin.h
index 1640e25..5556763 100644
--- a/gcc/plugin.h
+++ b/gcc/plugin.h
@@ -197,7 +197,9 @@ invoke_plugin_callbacks (int event ATTRIBUTE_UNUSED,
 /* In attribs.c.  */
 
 extern void register_attribute (const struct attribute_spec *attr);
+/* The default argument for the third parameter is given in attribs.h.  */
 extern struct scoped_attributes* register_scoped_attributes (const struct attribute_spec *,
-							     const char *);
+							     const char *,
+							     bool);
 
 #endif /* PLUGIN_H */
diff --git a/gcc/pointer-query.cc b/gcc/pointer-query.cc
index 910f452..a0e4543 100644
--- a/gcc/pointer-query.cc
+++ b/gcc/pointer-query.cc
@@ -43,8 +43,8 @@
 #include "tree-ssanames.h"
 #include "target.h"
 
-static bool compute_objsize_r (tree, int, access_ref *, ssa_name_limit_t &,
-			       pointer_query *);
+static bool compute_objsize_r (tree, gimple *, int, access_ref *,
+			       ssa_name_limit_t &, pointer_query *);
 
 /* Wrapper around the wide_int overload of get_range that accepts
    offset_int instead.  For middle end expressions returns the same
@@ -115,7 +115,7 @@ get_offset_range (tree x, gimple *stmt, offset_int r[2], range_query *rvals)
 
 static tree
 gimple_call_return_array (gimple *stmt, offset_int offrng[2], bool *past_end,
-			  range_query *rvals)
+			  ssa_name_limit_t &snlim, pointer_query *qry)
 {
   /* Clear and set below for the rare function(s) that might return
      a past-the-end pointer.  */
@@ -191,7 +191,7 @@ gimple_call_return_array (gimple *stmt, offset_int offrng[2], bool *past_end,
 	offrng[0] = 0;
 	offrng[1] = HOST_WIDE_INT_M1U;
 	tree off = gimple_call_arg (stmt, 2);
-	bool off_valid = get_offset_range (off, stmt, offrng, rvals);
+	bool off_valid = get_offset_range (off, stmt, offrng, qry->rvals);
 	if (!off_valid || offrng[0] != offrng[1])
 	  {
 	    /* If the offset is either indeterminate or in some range,
@@ -199,7 +199,7 @@ gimple_call_return_array (gimple *stmt, offset_int offrng[2], bool *past_end,
 	       of the source object.  */
 	    access_ref aref;
 	    tree src = gimple_call_arg (stmt, 1);
-	    if (compute_objsize (src, 1, &aref, rvals)
+	    if (compute_objsize (src, stmt, 1, &aref, qry)
 		&& aref.sizrng[1] < offrng[1])
 	      offrng[1] = aref.sizrng[1];
 	  }
@@ -212,7 +212,7 @@ gimple_call_return_array (gimple *stmt, offset_int offrng[2], bool *past_end,
     case BUILT_IN_MEMCHR:
       {
 	tree off = gimple_call_arg (stmt, 2);
-	if (get_offset_range (off, stmt, offrng, rvals))
+	if (get_offset_range (off, stmt, offrng, qry->rvals))
 	  offrng[1] -= 1;
 	else
 	  offrng[1] = HOST_WIDE_INT_M1U;
@@ -233,7 +233,7 @@ gimple_call_return_array (gimple *stmt, offset_int offrng[2], bool *past_end,
       {
 	access_ref aref;
 	tree src = gimple_call_arg (stmt, 1);
-	if (compute_objsize (src, 1, &aref, rvals))
+	if (compute_objsize_r (src, stmt, 1, &aref, snlim, qry))
 	  offrng[1] = aref.sizrng[1] - 1;
 	else
 	  offrng[1] = HOST_WIDE_INT_M1U;
@@ -250,7 +250,7 @@ gimple_call_return_array (gimple *stmt, offset_int offrng[2], bool *past_end,
 	   and the source object size.  */
 	offrng[1] = HOST_WIDE_INT_M1U;
 	tree off = gimple_call_arg (stmt, 2);
-	if (!get_offset_range (off, stmt, offrng, rvals)
+	if (!get_offset_range (off, stmt, offrng, qry->rvals)
 	    || offrng[0] != offrng[1])
 	  {
 	    /* If the offset is either indeterminate or in some range,
@@ -258,7 +258,7 @@ gimple_call_return_array (gimple *stmt, offset_int offrng[2], bool *past_end,
 	       of the source object.  */
 	    access_ref aref;
 	    tree src = gimple_call_arg (stmt, 1);
-	    if (compute_objsize (src, 1, &aref, rvals)
+	    if (compute_objsize_r (src, stmt, 1, &aref, snlim, qry)
 		&& aref.sizrng[1] < offrng[1])
 	      offrng[1] = aref.sizrng[1];
 	  }
@@ -445,7 +445,7 @@ get_size_range (tree exp, tree range[2], int flags /* = 0 */)
 
 tree
 gimple_call_alloc_size (gimple *stmt, wide_int rng1[2] /* = NULL */,
-			range_query * /* = NULL */)
+			range_query *qry /* = NULL */)
 {
   if (!stmt || !is_gimple_call (stmt))
     return NULL_TREE;
@@ -503,7 +503,7 @@ gimple_call_alloc_size (gimple *stmt, wide_int rng1[2] /* = NULL */,
   {
     tree r[2];
     /* Determine the largest valid range size, including zero.  */
-    if (!get_size_range (size, r, SR_ALLOW_ZERO | SR_USE_LARGEST))
+    if (!get_size_range (qry, size, stmt, r, SR_ALLOW_ZERO | SR_USE_LARGEST))
       return NULL_TREE;
     rng1[0] = wi::to_wide (r[0], prec);
     rng1[1] = wi::to_wide (r[1], prec);
@@ -519,7 +519,7 @@ gimple_call_alloc_size (gimple *stmt, wide_int rng1[2] /* = NULL */,
   {
     tree r[2];
       /* As above, use the full non-negative range on failure.  */
-    if (!get_size_range (n, r, SR_ALLOW_ZERO | SR_USE_LARGEST))
+    if (!get_size_range (qry, n, stmt, r, SR_ALLOW_ZERO | SR_USE_LARGEST))
       return NULL_TREE;
     rng2[0] = wi::to_wide (r[0], prec);
     rng2[1] = wi::to_wide (r[1], prec);
@@ -546,7 +546,7 @@ gimple_call_alloc_size (gimple *stmt, wide_int rng1[2] /* = NULL */,
    Set STATIC_ARRAY if the array parameter has been declared [static].
    Return the function parameter on success and null otherwise.  */
 
-tree
+static tree
 gimple_parm_array_size (tree ptr, wide_int rng[2],
 			bool *static_array /* = NULL */)
 {
@@ -596,10 +596,17 @@ gimple_parm_array_size (tree ptr, wide_int rng[2],
   return var;
 }
 
-access_ref::access_ref (tree bound /* = NULL_TREE */,
+/* Given a statement STMT, set the bounds of the reference to at most
+   as many bytes as BOUND or unknown when null, and at least one when
+   the MINACCESS is true unless BOUND is a constant zero.  STMT is
+   used for context to get accurate range info.  */
+
+access_ref::access_ref (range_query *qry /* = nullptr */,
+			tree bound /* = NULL_TREE */,
+			gimple *stmt /* = nullptr */,
 			bool minaccess /* = false */)
-: ref (), eval ([](tree x){ return x; }), deref (), trail1special (true),
-  base0 (true), parmarray ()
+  : ref (), eval ([](tree x){ return x; }), deref (), trail1special (true),
+    base0 (true), parmarray ()
 {
   /* Set to valid.  */
   offrng[0] = offrng[1] = 0;
@@ -615,7 +622,7 @@ access_ref::access_ref (tree bound /* = NULL_TREE */,
      set the bounds of the access to reflect both it and MINACCESS.
      BNDRNG[0] is the size of the minimum access.  */
   tree rng[2];
-  if (bound && get_size_range (bound, rng, SR_ALLOW_ZERO))
+  if (bound && get_size_range (qry, bound, stmt, rng, SR_ALLOW_ZERO))
     {
       bndrng[0] = wi::to_offset (rng[0]);
       bndrng[1] = wi::to_offset (rng[1]);
@@ -696,7 +703,8 @@ access_ref::get_ref (vec<access_ref> *all_refs,
     {
       access_ref phi_arg_ref;
       tree arg = gimple_phi_arg_def (phi_stmt, i);
-      if (!compute_objsize_r (arg, ostype, &phi_arg_ref, *psnlim, qry)
+      if (!compute_objsize_r (arg, phi_stmt, ostype, &phi_arg_ref, *psnlim,
+			      qry)
 	  || phi_arg_ref.sizrng[0] < 0)
 	/* A PHI with all null pointer arguments.  */
 	return NULL_TREE;
@@ -1312,7 +1320,7 @@ pointer_query::get_ref (tree ptr, int ostype /* = 1 */) const
    there or compute it and insert it into the cache if it's nonnonull.  */
 
 bool
-pointer_query::get_ref (tree ptr, access_ref *pref, int ostype /* = 1 */)
+pointer_query::get_ref (tree ptr, gimple *stmt, access_ref *pref, int ostype /* = 1 */)
 {
   const unsigned version
     = TREE_CODE (ptr) == SSA_NAME ? SSA_NAME_VERSION (ptr) : 0;
@@ -1335,7 +1343,7 @@ pointer_query::get_ref (tree ptr, access_ref *pref, int ostype /* = 1 */)
       ++misses;
     }
 
-  if (!compute_objsize (ptr, ostype, pref, this))
+  if (!compute_objsize (ptr, stmt, ostype, pref, this))
     {
       ++failures;
       return false;
@@ -1502,7 +1510,7 @@ static bool
 handle_min_max_size (tree ptr, int ostype, access_ref *pref,
 		     ssa_name_limit_t &snlim, pointer_query *qry)
 {
-  const gimple *stmt = SSA_NAME_DEF_STMT (ptr);
+  gimple *stmt = SSA_NAME_DEF_STMT (ptr);
   const tree_code code = gimple_assign_rhs_code (stmt);
 
   /* In a valid MAX_/MIN_EXPR both operands must refer to the same array.
@@ -1512,7 +1520,7 @@ handle_min_max_size (tree ptr, int ostype, access_ref *pref,
      for the expression.  */
   access_ref aref[2] = { *pref, *pref };
   tree arg1 = gimple_assign_rhs1 (stmt);
-  if (!compute_objsize_r (arg1, ostype, &aref[0], snlim, qry))
+  if (!compute_objsize_r (arg1, stmt, ostype, &aref[0], snlim, qry))
     {
       aref[0].base0 = false;
       aref[0].offrng[0] = aref[0].offrng[1] = 0;
@@ -1521,7 +1529,7 @@ handle_min_max_size (tree ptr, int ostype, access_ref *pref,
     }
 
   tree arg2 = gimple_assign_rhs2 (stmt);
-  if (!compute_objsize_r (arg2, ostype, &aref[1], snlim, qry))
+  if (!compute_objsize_r (arg2, stmt, ostype, &aref[1], snlim, qry))
     {
       aref[1].base0 = false;
       aref[1].offrng[0] = aref[1].offrng[1] = 0;
@@ -1589,8 +1597,9 @@ handle_min_max_size (tree ptr, int ostype, access_ref *pref,
    on success and false on failure.  */
 
 static bool
-handle_array_ref (tree aref, bool addr, int ostype, access_ref *pref,
-		  ssa_name_limit_t &snlim, pointer_query *qry)
+handle_array_ref (tree aref, gimple *stmt, bool addr, int ostype,
+		  access_ref *pref, ssa_name_limit_t &snlim,
+		  pointer_query *qry)
 {
   gcc_assert (TREE_CODE (aref) == ARRAY_REF);
 
@@ -1603,7 +1612,7 @@ handle_array_ref (tree aref, bool addr, int ostype, access_ref *pref,
        of known bound.  */
     return false;
 
-  if (!compute_objsize_r (arefop, ostype, pref, snlim, qry))
+  if (!compute_objsize_r (arefop, stmt, ostype, pref, snlim, qry))
     return false;
 
   offset_int orng[2];
@@ -1668,7 +1677,7 @@ handle_array_ref (tree aref, bool addr, int ostype, access_ref *pref,
    MREF.  Return true on success and false on failure.  */
 
 static bool
-handle_mem_ref (tree mref, int ostype, access_ref *pref,
+handle_mem_ref (tree mref, gimple *stmt, int ostype, access_ref *pref,
 		ssa_name_limit_t &snlim, pointer_query *qry)
 {
   gcc_assert (TREE_CODE (mref) == MEM_REF);
@@ -1690,7 +1699,7 @@ handle_mem_ref (tree mref, int ostype, access_ref *pref,
     }
 
   tree mrefop = TREE_OPERAND (mref, 0);
-  if (!compute_objsize_r (mrefop, ostype, pref, snlim, qry))
+  if (!compute_objsize_r (mrefop, stmt, ostype, pref, snlim, qry))
     return false;
 
   offset_int orng[2];
@@ -1723,7 +1732,7 @@ handle_mem_ref (tree mref, int ostype, access_ref *pref,
    to influence code generation or optimization.  */
 
 static bool
-compute_objsize_r (tree ptr, int ostype, access_ref *pref,
+compute_objsize_r (tree ptr, gimple *stmt, int ostype, access_ref *pref,
 		   ssa_name_limit_t &snlim, pointer_query *qry)
 {
   STRIP_NOPS (ptr);
@@ -1774,7 +1783,7 @@ compute_objsize_r (tree ptr, int ostype, access_ref *pref,
   if (code == BIT_FIELD_REF)
     {
       tree ref = TREE_OPERAND (ptr, 0);
-      if (!compute_objsize_r (ref, ostype, pref, snlim, qry))
+      if (!compute_objsize_r (ref, stmt, ostype, pref, snlim, qry))
 	return false;
 
       offset_int off = wi::to_offset (pref->eval (TREE_OPERAND (ptr, 2)));
@@ -1796,7 +1805,7 @@ compute_objsize_r (tree ptr, int ostype, access_ref *pref,
 	  /* In OSTYPE zero (for raw memory functions like memcpy), use
 	     the maximum size instead if the identity of the enclosing
 	     object cannot be determined.  */
-	  if (!compute_objsize_r (ref, ostype, pref, snlim, qry))
+	  if (!compute_objsize_r (ref, stmt, ostype, pref, snlim, qry))
 	    return false;
 
 	  /* Otherwise, use the size of the enclosing object and add
@@ -1850,15 +1859,15 @@ compute_objsize_r (tree ptr, int ostype, access_ref *pref,
     }
 
   if (code == ARRAY_REF)
-    return handle_array_ref (ptr, addr, ostype, pref, snlim, qry);
+    return handle_array_ref (ptr, stmt, addr, ostype, pref, snlim, qry);
 
   if (code == MEM_REF)
-    return handle_mem_ref (ptr, ostype, pref, snlim, qry);
+    return handle_mem_ref (ptr, stmt, ostype, pref, snlim, qry);
 
   if (code == TARGET_MEM_REF)
     {
       tree ref = TREE_OPERAND (ptr, 0);
-      if (!compute_objsize_r (ref, ostype, pref, snlim, qry))
+      if (!compute_objsize_r (ref, stmt, ostype, pref, snlim, qry))
 	return false;
 
       /* TODO: Handle remaining operands.  Until then, add maximum offset.  */
@@ -1903,7 +1912,7 @@ compute_objsize_r (tree ptr, int ostype, access_ref *pref,
   if (code == POINTER_PLUS_EXPR)
     {
       tree ref = TREE_OPERAND (ptr, 0);
-      if (!compute_objsize_r (ref, ostype, pref, snlim, qry))
+      if (!compute_objsize_r (ref, stmt, ostype, pref, snlim, qry))
 	return false;
 
       /* Clear DEREF since the offset is being applied to the target
@@ -1922,7 +1931,7 @@ compute_objsize_r (tree ptr, int ostype, access_ref *pref,
   if (code == VIEW_CONVERT_EXPR)
     {
       ptr = TREE_OPERAND (ptr, 0);
-      return compute_objsize_r (ptr, ostype, pref, snlim, qry);
+      return compute_objsize_r (ptr, stmt, ostype, pref, snlim, qry);
     }
 
   if (code == SSA_NAME)
@@ -1951,7 +1960,7 @@ compute_objsize_r (tree ptr, int ostype, access_ref *pref,
 	    }
 	}
 
-      gimple *stmt = SSA_NAME_DEF_STMT (ptr);
+      stmt = SSA_NAME_DEF_STMT (ptr);
       if (is_gimple_call (stmt))
 	{
 	  /* If STMT is a call to an allocation function get the size
@@ -1979,9 +1988,9 @@ compute_objsize_r (tree ptr, int ostype, access_ref *pref,
 	      bool past_end;
 	      offset_int offrng[2];
 	      if (tree ret = gimple_call_return_array (stmt, offrng,
-						       &past_end, rvals))
+						       &past_end, snlim, qry))
 		{
-		  if (!compute_objsize_r (ret, ostype, pref, snlim, qry))
+		  if (!compute_objsize_r (ret, stmt, ostype, pref, snlim, qry))
 		    return false;
 
 		  /* Cap OFFRNG[1] to at most the remaining size of
@@ -2076,14 +2085,14 @@ compute_objsize_r (tree ptr, int ostype, access_ref *pref,
       if (code == ASSERT_EXPR)
 	{
 	  rhs = TREE_OPERAND (rhs, 0);
-	  return compute_objsize_r (rhs, ostype, pref, snlim, qry);
+	  return compute_objsize_r (rhs, stmt, ostype, pref, snlim, qry);
 	}
 
       if (code == POINTER_PLUS_EXPR
 	  && TREE_CODE (TREE_TYPE (rhs)) == POINTER_TYPE)
 	{
 	  /* Compute the size of the object first. */
-	  if (!compute_objsize_r (rhs, ostype, pref, snlim, qry))
+	  if (!compute_objsize_r (rhs, stmt, ostype, pref, snlim, qry))
 	    return false;
 
 	  offset_int orng[2];
@@ -2099,7 +2108,7 @@ compute_objsize_r (tree ptr, int ostype, access_ref *pref,
 
       if (code == ADDR_EXPR || code == SSA_NAME)
 	{
-	  if (!compute_objsize_r (rhs, ostype, pref, snlim, qry))
+	  if (!compute_objsize_r (rhs, stmt, ostype, pref, snlim, qry))
 	    return false;
 	  qry->put_ref (ptr, *pref);
 	  return true;
@@ -2128,18 +2137,21 @@ compute_objsize_r (tree ptr, int ostype, access_ref *pref,
    instead.  */
 
 tree
-compute_objsize (tree ptr, int ostype, access_ref *pref,
-		 range_query *rvals /* = NULL */)
+compute_objsize (tree ptr, gimple *stmt, int ostype, access_ref *pref,
+		 pointer_query *ptr_qry)
 {
   pointer_query qry;
-  qry.rvals = rvals;
+  if (ptr_qry)
+    ptr_qry->depth = 0;
+  else
+    ptr_qry = &qry;
 
   /* Clear and invalidate in case *PREF is being reused.  */
   pref->offrng[0] = pref->offrng[1] = 0;
   pref->sizrng[0] = pref->sizrng[1] = -1;
 
   ssa_name_limit_t snlim;
-  if (!compute_objsize_r (ptr, ostype, pref, snlim, &qry))
+  if (!compute_objsize_r (ptr, stmt, ostype, pref, snlim, ptr_qry))
     return NULL_TREE;
 
   offset_int maxsize = pref->size_remaining ();
@@ -2152,26 +2164,12 @@ compute_objsize (tree ptr, int ostype, access_ref *pref,
    transition to the pointer_query API.  */
 
 tree
-compute_objsize (tree ptr, int ostype, access_ref *pref, pointer_query *ptr_qry)
+compute_objsize (tree ptr, gimple *stmt, int ostype, access_ref *pref,
+		 range_query *rvals /* = NULL */)
 {
   pointer_query qry;
-  if (ptr_qry)
-    ptr_qry->depth = 0;
-  else
-    ptr_qry = &qry;
-
-  /* Clear and invalidate in case *PREF is being reused.  */
-  pref->offrng[0] = pref->offrng[1] = 0;
-  pref->sizrng[0] = pref->sizrng[1] = -1;
-
-  ssa_name_limit_t snlim;
-  if (!compute_objsize_r (ptr, ostype, pref, snlim, ptr_qry))
-    return NULL_TREE;
-
-  offset_int maxsize = pref->size_remaining ();
-  if (pref->base0 && pref->offrng[0] < 0 && pref->offrng[1] >= 0)
-    pref->offrng[0] = 0;
-  return wide_int_to_tree (sizetype, maxsize);
+  qry.rvals = rvals;
+  return compute_objsize (ptr, stmt, ostype, pref, &qry);
 }
 
 /* Legacy wrapper around the above.  The function should be removed
@@ -2184,7 +2182,7 @@ compute_objsize (tree ptr, int ostype, tree *pdecl /* = NULL */,
   /* Set the initial offsets to zero and size to negative to indicate
      none has been computed yet.  */
   access_ref ref;
-  tree size = compute_objsize (ptr, ostype, &ref, rvals);
+  tree size = compute_objsize (ptr, nullptr, ostype, &ref, rvals);
   if (!size || !ref.base0)
     return NULL_TREE;
 
@@ -2196,3 +2194,167 @@ compute_objsize (tree ptr, int ostype, tree *pdecl /* = NULL */,
 
   return size;
 }
+
+/* Determine the offset *FLDOFF of the first byte of a struct member
+   of TYPE (possibly recursively) into which the byte offset OFF points,
+   starting after the field START_AFTER if it's non-null.  On success,
+   if nonnull, set *FLDOFF to the offset of the first byte, and return
+   the field decl.  If nonnull, set *NEXTOFF to the offset of the next
+   field (which reflects any padding between the returned field and
+   the next).  Otherwise, if no such member can be found, return null.  */
+
+tree
+field_at_offset (tree type, tree start_after, HOST_WIDE_INT off,
+		 HOST_WIDE_INT *fldoff /* = nullptr */,
+		 HOST_WIDE_INT *nextoff /* = nullptr */)
+{
+  tree first_fld = TYPE_FIELDS (type);
+
+  HOST_WIDE_INT offbuf = 0, nextbuf = 0;
+  if (!fldoff)
+    fldoff = &offbuf;
+  if (!nextoff)
+    nextoff = &nextbuf;
+
+  *nextoff = 0;
+
+  /* The field to return.  */
+  tree last_fld = NULL_TREE;
+  /* The next field to advance to.  */
+  tree next_fld = NULL_TREE;
+
+  /* NEXT_FLD's cached offset.  */
+  HOST_WIDE_INT next_pos = -1;
+
+  for (tree fld = first_fld; fld; fld = next_fld)
+    {
+      next_fld = fld;
+      do
+	/* Advance to the next relevant data member.  */
+	next_fld = TREE_CHAIN (next_fld);
+      while (next_fld
+	     && (TREE_CODE (next_fld) != FIELD_DECL
+		 || DECL_ARTIFICIAL (next_fld)));
+
+      if (TREE_CODE (fld) != FIELD_DECL || DECL_ARTIFICIAL (fld))
+	continue;
+
+      if (fld == start_after)
+	continue;
+
+      tree fldtype = TREE_TYPE (fld);
+      /* The offset of FLD within its immediately enclosing structure.  */
+      HOST_WIDE_INT fldpos = next_pos < 0 ? int_byte_position (fld) : next_pos;
+
+      /* If the size is not available the field is a flexible array
+	 member.  Treat this case as success.  */
+      tree typesize = TYPE_SIZE_UNIT (fldtype);
+      HOST_WIDE_INT fldsize = (tree_fits_uhwi_p (typesize)
+			       ? tree_to_uhwi (typesize)
+			       : off);
+
+      /* If OFF is beyond the end of the current field continue.  */
+      HOST_WIDE_INT fldend = fldpos + fldsize;
+      if (fldend < off)
+	continue;
+
+      if (next_fld)
+	{
+	  /* If OFF is equal to the offset of the next field continue
+	     to it and skip the array/struct business below.  */
+	  next_pos = int_byte_position (next_fld);
+	  *nextoff = *fldoff + next_pos;
+	  if (*nextoff == off && TREE_CODE (type) != UNION_TYPE)
+	    continue;
+	}
+      else
+	*nextoff = HOST_WIDE_INT_MAX;
+
+      /* OFF refers somewhere into the current field or just past its end,
+	 which could mean it refers to the next field.  */
+      if (TREE_CODE (fldtype) == ARRAY_TYPE)
+	{
+	  /* Will be set to the offset of the first byte of the array
+	     element (which may be an array) of FLDTYPE into which
+	     OFF - FLDPOS points (which may be past ELTOFF).  */
+	  HOST_WIDE_INT eltoff = 0;
+	  if (tree ft = array_elt_at_offset (fldtype, off - fldpos, &eltoff))
+	    fldtype = ft;
+	  else
+	    continue;
+
+	  /* Advance the position to include the array element above.
+	     If OFF - FLPOS refers to a member of FLDTYPE, the member
+	     will be determined below.  */
+	  fldpos += eltoff;
+	}
+
+      *fldoff += fldpos;
+
+      if (TREE_CODE (fldtype) == RECORD_TYPE)
+	/* Drill down into the current field if it's a struct.  */
+	fld = field_at_offset (fldtype, start_after, off - fldpos,
+			       fldoff, nextoff);
+
+      last_fld = fld;
+
+      /* Unless the offset is just past the end of the field return it.
+	 Otherwise save it and return it only if the offset of the next
+	 next field is greater (i.e., there is padding between the two)
+	 or if there is no next field.  */
+      if (off < fldend)
+	break;
+    }
+
+  if (*nextoff == HOST_WIDE_INT_MAX && next_fld)
+    *nextoff = next_pos;
+
+  return last_fld;
+}
+
+/* Determine the offset *ELTOFF of the first byte of the array element
+   of array ARTYPE into which the byte offset OFF points.  On success
+   set *ELTOFF to the offset of the first byte and return type.
+   Otherwise, if no such element can be found, return null.  */
+
+tree
+array_elt_at_offset (tree artype, HOST_WIDE_INT off,
+		     HOST_WIDE_INT *eltoff /* = nullptr */,
+		     HOST_WIDE_INT *subar_size /* = nullptr */)
+{
+  gcc_assert (TREE_CODE (artype) == ARRAY_TYPE);
+
+  HOST_WIDE_INT dummy;
+  if (!eltoff)
+    eltoff = &dummy;
+  if (!subar_size)
+    subar_size = &dummy;
+
+  tree eltype = artype;
+  while (TREE_CODE (TREE_TYPE (eltype)) == ARRAY_TYPE)
+    eltype = TREE_TYPE (eltype);
+
+  tree subartype = eltype;
+  if (RECORD_OR_UNION_TYPE_P (TREE_TYPE (eltype))
+      || TYPE_MODE (TREE_TYPE (eltype)) != TYPE_MODE (char_type_node))
+    eltype = TREE_TYPE (eltype);
+
+  *subar_size = int_size_in_bytes (subartype);
+
+  if (eltype == artype)
+    {
+      *eltoff = 0;
+      return artype;
+    }
+
+  HOST_WIDE_INT artype_size = int_size_in_bytes (artype);
+  HOST_WIDE_INT eltype_size = int_size_in_bytes (eltype);
+
+  if (off < artype_size)// * eltype_size)
+    {
+      *eltoff = (off / eltype_size) * eltype_size;
+      return TREE_CODE (eltype) == ARRAY_TYPE ? TREE_TYPE (eltype) : eltype;
+    }
+
+  return NULL_TREE;
+}
diff --git a/gcc/pointer-query.h b/gcc/pointer-query.h
index 3c8172c..c8215b6 100644
--- a/gcc/pointer-query.h
+++ b/gcc/pointer-query.h
@@ -60,18 +60,16 @@ class pointer_query;
 /* Describes a reference to an object used in an access.  */
 struct access_ref
 {
-  /* Set the bounds of the reference to at most as many bytes
-     as the first argument or unknown when null, and at least
-     one when the second argument is true unless the first one
-     is a constant zero.  */
-  access_ref (tree = NULL_TREE, bool = false);
+  /* Set the bounds of the reference.  */
+  access_ref (range_query *query = nullptr, tree = NULL_TREE,
+	      gimple * = nullptr, bool = false);
 
   /* Return the PHI node REF refers to or null if it doesn't.  */
   gphi *phi () const;
 
   /* Return the object to which REF refers.  */
-  tree get_ref (vec<access_ref> *, access_ref * = NULL, int = 1,
-		ssa_name_limit_t * = NULL, pointer_query * = NULL) const;
+  tree get_ref (vec<access_ref> *, access_ref * = nullptr, int = 1,
+		ssa_name_limit_t * = nullptr, pointer_query * = nullptr) const;
 
   /* Return true if OFFRNG is the constant zero.  */
   bool offset_zero () const
@@ -85,7 +83,7 @@ struct access_ref
 
   /* Return the maximum amount of space remaining and if non-null, set
      argument to the minimum.  */
-  offset_int size_remaining (offset_int * = NULL) const;
+  offset_int size_remaining (offset_int * = nullptr) const;
 
 /* Return true if the offset and object size are in range for SIZE.  */
   bool offset_in_range (const offset_int &) const;
@@ -172,13 +170,13 @@ public:
   };
 
   /* Construct an object with the given Ranger instance and cache.  */
-  explicit pointer_query (range_query * = NULL, cache_type * = NULL);
+  explicit pointer_query (range_query * = nullptr, cache_type * = nullptr);
 
   /* Retrieve the access_ref for a variable from cache if it's there.  */
   const access_ref* get_ref (tree, int = 1) const;
 
   /* Retrieve the access_ref for a variable from cache or compute it.  */
-  bool get_ref (tree, access_ref*, int = 1);
+  bool get_ref (tree, gimple *, access_ref*, int = 1);
 
   /* Add an access_ref for the SSA_NAME to the cache.  */
   void put_ref (tree, const access_ref&, int = 1);
@@ -208,19 +206,23 @@ struct access_data
 {
   /* Set the access to at most MAXWRITE and MAXREAD bytes, and
      at least 1 when MINWRITE or MINREAD, respectively, is set.  */
-  access_data (gimple *stmt, access_mode mode,
+  access_data (range_query *query, gimple *stmt, access_mode mode,
 	       tree maxwrite = NULL_TREE, bool minwrite = false,
 	       tree maxread = NULL_TREE, bool minread = false)
     : stmt (stmt), call (),
-      dst (maxwrite, minwrite), src (maxread, minread), mode (mode) { }
+      dst (query, maxwrite, stmt, minwrite),
+      src (query, maxread, stmt, minread),
+      mode (mode) { }
 
   /* Set the access to at most MAXWRITE and MAXREAD bytes, and
      at least 1 when MINWRITE or MINREAD, respectively, is set.  */
-  access_data (tree expr, access_mode mode,
+  access_data (range_query *query, tree expr, access_mode mode,
 	       tree maxwrite = NULL_TREE, bool minwrite = false,
 	       tree maxread = NULL_TREE, bool minread = false)
     : stmt (), call (expr),
-      dst (maxwrite, minwrite), src (maxread, minread), mode (mode) { }
+      dst (query, maxwrite, nullptr, minwrite),
+      src (query, maxread, nullptr, minread),
+      mode (mode) { }
 
   /* Access statement.  */
   gimple *stmt;
@@ -245,14 +247,32 @@ extern bool get_size_range (tree, tree[2], int = 0);
 extern bool get_size_range (range_query *, tree, gimple *, tree[2], int = 0);
 
 class range_query;
-extern tree gimple_call_alloc_size (gimple *, wide_int[2] = NULL,
-				    range_query * = NULL);
-extern tree gimple_parm_array_size (tree, wide_int[2], bool * = NULL);
+extern tree gimple_call_alloc_size (gimple *, wide_int[2] = nullptr,
+				    range_query * = nullptr);
+
+/* Compute the size of an object referenced by the first argument in
+   a statement given by second argument, using Object Size Type given
+   by third argument.  Store result in an access_ref.  */
+extern tree compute_objsize (tree, gimple *, int, access_ref *,
+			     range_query * = nullptr);
+extern tree compute_objsize (tree, gimple *, int, access_ref *,
+			     pointer_query *);
+inline tree compute_objsize (tree ptr, int ostype, access_ref *pref)
+{
+  return compute_objsize (ptr, nullptr, ostype, pref, (range_query *)nullptr);
+}
 
-extern tree compute_objsize (tree, int, access_ref *, range_query * = NULL);
 /* Legacy/transitional API.  Should not be used in new code.  */
-extern tree compute_objsize (tree, int, access_ref *, pointer_query *);
-extern tree compute_objsize (tree, int, tree * = NULL, tree * = NULL,
-			     range_query * = NULL);
+extern tree compute_objsize (tree, int, tree * = nullptr, tree * = nullptr,
+			     range_query * = nullptr);
+
+/* Return the field at the constant offset.  */
+extern tree field_at_offset (tree, tree, HOST_WIDE_INT,
+			     HOST_WIDE_INT * = nullptr,
+			     HOST_WIDE_INT * = nullptr);
+/* Return the array at the constant offset.  */
+extern tree array_elt_at_offset (tree, HOST_WIDE_INT,
+				 HOST_WIDE_INT * = nullptr,
+				 HOST_WIDE_INT * = nullptr);
 
 #endif   // GCC_POINTER_QUERY_H
diff --git a/gcc/print-rtl.c b/gcc/print-rtl.c
index 567d287..a1afff6 100644
--- a/gcc/print-rtl.c
+++ b/gcc/print-rtl.c
@@ -84,10 +84,12 @@ int flag_dump_unnumbered_links = 0;
 /* Constructor for rtx_writer.  */
 
 rtx_writer::rtx_writer (FILE *outf, int ind, bool simple, bool compact,
-			rtx_reuse_manager *reuse_manager)
+			rtx_reuse_manager *reuse_manager ATTRIBUTE_UNUSED)
 : m_outfile (outf), m_sawclose (0), m_indent (ind),
-  m_in_call_function_usage (false), m_simple (simple), m_compact (compact),
-  m_rtx_reuse_manager (reuse_manager)
+  m_in_call_function_usage (false), m_simple (simple), m_compact (compact)
+#ifndef GENERATOR_FILE
+  , m_rtx_reuse_manager (reuse_manager)
+#endif
 {
 }
 
diff --git a/gcc/print-rtl.h b/gcc/print-rtl.h
index 66f8c36..d8d9e2c 100644
--- a/gcc/print-rtl.h
+++ b/gcc/print-rtl.h
@@ -68,8 +68,10 @@ class rtx_writer
      - insn names are prefixed with "c" (e.g. "cinsn", "cnote", etc).  */
   bool m_compact;
 
+#ifndef GENERATOR_FILE
   /* An optional instance of rtx_reuse_manager.  */
   rtx_reuse_manager *m_rtx_reuse_manager;
+#endif
 };
 
 #ifdef BUFSIZ
diff --git a/gcc/profile.c b/gcc/profile.c
index c33c833..d07002d 100644
--- a/gcc/profile.c
+++ b/gcc/profile.c
@@ -1375,7 +1375,7 @@ branch_prob (bool thunk)
       /* Initialize the output.  */
       output_location (&streamed_locations, NULL, 0, NULL, NULL);
 
-      hash_set<int_hash <location_t, 0, 2> > seen_locations;
+      hash_set<location_hash> seen_locations;
 
       FOR_EACH_BB_FN (bb, cfun)
 	{
@@ -1385,6 +1385,7 @@ branch_prob (bool thunk)
 	  if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb)
 	    {
 	      location_t loc = DECL_SOURCE_LOCATION (current_function_decl);
+	      gcc_checking_assert (!RESERVED_LOCATION_P (loc));
 	      seen_locations.add (loc);
 	      expanded_location curr_location = expand_location (loc);
 	      output_location (&streamed_locations, curr_location.file,
diff --git a/gcc/read-rtl-function.c b/gcc/read-rtl-function.c
index 941d1e1..5699f57 100644
--- a/gcc/read-rtl-function.c
+++ b/gcc/read-rtl-function.c
@@ -1498,6 +1498,9 @@ function_reader::consolidate_singletons (rtx x)
     case CONST_INT:
       return gen_rtx_CONST_INT (GET_MODE (x), INTVAL (x));
 
+    case CONST_VECTOR:
+      return gen_rtx_CONST_VECTOR (GET_MODE (x), XVEC (x, 0));
+
     default:
       break;
     }
diff --git a/gcc/read-rtl.c b/gcc/read-rtl.c
index 0411666..b33dee0 100644
--- a/gcc/read-rtl.c
+++ b/gcc/read-rtl.c
@@ -36,12 +36,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "read-md.h"
 #include "gensupport.h"
 
-#ifndef GENERATOR_FILE
-#include "function.h"
-#include "memmodel.h"
-#include "emit-rtl.h"
-#endif
-
 /* One element in a singly-linked list of (integer, string) pairs.  */
 struct map_value {
   struct map_value *next;
diff --git a/gcc/real.h b/gcc/real.h
index 015163d..39dd34e 100644
--- a/gcc/real.h
+++ b/gcc/real.h
@@ -178,13 +178,12 @@ struct real_format
    decimal float modes indexed by (MODE - first decimal float mode) +
    the number of float modes.  */
 extern const struct real_format *
-  real_format_for_mode[MAX_MODE_FLOAT - MIN_MODE_FLOAT + 1
-		       + MAX_MODE_DECIMAL_FLOAT - MIN_MODE_DECIMAL_FLOAT + 1];
+  real_format_for_mode[NUM_MODE_FLOAT + NUM_MODE_DECIMAL_FLOAT];
 
 #define REAL_MODE_FORMAT(MODE)						\
   (real_format_for_mode[DECIMAL_FLOAT_MODE_P (MODE)			\
 			? (((MODE) - MIN_MODE_DECIMAL_FLOAT)		\
-			   + (MAX_MODE_FLOAT - MIN_MODE_FLOAT + 1))	\
+			   + NUM_MODE_FLOAT)				\
 			: GET_MODE_CLASS (MODE) == MODE_FLOAT		\
 			? ((MODE) - MIN_MODE_FLOAT)			\
 			: (gcc_unreachable (), 0)])
diff --git a/gcc/rtl.h b/gcc/rtl.h
index 5473cc9..6a6de1c 100644
--- a/gcc/rtl.h
+++ b/gcc/rtl.h
@@ -3425,6 +3425,7 @@ public:
 			    rtx, rtx, rtx);
   rtx simplify_gen_relational (rtx_code, machine_mode, machine_mode, rtx, rtx);
   rtx simplify_gen_subreg (machine_mode, rtx, machine_mode, poly_uint64);
+  rtx simplify_gen_vec_select (rtx, unsigned int);
 
   /* Tracks the level of MEM nesting for the value being simplified:
      0 means the value is not in a MEM, >0 means it is.  This is needed
@@ -3527,6 +3528,12 @@ simplify_gen_subreg (machine_mode outermode, rtx op, machine_mode innermode,
 }
 
 inline rtx
+simplify_gen_vec_select (rtx op, unsigned int index)
+{
+  return simplify_context ().simplify_gen_vec_select (op, index);
+}
+
+inline rtx
 lowpart_subreg (machine_mode outermode, rtx op, machine_mode innermode)
 {
   return simplify_context ().lowpart_subreg (outermode, op, innermode);
diff --git a/gcc/sbitmap.h b/gcc/sbitmap.h
index 17660e5..5fb0e00 100644
--- a/gcc/sbitmap.h
+++ b/gcc/sbitmap.h
@@ -114,7 +114,7 @@ bitmap_check_sizes (const_sbitmap a, const_sbitmap b)
 }
 
 /* Test if bit number bitno in the bitmap is set.  */
-static inline SBITMAP_ELT_TYPE
+static inline bool
 bitmap_bit_p (const_sbitmap map, int bitno)
 {
   bitmap_check_index (map, bitno);
@@ -124,26 +124,36 @@ bitmap_bit_p (const_sbitmap map, int bitno)
   return (map->elms[i] >> s) & (SBITMAP_ELT_TYPE) 1;
 }
 
-/* Set bit number BITNO in the sbitmap MAP.  */
+/* Set bit number BITNO in the sbitmap MAP.
+   Return true if the bit changed.  */
 
-static inline void
+static inline bool
 bitmap_set_bit (sbitmap map, int bitno)
 {
   bitmap_check_index (map, bitno);
 
-  map->elms[bitno / SBITMAP_ELT_BITS]
-    |= (SBITMAP_ELT_TYPE) 1 << (bitno) % SBITMAP_ELT_BITS;
+  size_t i = bitno / SBITMAP_ELT_BITS;
+  unsigned int s = bitno % SBITMAP_ELT_BITS;
+  if (map->elms[i] & ((SBITMAP_ELT_TYPE) 1 << s))
+    return false;
+  map->elms[i] |= (SBITMAP_ELT_TYPE) 1 << s;
+  return true;
 }
 
-/* Reset bit number BITNO in the sbitmap MAP.  */
+/* Reset bit number BITNO in the sbitmap MAP.
+   Return true if the bit changed.  */
 
-static inline void
+static inline bool
 bitmap_clear_bit (sbitmap map, int bitno)
 {
   bitmap_check_index (map, bitno);
 
-  map->elms[bitno / SBITMAP_ELT_BITS]
-    &= ~((SBITMAP_ELT_TYPE) 1 << (bitno) % SBITMAP_ELT_BITS);
+  size_t i = bitno / SBITMAP_ELT_BITS;
+  unsigned int s = bitno % SBITMAP_ELT_BITS;
+  if (!(map->elms[i] & ((SBITMAP_ELT_TYPE) 1 << s)))
+    return false;
+  map->elms[i] &= ~((SBITMAP_ELT_TYPE) 1 << s);
+  return true;
 }
 
 /* The iterator for sbitmap.  */
diff --git a/gcc/selftest.c b/gcc/selftest.c
index 8f1cde0..13ec4f0 100644
--- a/gcc/selftest.c
+++ b/gcc/selftest.c
@@ -89,7 +89,7 @@ assert_streq (const location &loc,
 	if (strcmp (val1, val2) == 0)
 	  pass (loc, "ASSERT_STREQ");
 	else
-	  fail_formatted (loc, "ASSERT_STREQ (%s, %s) val1=\"%s\" val2=\"%s\"",
+	  fail_formatted (loc, "ASSERT_STREQ (%s, %s)\n val1=\"%s\"\n val2=\"%s\"\n",
 			  desc_val1, desc_val2, val1, val2);
       }
 }
@@ -192,6 +192,21 @@ temp_source_file::temp_source_file (const location &loc,
   fclose (out);
 }
 
+/* As above, but with a size, to allow for NUL bytes in CONTENT.  */
+
+temp_source_file::temp_source_file (const location &loc,
+				    const char *suffix,
+				    const char *content,
+				    size_t sz)
+: named_temp_file (suffix)
+{
+  FILE *out = fopen (get_filename (), "w");
+  if (!out)
+    fail_formatted (loc, "unable to open tempfile: %s", get_filename ());
+  fwrite (content, sz, 1, out);
+  fclose (out);
+}
+
 /* Avoid introducing locale-specific differences in the results
    by hardcoding open_quote and close_quote.  */
 
diff --git a/gcc/selftest.h b/gcc/selftest.h
index 58d8d38..24ef57cb 100644
--- a/gcc/selftest.h
+++ b/gcc/selftest.h
@@ -112,6 +112,8 @@ class temp_source_file : public named_temp_file
  public:
   temp_source_file (const location &loc, const char *suffix,
 		    const char *content);
+  temp_source_file (const location &loc, const char *suffix,
+		    const char *content, size_t sz);
 };
 
 /* RAII-style class for avoiding introducing locale-specific differences
diff --git a/gcc/simplify-rtx.c b/gcc/simplify-rtx.c
index 2bb18fb..eea7476 100644
--- a/gcc/simplify-rtx.c
+++ b/gcc/simplify-rtx.c
@@ -899,10 +899,12 @@ simplify_context::simplify_unary_operation (rtx_code code, machine_mode mode,
 static bool
 exact_int_to_float_conversion_p (const_rtx op)
 {
-  int out_bits = significand_size (GET_MODE_INNER (GET_MODE (op)));
   machine_mode op0_mode = GET_MODE (XEXP (op, 0));
-  /* Constants shouldn't reach here.  */
-  gcc_assert (op0_mode != VOIDmode);
+  /* Constants can reach here with -frounding-math, if they do then
+     the conversion isn't exact.  */
+  if (op0_mode == VOIDmode)
+    return false;
+  int out_bits = significand_size (GET_MODE_INNER (GET_MODE (op)));
   int in_prec = GET_MODE_UNIT_PRECISION (op0_mode);
   int in_bits = in_prec;
   if (HWI_COMPUTABLE_MODE_P (op0_mode))
@@ -1917,6 +1919,19 @@ simplify_const_unary_operation (enum rtx_code code, machine_mode mode,
         return 0;
 
       d = real_value_truncate (mode, d);
+
+      /* Avoid the folding if flag_rounding_math is on and the
+	 conversion is not exact.  */
+      if (HONOR_SIGN_DEPENDENT_ROUNDING (mode))
+	{
+	  bool fail = false;
+	  wide_int w = real_to_integer (&d, &fail,
+					GET_MODE_PRECISION
+					  (as_a <scalar_int_mode> (op_mode)));
+	  if (fail || wi::ne_p (w, wide_int (rtx_mode_t (op, op_mode))))
+	    return 0;
+	}
+
       return const_double_from_real_value (d, mode);
     }
   else if (code == UNSIGNED_FLOAT && CONST_SCALAR_INT_P (op))
@@ -1941,6 +1956,19 @@ simplify_const_unary_operation (enum rtx_code code, machine_mode mode,
         return 0;
 
       d = real_value_truncate (mode, d);
+
+      /* Avoid the folding if flag_rounding_math is on and the
+	 conversion is not exact.  */
+      if (HONOR_SIGN_DEPENDENT_ROUNDING (mode))
+	{
+	  bool fail = false;
+	  wide_int w = real_to_integer (&d, &fail,
+					GET_MODE_PRECISION
+					  (as_a <scalar_int_mode> (op_mode)));
+	  if (fail || wi::ne_p (w, wide_int (rtx_mode_t (op, op_mode))))
+	    return 0;
+	}
+
       return const_double_from_real_value (d, mode);
     }
 
@@ -2068,6 +2096,11 @@ simplify_const_unary_operation (enum rtx_code code, machine_mode mode,
 	     and the operand is a signaling NaN.  */
 	  if (HONOR_SNANS (mode) && REAL_VALUE_ISSIGNALING_NAN (d))
 	    return NULL_RTX;
+	  /* Or if flag_rounding_math is on and the truncation is not
+	     exact.  */
+	  if (HONOR_SIGN_DEPENDENT_ROUNDING (mode)
+	      && !exact_real_truncate (mode, &d))
+	    return NULL_RTX;
 	  d = real_value_truncate (mode, d);
 	  break;
 	case FLOAT_EXTEND:
@@ -4064,9 +4097,25 @@ simplify_context::simplify_binary_operation_1 (rtx_code code,
 	}
       break;
 
-    case ASHIFT:
     case SS_ASHIFT:
+      if (CONST_INT_P (trueop0)
+	  && HWI_COMPUTABLE_MODE_P (mode)
+	  && (UINTVAL (trueop0) == (GET_MODE_MASK (mode) >> 1)
+	      || mode_signbit_p (mode, trueop0))
+	  && ! side_effects_p (op1))
+	return op0;
+      goto simplify_ashift;
+
     case US_ASHIFT:
+      if (CONST_INT_P (trueop0)
+	  && HWI_COMPUTABLE_MODE_P (mode)
+	  && UINTVAL (trueop0) == GET_MODE_MASK (mode)
+	  && ! side_effects_p (op1))
+	return op0;
+      /* FALLTHRU */
+
+    case ASHIFT:
+simplify_ashift:
       if (trueop1 == CONST0_RTX (mode))
 	return op0;
       if (trueop0 == CONST0_RTX (mode) && ! side_effects_p (op1))
@@ -5004,6 +5053,8 @@ simplify_const_binary_operation (enum rtx_code code, machine_mode mode,
 	case LSHIFTRT:
 	case ASHIFTRT:
 	case ASHIFT:
+	case SS_ASHIFT:
+	case US_ASHIFT:
 	  {
 	    wide_int wop1 = pop1;
 	    if (SHIFT_COUNT_TRUNCATED)
@@ -5025,6 +5076,24 @@ simplify_const_binary_operation (enum rtx_code code, machine_mode mode,
 		result = wi::lshift (pop0, wop1);
 		break;
 
+	      case SS_ASHIFT:
+		if (wi::leu_p (wop1, wi::clrsb (pop0)))
+		  result = wi::lshift (pop0, wop1);
+		else if (wi::neg_p (pop0))
+		  result = wi::min_value (int_mode, SIGNED);
+		else
+		  result = wi::max_value (int_mode, SIGNED);
+		break;
+
+	      case US_ASHIFT:
+		if (wi::eq_p (pop0, 0))
+		  result = pop0;
+		else if (wi::leu_p (wop1, wi::clz (pop0)))
+		  result = wi::lshift (pop0, wop1);
+		else
+		  result = wi::max_value (int_mode, UNSIGNED);
+		break;
+
 	      default:
 		gcc_unreachable ();
 	      }
@@ -7553,6 +7622,28 @@ simplify_context::lowpart_subreg (machine_mode outer_mode, rtx expr,
 			      subreg_lowpart_offset (outer_mode, inner_mode));
 }
 
+/* Generate RTX to select element at INDEX out of vector OP.  */
+
+rtx
+simplify_context::simplify_gen_vec_select (rtx op, unsigned int index)
+{
+  gcc_assert (VECTOR_MODE_P (GET_MODE (op)));
+
+  scalar_mode imode = GET_MODE_INNER (GET_MODE (op));
+
+  if (known_eq (index * GET_MODE_SIZE (imode),
+		subreg_lowpart_offset (imode, GET_MODE (op))))
+    {
+      rtx res = lowpart_subreg (imode, op, GET_MODE (op));
+      if (res)
+	return res;
+    }
+
+  rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (index)));
+  return gen_rtx_VEC_SELECT (imode, op, tmp);
+}
+
+
 /* Simplify X, an rtx expression.
 
    Return the simplified expression or NULL if no simplifications
diff --git a/gcc/system.h b/gcc/system.h
index adde3e2..4ac656c 100644
--- a/gcc/system.h
+++ b/gcc/system.h
@@ -1305,4 +1305,17 @@ startswith (const char *str, const char *prefix)
   return strncmp (str, prefix, strlen (prefix)) == 0;
 }
 
+/* Return true if STR string ends with SUFFIX.  */
+
+static inline bool
+endswith (const char *str, const char *suffix)
+{
+  size_t str_len = strlen (str);
+  size_t suffix_len = strlen (suffix);
+  if (str_len < suffix_len)
+    return false;
+
+  return memcmp (str + str_len - suffix_len, suffix, suffix_len) == 0;
+}
+
 #endif /* ! GCC_SYSTEM_H */
diff --git a/gcc/target.def b/gcc/target.def
index 927ebfc..6bc39d2 100644
--- a/gcc/target.def
+++ b/gcc/target.def
@@ -2037,7 +2037,7 @@ stores.",
 
 /* Target function to initialize the cost model for a loop or block.  */
 DEFHOOK
-(init_cost,
+(create_costs,
  "This hook should initialize target-specific data structures in preparation\n\
 for modeling the costs of vectorizing a loop or basic block.  The default\n\
 allocates three unsigned integers for accumulating costs for the prologue,\n\
@@ -2046,50 +2046,9 @@ non-NULL, it identifies the loop being vectorized; otherwise a single block\n\
 is being vectorized.  If @var{costing_for_scalar} is true, it indicates the\n\
 current cost model is for the scalar version of a loop or block; otherwise\n\
 it is for the vector version.",
- void *,
- (class loop *loop_info, bool costing_for_scalar),
- default_init_cost)
-
-/* Target function to record N statements of the given kind using the
-   given vector type within the cost model data for the current loop or
-    block.  */
-DEFHOOK
-(add_stmt_cost,
- "This hook should update the target-specific @var{data} in response to\n\
-adding @var{count} copies of the given @var{kind} of statement to a\n\
-loop or basic block.  The default adds the builtin vectorizer cost for\n\
-the copies of the statement to the accumulator specified by @var{where},\n\
-(the prologue, body, or epilogue) and returns the amount added.  The\n\
-return value should be viewed as a tentative cost that may later be\n\
-revised.",
- unsigned,
- (class vec_info *, void *data, int count, enum vect_cost_for_stmt kind,
-  class _stmt_vec_info *stmt_info, tree vectype, int misalign,
-  enum vect_cost_model_location where),
- default_add_stmt_cost)
-
-/* Target function to calculate the total cost of the current vectorized
-   loop or block.  */
-DEFHOOK
-(finish_cost,
- "This hook should complete calculations of the cost of vectorizing a loop\n\
-or basic block based on @var{data}, and return the prologue, body, and\n\
-epilogue costs as unsigned integers.  The default returns the value of\n\
-the three accumulators.",
- void,
- (void *data, unsigned *prologue_cost, unsigned *body_cost,
-  unsigned *epilogue_cost),
- default_finish_cost)
-
-/* Function to delete target-specific cost modeling data.  */
-DEFHOOK
-(destroy_cost_data,
- "This hook should release @var{data} and any related data structures\n\
-allocated by TARGET_VECTORIZE_INIT_COST.  The default releases the\n\
-accumulator.",
- void,
- (void *data),
- default_destroy_cost_data)
+ class vector_costs *,
+ (vec_info *vinfo, bool costing_for_scalar),
+ default_vectorize_create_costs)
 
 HOOK_VECTOR_END (vectorize)
 
@@ -6805,6 +6764,16 @@ collecting constructors and destructors to be run at startup and exit.\n\
 It is false if we must use @command{collect2}.",
  bool, false)
 
+/* True if the target wants DTORs to be run from cxa_atexit.  */
+DEFHOOKPOD
+(dtors_from_cxa_atexit,
+ "This value is true if the target wants destructors to be queued to be\n\
+run from __cxa_atexit.  If this is the case then, for each priority level,\n\
+a new constructor will be entered that registers the destructors for that\n\
+level with __cxa_atexit (and there will be no destructors emitted).\n\
+It is false the method implied by @code{have_ctors_dtors} is used.",
+ bool, false)
+
 /* True if thread-local storage is supported.  */
 DEFHOOKPOD
 (have_tls,
diff --git a/gcc/targhooks.c b/gcc/targhooks.c
index 0b525bb..6f071f8 100644
--- a/gcc/targhooks.c
+++ b/gcc/targhooks.c
@@ -1474,65 +1474,10 @@ default_empty_mask_is_expensive (unsigned ifn)
    loop body, and epilogue) for a vectorized loop or block.  So allocate an
    array of three unsigned ints, set it to zero, and return its address.  */
 
-void *
-default_init_cost (class loop *loop_info ATTRIBUTE_UNUSED,
-		   bool costing_for_scalar ATTRIBUTE_UNUSED)
-{
-  unsigned *cost = XNEWVEC (unsigned, 3);
-  cost[vect_prologue] = cost[vect_body] = cost[vect_epilogue] = 0;
-  return cost;
-}
-
-/* By default, the cost model looks up the cost of the given statement
-   kind and mode, multiplies it by the occurrence count, accumulates
-   it into the cost specified by WHERE, and returns the cost added.  */
-
-unsigned
-default_add_stmt_cost (class vec_info *vinfo, void *data, int count,
-		       enum vect_cost_for_stmt kind,
-		       class _stmt_vec_info *stmt_info, tree vectype,
-		       int misalign,
-		       enum vect_cost_model_location where)
-{
-  unsigned *cost = (unsigned *) data;
-  unsigned retval = 0;
-  int stmt_cost = targetm.vectorize.builtin_vectorization_cost (kind, vectype,
-								misalign);
-   /* Statements in an inner loop relative to the loop being
-      vectorized are weighted more heavily.  The value here is
-      arbitrary and could potentially be improved with analysis.  */
-  if (where == vect_body && stmt_info
-      && stmt_in_inner_loop_p (vinfo, stmt_info))
-    {
-      loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
-      gcc_assert (loop_vinfo);
-      count *= LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo);
-    }
-
-  retval = (unsigned) (count * stmt_cost);
-  cost[where] += retval;
-
-  return retval;
-}
-
-/* By default, the cost model just returns the accumulated costs.  */
-
-void
-default_finish_cost (void *data, unsigned *prologue_cost,
-		     unsigned *body_cost, unsigned *epilogue_cost)
-{
-  unsigned *cost = (unsigned *) data;
-  *prologue_cost = cost[vect_prologue];
-  *body_cost     = cost[vect_body];
-  *epilogue_cost = cost[vect_epilogue];
-}
-
-/* Free the cost data.  */
-
-void
-default_destroy_cost_data (void *data)
+vector_costs *
+default_vectorize_create_costs (vec_info *vinfo, bool costing_for_scalar)
 {
-  free (data);
+  return new vector_costs (vinfo, costing_for_scalar);
 }
 
 /* Determine whether or not a pointer mode is valid. Assume defaults
diff --git a/gcc/targhooks.h b/gcc/targhooks.h
index 7f7f25a..11e9d7d 100644
--- a/gcc/targhooks.h
+++ b/gcc/targhooks.h
@@ -118,13 +118,7 @@ extern opt_machine_mode default_vectorize_related_mode (machine_mode,
 							poly_uint64);
 extern opt_machine_mode default_get_mask_mode (machine_mode);
 extern bool default_empty_mask_is_expensive (unsigned);
-extern void *default_init_cost (class loop *, bool);
-extern unsigned default_add_stmt_cost (class vec_info *, void *, int,
-				       enum vect_cost_for_stmt,
-				       class _stmt_vec_info *, tree, int,
-				       enum vect_cost_model_location);
-extern void default_finish_cost (void *, unsigned *, unsigned *, unsigned *);
-extern void default_destroy_cost_data (void *);
+extern vector_costs *default_vectorize_create_costs (vec_info *, bool);
 
 /* OpenACC hooks.  */
 extern bool default_goacc_validate_dims (tree, int [], int, unsigned);
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 602b727..b221cba 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,1810 @@
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* lib/g++-dg.exp: Handle "impcx".
+	* lib/target-supports.exp
+	(check_effective_target_implicit_constexpr): New.
+	* g++.dg/abi/abi-tag16.C:
+	* g++.dg/abi/abi-tag18a.C:
+	* g++.dg/abi/guard4.C:
+	* g++.dg/abi/lambda-defarg1.C:
+	* g++.dg/abi/mangle26.C:
+	* g++.dg/cpp0x/constexpr-diag3.C:
+	* g++.dg/cpp0x/constexpr-ex1.C:
+	* g++.dg/cpp0x/constexpr-ice5.C:
+	* g++.dg/cpp0x/constexpr-incomplete2.C:
+	* g++.dg/cpp0x/constexpr-memfn1.C:
+	* g++.dg/cpp0x/constexpr-neg3.C:
+	* g++.dg/cpp0x/constexpr-specialization.C:
+	* g++.dg/cpp0x/inh-ctor19.C:
+	* g++.dg/cpp0x/inh-ctor30.C:
+	* g++.dg/cpp0x/lambda/lambda-mangle3.C:
+	* g++.dg/cpp0x/lambda/lambda-mangle5.C:
+	* g++.dg/cpp1y/auto-fn12.C:
+	* g++.dg/cpp1y/constexpr-loop5.C:
+	* g++.dg/cpp1z/constexpr-lambda7.C:
+	* g++.dg/cpp2a/constexpr-dtor3.C:
+	* g++.dg/cpp2a/constexpr-new13.C:
+	* g++.dg/cpp2a/constinit11.C:
+	* g++.dg/cpp2a/constinit12.C:
+	* g++.dg/cpp2a/constinit14.C:
+	* g++.dg/cpp2a/constinit15.C:
+	* g++.dg/cpp2a/spaceship-constexpr1.C:
+	* g++.dg/cpp2a/spaceship-eq3.C:
+	* g++.dg/cpp2a/udlit-class-nttp-neg2.C:
+	* g++.dg/debug/dwarf2/auto1.C:
+	* g++.dg/debug/dwarf2/cdtor-1.C:
+	* g++.dg/debug/dwarf2/lambda1.C:
+	* g++.dg/debug/dwarf2/pr54508.C:
+	* g++.dg/debug/dwarf2/pubnames-2.C:
+	* g++.dg/debug/dwarf2/pubnames-3.C:
+	* g++.dg/ext/is_literal_type3.C:
+	* g++.dg/ext/visibility/template7.C:
+	* g++.dg/gcov/gcov-12.C:
+	* g++.dg/gcov/gcov-2.C:
+	* g++.dg/ipa/devirt-35.C:
+	* g++.dg/ipa/devirt-36.C:
+	* g++.dg/ipa/devirt-37.C:
+	* g++.dg/ipa/devirt-44.C:
+	* g++.dg/ipa/imm-devirt-1.C:
+	* g++.dg/lookup/builtin5.C:
+	* g++.dg/lto/inline-crossmodule-1_0.C:
+	* g++.dg/modules/enum-1_a.C:
+	* g++.dg/modules/fn-inline-1_c.C:
+	* g++.dg/modules/pmf-1_b.C:
+	* g++.dg/modules/used-1_c.C:
+	* g++.dg/tls/thread_local11.C:
+	* g++.dg/tls/thread_local11a.C:
+	* g++.dg/tm/pr46653.C:
+	* g++.dg/ubsan/pr70035.C:
+	* g++.old-deja/g++.other/delete6.C:
+	* g++.dg/modules/pmf-1_a.H:
+	Adjust for implicit constexpr.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* g++.dg/ext/flexary37.C: Remove expected error.
+
+2021-11-15  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	* gcc.dg/fold-stringops-2.c: Define size_t.
+	(safe1): Adjust.
+	(safe4): New test.
+	* gcc.dg/fold-stringops-3.c: New test.
+
+2021-11-15  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	* gcc.dg/Wobjsize-1.c: Make warning change line agnostic.
+	* gcc.dg/fold-stringops-2.c: New test.
+
+2021-11-15  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	* gcc.dg/fold-stringops-1.c: New test.
+
+2021-11-15  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR middle-end/103184
+	* gcc.dg/pr103184-1.c: New test.
+	* gcc.dg/pr103184-2.c: Likewise.
+
+2021-11-15  Iain Sandoe  <iain@sandoe.co.uk>
+
+	* gcc.dg/vect/tsvc/tsvc.h: Use malloc for Darwin 9 and
+	earlier.
+
+2021-11-15  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gcc.dg/pr103207.c: New test.
+
+2021-11-15  Jakub Jelinek  <jakub@redhat.com>
+
+	PR debug/100469
+	* g++.dg/opt/pr100469.C: New test.
+
+2021-11-15  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/103205
+	* gcc.target/i386/pr103205-2.c: New test.
+
+2021-11-15  Aldy Hernandez  <aldyh@redhat.com>
+
+	* g++.dg/tree-ssa/pr31146-2.C: Add -fno-thread-jumps.
+
+2021-11-15  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/103237
+	* gcc.dg/torture/pr103237.c: New testcase.
+
+2021-11-15  Hongyu Wang  <hongyu.wang@intel.com>
+
+	PR target/103069
+	* gcc.target/i386/pr103069-1.c: New test.
+	* gcc.target/i386/pr103069-2.c: Ditto.
+
+2021-11-15  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/103219
+	* gcc.dg/torture/pr103219.c: New testcase.
+
+2021-11-15  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* gcc.target/i386/pr101346.c: Require dfp support.
+
+2021-11-15  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/103205
+	* gcc.target/i386/pr103205.c: New test.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* g++.dg/cpp0x/lambda/lambda-switch.C: Adjust expected location.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* g++.dg/cpp1y/constexpr-89285-2.C: Expect error.
+	* g++.dg/cpp1y/constexpr-89285.C: Adjust error.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	PR c++/70690
+	* g++.dg/init/array41a.C: New test.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* g++.dg/cpp2a/constexpr-virtual20.C: New test.
+
+2021-11-15  Hans-Peter Nilsson  <hp@axis.com>
+
+	* gcc.dg/uninit-pred-9_b.c: Correct last adjustment, for CRIS.
+
+2021-11-15  Andrew Pinski  <apinski@marvell.com>
+
+	* gcc.dg/tree-ssa/pr100278.c: Move to ...
+	* gcc.c-torture/compile/pr100278.c: Here.
+	Remove dg-do and dg-options.
+	* gcc.dg/tree-ssa/pr101189.c: Move to ...
+	* gcc.c-torture/compile/pr101189.c: Here.
+	Remove dg-do and dg-options.
+	* gcc.dg/tree-ssa/pr100453.c: Move to ...
+	* gcc.c-torture/execute/pr100453.c: Here.
+	Remove dg-do and dg-options.
+	* gcc.dg/tree-ssa/pr101335.c: Move to ...
+	* gcc.c-torture/execute/pr101335.c: Here
+	Remove dg-do and dg-options.
+
+2021-11-14  Maciej W. Rozycki  <macro@embecosm.com>
+
+	* gcc.target/vax/setmem.c: New test.
+
+2021-11-14  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gcc.dg/tree-ssa/modref-dse-3.c: New test.
+
+2021-11-14  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gcc.dg/pr103229.c: Removed.
+
+2021-11-14  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gcc.dg/pr103229.c: New test.
+
+2021-11-14  Martin Liska  <mliska@suse.cz>
+
+	* c-c++-common/tsan/free_race.c: Remove unnecessary -ldl.
+	* c-c++-common/tsan/free_race2.c: Likewise.
+
+2021-11-14  Jan Hubicka  <hubicka@ucw.cz>
+
+	* c-c++-common/asan/null-deref-1.c: Update template.
+	* c-c++-common/tsan/free_race.c: Update template.
+	* c-c++-common/tsan/free_race2.c: Update template.
+	* gcc.dg/ipa/ipa-sra-4.c: Update template.
+
+2021-11-13  Jan Hubicka  <hubicka@ucw.cz>
+
+	* g++.dg/cpp1z/inh-ctor23.C: Fix template
+	* g++.dg/ipa/ipa-icf-4.C: Fix template
+	* gcc.dg/tree-ssa/modref-dse-1.c: New test.
+	* gcc.dg/tree-ssa/modref-dse-2.c: New test.
+
+2021-11-13  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/analyzer/pr93382.c: Tweak expected wording.
+	* gcc.dg/analyzer/taint-alloc-1.c: New test.
+	* gcc.dg/analyzer/taint-alloc-2.c: New test.
+	* gcc.dg/analyzer/taint-divisor-1.c: New test.
+	* gcc.dg/analyzer/taint-1.c: Rename to...
+	* gcc.dg/analyzer/taint-read-index-1.c: ...this.  Tweak expected
+	wording.  Mark some events as xfail.
+	* gcc.dg/analyzer/taint-read-offset-1.c: New test.
+	* gcc.dg/analyzer/taint-size-1.c: New test.
+	* gcc.dg/analyzer/taint-write-index-1.c: New test.
+	* gcc.dg/analyzer/taint-write-offset-1.c: New test.
+
+2021-11-13  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gcc.dg/pr103222.c: New test.
+
+2021-11-12  Jan Hubicka  <jh@suse.cz>
+
+	PR tree-optimization/103209
+	* gcc.dg/tree-ssa/pta-callused.c: Update template.
+	* gcc.c-torture/execute/pr103209.c: New test.
+
+2021-11-12  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/stp_1.c: New test.
+
+2021-11-12  Tobias Burnus  <tobias@codesourcery.com>
+
+	* gfortran.dg/gomp/unexpected-end.f90: Update dg-error.
+	* gfortran.dg/gomp/clauses-1.f90: New test.
+	* gfortran.dg/gomp/nowait-2.f90: New test.
+	* gfortran.dg/gomp/nowait-3.f90: New test.
+
+2021-11-12  Martin Liska  <mliska@suse.cz>
+
+	PR testsuite/103051
+	* gcc.dg/vect/tsvc/vect-tsvc-s112.c: Skip test for old Power
+	CPUs.
+
+2021-11-12  Chung-Lin Tang  <cltang@codesourcery.com>
+
+	* c-c++-common/gomp/target-implicit-map-1.c: New test.
+	* c-c++-common/goacc/combined-reduction.c: Adjust scan test pattern.
+	* c-c++-common/goacc/firstprivate-mappings-1.c: Likewise.
+	* c-c++-common/goacc/mdc-1.c: Likewise.
+	* g++.dg/goacc/firstprivate-mappings-1.C: Likewise.
+
+2021-11-12  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/97896
+	* gfortran.dg/index_5.f90: New.
+
+2021-11-12  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/103204
+	* gcc.dg/torture/pr103204.c: New testcase.
+
+2021-11-11  Bill Schmidt  <wschmidt@linux.ibm.com>
+
+	* gcc.target/powerpc/test_mffsl.c: Require Power9.
+
+2021-11-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gfortran.dg/inline_matmul_17.f90: Fix template
+
+2021-11-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	* c-c++-common/tm/inline-asm.c: Disable pure-const.
+	* g++.dg/ipa/modref-1.C: Update template.
+	* gcc.dg/tree-ssa/modref-11.c: Disable pure-const.
+	* gcc.dg/tree-ssa/modref-14.c: New test.
+	* gcc.dg/tree-ssa/modref-8.c: Do not optimize sibling calls.
+	* gfortran.dg/do_subscript_3.f90: Add -O0.
+
+2021-11-11  Tobias Burnus  <tobias@codesourcery.com>
+
+	* gfortran.dg/gomp/num-teams-1.f90: New file.
+	* gfortran.dg/gomp/num-teams-2.f90: New file.
+
+2021-11-11  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* g++.dg/warn/Wmismatched-new-delete-5.C: Add
+	-fdelete-null-pointer-checks.
+	* gcc.dg/attr-returns-nonnull.c: Likewise.
+	* gcc.dg/debug/btf/btf-datasec-1.c: Add -G0 option for nios2.
+	* gcc.dg/ifcvt-4.c: Skip on nios2.
+	* gcc.dg/struct-by-value-1.c: Add -G0 option for nios2.
+
+2021-11-11  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/103188
+	* gcc.dg/torture/pr103188.c: New testcase.
+
+2021-11-11  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/103181
+	* gcc.dg/torture/pr103181.c: New testcase.
+
+2021-11-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR debug/101378
+	* g++.dg/debug/dwarf2/pr101378.C: New test.
+
+2021-11-11  Prathamesh Kulkarni  <prathamesh.kulkarni@linaro.org>
+
+	PR target/102376
+	* gcc.target/aarch64/pr102376.c: New test.
+
+2021-11-11  Jakub Jelinek  <jakub@redhat.com>
+
+	* c-c++-common/gomp/clauses-1.c (bar): Supply lower-bound expression
+	to half of the num_teams clauses.
+	* c-c++-common/gomp/num-teams-1.c: New test.
+	* c-c++-common/gomp/num-teams-2.c: New test.
+	* g++.dg/gomp/attrs-1.C (bar): Supply lower-bound expression
+	to half of the num_teams clauses.
+	* g++.dg/gomp/attrs-2.C (bar): Likewise.
+	* g++.dg/gomp/num-teams-1.C: New test.
+	* g++.dg/gomp/num-teams-2.C: New test.
+
+2021-11-11  liuhongt  <hongtao.liu@intel.com>
+
+	* gcc.target/i386/pr103151.c: New test.
+
+2021-11-10  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gcc.dg/tree-ssa/pr102906.c: New test.
+
+2021-11-10  Andrew Pinski  <apinski@marvell.com>
+
+	PR target/103170
+	* gcc.c-torture/compile/vector-dup-1.c: New test.
+
+2021-11-10  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR tree-optimization/102892
+	* gcc.dg/pr102892-1.c: New file.
+	* gcc.dg/pr102892-2.c: Likewise.
+
+2021-11-10  Martin Sebor  <msebor@redhat.com>
+
+	PR testsuite/103161
+	* gcc.dg/tree-ssa/builtin-sprintf-warn-16.c: Avoid relying on
+	argument evaluation order.  Cast width and precision to signed
+	to avoid undefined behavior.
+
+2021-11-10  qing zhao  <qing.zhao@oracle.com>
+
+	* gcc.target/i386/auto-init-6.c: _Complex long double is initialized
+	to zero now with -ftrivial-auto-var-init=pattern.
+
+2021-11-10  Tamar Christina  <tamar.christina@arm.com>
+
+	* gcc.target/aarch64/sve/pred-not-gen-1.c: Update testcase.
+	* gcc.target/aarch64/sve/pred-not-gen-2.c: Update testcase.
+	* gcc.target/aarch64/sve/pred-not-gen-3.c: Update testcase.
+	* gcc.target/aarch64/sve/pred-not-gen-4.c: Update testcase.
+
+2021-11-10  Tamar Christina  <tamar.christina@arm.com>
+
+	* gcc.target/aarch64/shrn-combine-8.c: Update.
+	* gcc.target/aarch64/shrn-combine-9.c: Update.
+
+2021-11-10  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gcc.dg/tree-ssa/modref-13.c: Fix typo.
+
+2021-11-10  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/101940
+	* c-c++-common/Wno-attributes-1.c: New test.
+	* c-c++-common/Wno-attributes-2.c: New test.
+	* c-c++-common/Wno-attributes-3.c: New test.
+
+2021-11-10  Jan Hubicka  <jh@suse.cz>
+
+	* g++.dg/ipa/modref-1.C: Update template.
+	* gcc.dg/ipa/modref-3.c: Update template.
+	* gcc.dg/lto/modref-3_0.c: Update template.
+	* gcc.dg/lto/modref-4_0.c: Update template.
+	* gcc.dg/tree-ssa/modref-10.c: Update template.
+	* gcc.dg/tree-ssa/modref-11.c: Update template.
+	* gcc.dg/tree-ssa/modref-5.c: Update template.
+	* gcc.dg/tree-ssa/modref-6.c: Update template.
+	* gcc.dg/tree-ssa/modref-13.c: New test.
+
+2021-11-10  Tamar Christina  <tamar.christina@arm.com>
+
+	PR testsuite/103042
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-long.c: Use
+	vect_long_long instead of vect_long.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-long.c:
+	Likewise.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-long.c: Likewise.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-long.c:
+	Likewise.
+
+2021-11-10  Tamar Christina  <tamar.christina@arm.com>
+
+	* gcc.dg/signbit-2.c: Turn off masks.
+	* gcc.dg/signbit-5.c: Likewise.
+
+2021-11-10  Richard Biener  <rguenther@suse.de>
+
+	PR testsuite/102690
+	* g++.dg/warn/Warray-bounds-16.C: XFAIL diagnostic part
+	and optimization.
+
+2021-11-10  Andre Vieira  <andre.simoesdiasvieira@arm.com>
+
+	* gcc.target/aarch64/simd/lowering_tbaa.c: New test.
+
+2021-11-10  Andre Vieira  <andre.simoesdiasvieira@arm.com>
+
+	* gcc.target/aarch64/fmla_intrinsic_1.c: Fix big-endian testism.
+	* gcc.target/aarch64/fmls_intrinsic_1.c: Likewise.
+	* gcc.target/aarch64/fmul_intrinsic_1.c: Likewise.
+
+2021-11-10  H.J. Lu  <hongjiu.lu@intel.com>
+	    Hongtao Liu  <hongtao.liu@intel.com>
+
+	PR middle-end/102566
+	* g++.target/i386/pr102566-1.C: New test.
+	* g++.target/i386/pr102566-2.C: Likewise.
+	* g++.target/i386/pr102566-3.C: Likewise.
+	* g++.target/i386/pr102566-4.C: Likewise.
+	* g++.target/i386/pr102566-5a.C: Likewise.
+	* g++.target/i386/pr102566-5b.C: Likewise.
+	* g++.target/i386/pr102566-6a.C: Likewise.
+	* g++.target/i386/pr102566-6b.C: Likewise.
+	* gcc.target/i386/pr102566-1a.c: Likewise.
+	* gcc.target/i386/pr102566-1b.c: Likewise.
+	* gcc.target/i386/pr102566-2.c: Likewise.
+	* gcc.target/i386/pr102566-3a.c: Likewise.
+	* gcc.target/i386/pr102566-3b.c: Likewise.
+	* gcc.target/i386/pr102566-4.c: Likewise.
+	* gcc.target/i386/pr102566-5.c: Likewise.
+	* gcc.target/i386/pr102566-6.c: Likewise.
+	* gcc.target/i386/pr102566-7.c: Likewise.
+	* gcc.target/i386/pr102566-8a.c: Likewise.
+	* gcc.target/i386/pr102566-8b.c: Likewise.
+	* gcc.target/i386/pr102566-9a.c: Likewise.
+	* gcc.target/i386/pr102566-9b.c: Likewise.
+	* gcc.target/i386/pr102566-10a.c: Likewise.
+	* gcc.target/i386/pr102566-10b.c: Likewise.
+	* gcc.target/i386/pr102566-11.c: Likewise.
+	* gcc.target/i386/pr102566-12.c: Likewise.
+	* gcc.target/i386/pr102566-13.c: New test.
+	* gcc.target/i386/pr102566-14.c: New test.
+
+2021-11-10  liuhongt  <hongtao.liu@intel.com>
+
+	* gcc.target/i386/ifcvt-reduction-logic-op.c: New test.
+
+2021-11-10  konglin1  <lingling.kong@intel.com>
+
+	* gcc.target/i386/avx512fp16-vector-complex-float.c: New test.
+
+2021-11-10  konglin1  <lingling.kong@intel.com>
+
+	* gcc.target/i386/avx512fp16vl-complex-broadcast-1.c: New test.
+
+2021-11-10  liuhongt  <hongtao.liu@intel.com>
+
+	* gcc.target/i386/pr102464-maxmin.c: New test.
+
+2021-11-10  Andrew Pinski  <apinski@marvell.com>
+
+	PR target/101529
+	* c-c++-common/torture/builtin-convertvector-2.c: New test.
+	* c-c++-common/torture/builtin-shufflevector-2.c: New test.
+
+2021-11-10  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* gcc.target/nios2/custom-fp-inline-1.c: New.
+	* gcc.target/nios2/custom-fp-inline-2.c: New.
+	* gcc.target/nios2/custom-fp-inline-3.c: New.
+	* gcc.target/nios2/custom-fp-inline-4.c: New.
+
+2021-11-09  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/bad-pragma-locations.c: New test.
+
+2021-11-09  Bill Schmidt  <wschmidt@linux.ibm.com>
+
+	* gcc.target/powerpc/p10_vec_xl_sext.c: Fix long long case.
+
+2021-11-09  Andrew Pinski  <apinski@marvell.com>
+
+	PR tree-optimization/10352
+	* gcc.c-torture/execute/pr10352-1.c: New test.
+
+2021-11-09  Marek Polacek  <polacek@redhat.com>
+
+	* g++.dg/init/bitfield6.C: New test.
+
+2021-11-09  Kito Cheng  <kito.cheng@sifive.com>
+
+	* gcc.target/riscv/pr102957.c: New.
+
+2021-11-09  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/103114
+	* g++.dg/ext/complex10.C: New test.
+
+2021-11-09  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/98394
+	PR c++/85846
+	* g++.dg/cpp2a/concepts-pr98394.C: New test.
+	* g++.dg/cpp2a/concepts-pr85846.C: New test.
+
+2021-11-09  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/100652
+	* g++.dg/cpp1y/var-templ69.C: New test.
+
+2021-11-09  Przemyslaw Wirkus  <przemyslaw.wirkus@arm.com>
+
+	* gcc.target/arm/multilib.exp: Update test with armv9-a entries.
+	* lib/target-supports.exp (v9a): Add new armflag.
+	(__ARM_ARCH_9A__): Add new armdef.
+
+2021-11-09  Martin Jambor  <mjambor@suse.cz>
+
+	PR ipa/103132
+	* gcc.dg/ipa/pr103132.c: New test.
+
+2021-11-09  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gcc.dg/tree-ssa/builtin-sprintf-warn-22.c: XFAIL.
+
+2021-11-09  Jeff Law  <jeffreyalaw@gmail.com>
+
+	* gcc.dg/tree-ssa/bittest.c: New test
+
+2021-11-08  Andrew MacLeod  <amacleod@redhat.com>
+
+	* g++.dg/pr103122.C: New.
+
+2021-11-08  Martin Jambor  <mjambor@suse.cz>
+
+	PR ipa/103099
+	PR ipa/103107
+	* g++.dg/ipa/pr103099.C: New test.
+	* gcc.dg/ipa/pr103107.c: Likewise.
+
+2021-11-08  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gcc.dg/pr103120.c: New test.
+
+2021-11-08  Martin Liska  <mliska@suse.cz>
+
+	PR gcov-profile/100520
+	* gcc.dg/pr100520.c: New test.
+
+2021-11-08  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/neoverse_v1_1.c: New test.
+
+2021-11-08  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gcc.dg/tree-ssa/modref-12.c: New test.
+
+2021-11-08  Haochen Gui  <guihaoc@gcc.gnu.org>
+
+	* gcc.target/powerpc/vec-minmax-1.c: New test.
+	* gcc.target/powerpc/vec-minmax-2.c: Likewise.
+
+2021-11-08  liuhongt  <hongtao.liu@intel.com>
+
+	* gcc.target/i386/pr102464-fma.c: New test.
+
+2021-11-07  Thomas Koenig  <tkoenig@gcc.gnu.org>
+
+	* gfortran.dg/co_reduce_2.f90: New test.
+	* gfortran.dg/coarray_collectives_14.f90: Change OPERATOR
+	to OPERATION.
+	* gfortran.dg/coarray_collectives_16.f90: Likewise.
+	* gfortran.dg/coarray_collectives_9.f90: Likewise.
+	Co-authored by: Steve Kargl <steve@gcc.gnu.org>
+
+2021-11-07  Sandra Loosemore  <sandra@codesourcery.com>
+
+	* gfortran.dg/bessel_3.f90: Expect additional diagnostics from
+	multiple bad arguments in the call.
+	* gfortran.dg/pr24823.f: Likewise.
+	* gfortran.dg/pr39937.f: Likewise.
+	* gfortran.dg/pr41011.f: Likewise.
+	* gfortran.dg/pr61318.f90: Likewise.
+	* gfortran.dg/c-interop/c407b-2.f90: Remove xfails.
+	* gfortran.dg/c-interop/c535b-2.f90: Likewise.
+
+2021-11-06  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/102715
+	* gfortran.dg/pr68019.f90: Adjust error message.
+	* gfortran.dg/pr102715.f90: New test.
+
+2021-11-06  David Edelsohn  <dje.gcc@gmail.com>
+
+	* gcc.dg/vect/tsvc/tsvc.h (init): Use posix_memalign on AIX.
+
+2021-11-05  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/102817
+	* gfortran.dg/pr102817.f90: New test.
+
+2021-11-05  Jan Hubicka  <hubicka@ucw.cz>
+
+	PR ipa/103073
+	* g++.dg/torture/pr103073.C: New test.
+	* gcc.dg/tree-ssa/modref-11.c: New test.
+
+2021-11-05  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/69419
+	* gfortran.dg/pr69419.f90: New test.
+
+2021-11-05  Bernhard Reutner-Fischer  <aldot@gcc.gnu.org>
+
+	PR fortran/100972
+	* gfortran.dg/implicit_14.f90: Adjust error.
+	* gfortran.dg/external_implicit_none_3.f08: New test.
+
+2021-11-05  Iain Sandoe  <iain@sandoe.co.uk>
+
+	* gcc.dg/vect/tsvc/tsvc.h: Do not try to include malloc.h
+	on Darwin also use posix_memalign ().
+
+2021-11-05  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gcc.dg/pr103093.c: New.
+
+2021-11-05  Wilco Dijkstra  <wdijkstr@arm.com>
+
+	PR target/103085
+	PR target/103085
+	* gcc.target/aarch64/pr103085.c: New test
+
+2021-11-05  Martin Liska  <mliska@suse.cz>
+
+	PR gcov-profile/102945
+	* gcc.dg/gcov-info-to-gcda.c: Filter supported targets.
+
+2021-11-05  Martin Liska  <mliska@suse.cz>
+
+	PR debug/102955
+	* g++.dg/pr102955.C: New test.
+
+2021-11-04  Jonathan Wakely  <jwakely@redhat.com>
+
+	* g++.dg/cpp0x/lambda/lambda-eh2.C: Add dg-warning for new
+	deprecation warnings.
+	* g++.dg/cpp0x/noexcept06.C: Likewise.
+	* g++.dg/cpp0x/noexcept07.C: Likewise.
+	* g++.dg/eh/forced3.C: Likewise.
+	* g++.dg/eh/unexpected1.C: Likewise.
+	* g++.old-deja/g++.eh/spec1.C: Likewise.
+	* g++.old-deja/g++.eh/spec2.C: Likewise.
+	* g++.old-deja/g++.eh/spec3.C: Likewise.
+	* g++.old-deja/g++.eh/spec4.C: Likewise.
+	* g++.old-deja/g++.mike/eh33.C: Likewise.
+	* g++.old-deja/g++.mike/eh34.C: Likewise.
+	* g++.old-deja/g++.mike/eh50.C: Likewise.
+	* g++.old-deja/g++.mike/eh51.C: Likewise.
+
+2021-11-04  Tamar Christina  <tamar.christina@arm.com>
+
+	* gcc.target/aarch64/advsimd-intrinsics/vshl-opt-1.c: New test.
+	* gcc.target/aarch64/advsimd-intrinsics/vshl-opt-2.c: New test.
+	* gcc.target/aarch64/advsimd-intrinsics/vshl-opt-3.c: New test.
+	* gcc.target/aarch64/advsimd-intrinsics/vshl-opt-4.c: New test.
+	* gcc.target/aarch64/advsimd-intrinsics/vshl-opt-5.c: New test.
+	* gcc.target/aarch64/advsimd-intrinsics/vshl-opt-6.c: New test.
+	* gcc.target/aarch64/advsimd-intrinsics/vshl-opt-7.c: New test.
+	* gcc.target/aarch64/advsimd-intrinsics/vshl-opt-8.c: New test.
+	* gcc.target/aarch64/signbit-2.c: New test.
+
+2021-11-04  Tamar Christina  <tamar.christina@arm.com>
+
+	* gcc.dg/signbit-2.c: New test.
+	* gcc.dg/signbit-3.c: New test.
+	* gcc.dg/signbit-4.c: New test.
+	* gcc.dg/signbit-5.c: New test.
+	* gcc.dg/signbit-6.c: New test.
+	* gcc.target/aarch64/signbit-1.c: New test.
+
+2021-11-04  Andrew MacLeod  <amacleod@redhat.com>
+
+	PR tree-optimization/103079
+	* gcc.dg/pr103079.c: New.
+
+2021-11-04  Martin Jambor  <mjambor@suse.cz>
+
+	PR ipa/93385
+	* gcc.dg/guality/ipa-sra-1.c: New test.
+
+2021-11-04  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* gcc.target/aarch64/vector_structure_intrinsics.c: New code
+	generation tests.
+
+2021-11-04  Jonathan Wright  <jonathan.wright@arm.com>
+
+	* gcc.target/aarch64/advsimd-intrinsics/bf16_vldN_lane_2.c:
+	Relax incorrect register number requirement.
+	* gcc.target/aarch64/sve/pcs/struct_3_256.c: Accept
+	equivalent codegen with fmov.
+
+2021-11-04  H.J. Lu  <hjl.tools@gmail.com>
+
+	* gcc.target/i386/amxtile-3.c: Check leal/addl for x32.
+
+2021-11-04  Tamar Christina  <tamar.christina@arm.com>
+
+	PR testsuite/103042
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-int.c: Update guards.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-long.c: Likewise.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-short.c: Likewise.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-int.c:
+	Likewise.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-long.c:
+	Likewise.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-short.c:
+	Likewise.
+	* gcc.dg/vect/complex/complex-add-pattern-template.c: Likewise.
+	* gcc.dg/vect/complex/complex-add-template.c: Likewise.
+	* gcc.dg/vect/complex/complex-operations-run.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-double.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-half-float.c:
+	Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-double.c:
+	Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-float.c:
+	Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-half-float.c:
+	Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-double.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-half-float.c:
+	Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-double.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-half-float.c:
+	Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-double.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-half-float.c:
+	Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-add-double.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-add-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-add-half-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-add-pattern-double.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-add-pattern-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-add-pattern-half-float.c:
+	Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-mla-double.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-mla-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-mla-half-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-mls-double.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-mls-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-mls-half-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-mul-double.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-mul-float.c: Likewise.
+	* gcc.dg/vect/complex/fast-math-complex-mul-half-float.c: Likewise.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-byte.c: Likewise.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-int.c: Likewise.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-long.c: Likewise.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-short.c: Likewise.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-byte.c:
+	Likewise.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-int.c:
+	Likewise.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-long.c:
+	Likewise.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-short.c:
+	Likewise.
+
+2021-11-04  Richard Biener  <rguenther@suse.de>
+
+	PR rtl-optimization/103075
+	* gcc.dg/pr103075.c: New testcase.
+
+2021-11-04  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR tree-optimization/103062
+	* gcc.dg/pr103062.c: New test.
+
+2021-11-04  Jiufu Guo  <guojiufu@linux.ibm.com>
+
+	* gcc.dg/vect/pr101145_1.c: Update case.
+	* gcc.dg/vect/pr101145_2.c: Update case.
+	* gcc.dg/vect/pr101145_3.c: Update case.
+
+2021-11-04  Martin Liska  <mliska@suse.cz>
+
+	* g++.dg/asan/asan_test.C: Disable one warning.
+
+2021-11-04  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.dg/rtl/aarch64/big-endian-cse-1.c: New test.
+
+2021-11-04  liuhongt  <hongtao.liu@intel.com>
+
+	* gcc.target/i386/pr101989-3.c: New test.
+
+2021-11-04  liuhongt  <hongtao.liu@intel.com>
+
+	* gcc.target/i386/pr102464-copysign-1.c: New test.
+
+2021-11-04  Hongyu Wang  <hongyu.wang@intel.com>
+
+	* gcc.target/i386/vnni-auto-vectorize-1.c: New test.
+	* gcc.target/i386/vnni-auto-vectorize-2.c: Ditto.
+
+2021-11-04  Hongyu Wang  <hongyu.wang@intel.com>
+
+	* gcc.target/i386/amxtile-3.c: New test.
+
+2021-11-04  Marek Polacek  <polacek@redhat.com>
+
+	* g++.dg/opt/pr102970.C: Only run in C++14 and up.
+
+2021-11-03  Joseph Myers  <joseph@codesourcery.com>
+
+	PR c/103031
+	* gcc.dg/init-rounding-math-1.c: New test.
+
+2021-11-03  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gcc.dg/tree-ssa/pr23744.c: Tweak output checks.
+	* gcc.dg/tree-ssa/vrp07.c: Ditto.
+	* gcc.dg/tree-ssa/vrp08.c: Ditto.
+	* gcc.dg/tree-ssa/vrp09.c: Ditto.
+	* gcc.dg/tree-ssa/vrp20.c: Ditto.
+	* gcc.dg/tree-ssa/vrp92.c: Ditto.
+	* jit.dg/test-sum-of-squares.c: Ditto.
+
+2021-11-03  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/102970
+	* g++.dg/opt/pr102970.C: New testcase.
+
+2021-11-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* g++.dg/torture/pr103040.C: New test.
+
+2021-11-02  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gcc.dg/ipa/modref-3.c: New test.
+
+2021-11-02  Jan Hubicka  <hubicka@ucw.cz>
+
+	* g++.dg/ipa/modref-1.C: Update template.
+	* gcc.dg/tree-ssa/modref-10.c: New test.
+
+2021-11-02  Stefan Schulze Frielinghaus  <stefansf@linux.ibm.com>
+
+	* gcc.dg/tree-ssa/ldist-rawmemchr-1.c: For IBM Z set arch to z13
+	and use z/Architecture since the tests require vector extensions.
+	* gcc.dg/tree-ssa/ldist-rawmemchr-2.c: Likewise.
+	* gcc.dg/tree-ssa/ldist-strlen-1.c: Likewise.
+	* gcc.dg/tree-ssa/ldist-strlen-3.c: Likewise.
+
+2021-11-02  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/103007
+	* g++.dg/pr103007.C: New test.
+
+2021-11-02  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/103038
+	* g++.dg/pr103038.C: New testcase.
+
+2021-11-02  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/103020
+	* gcc.target/i386/pr103020.c: New test.
+
+2021-11-02  Martin Liska  <mliska@suse.cz>
+
+	* gcc.dg/vect/vect.exp: Include also tsvc sub-directory.
+	* gcc.dg/vect/tsvc/license.txt: New test.
+	* gcc.dg/vect/tsvc/tsvc.h: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s000.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s111.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1111.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1112.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1113.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1115.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1119.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s112.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s113.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s114.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s115.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s116.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1161.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s118.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s119.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s121.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1213.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s122.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1221.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s123.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1232.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s124.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1244.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s125.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1251.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s126.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s127.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1279.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s128.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1281.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s131.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s13110.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s132.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1351.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s141.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s1421.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s151.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s152.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s161.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s162.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s171.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s172.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s173.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s174.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s175.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s176.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s2101.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s2102.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s211.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s2111.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s212.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s221.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s222.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s2233.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s2244.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s2251.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s2275.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s231.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s232.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s233.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s235.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s241.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s242.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s243.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s244.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s251.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s252.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s253.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s254.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s255.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s256.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s257.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s258.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s261.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s271.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s2710.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s2711.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s2712.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s272.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s273.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s274.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s275.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s276.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s277.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s278.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s279.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s281.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s291.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s292.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s293.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s311.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s3110.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s3111.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s31111.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s3112.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s3113.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s312.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s313.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s314.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s315.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s316.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s317.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s318.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s319.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s321.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s322.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s323.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s3251.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s331.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s332.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s341.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s342.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s343.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s351.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s352.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s353.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s4112.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s4113.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s4114.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s4115.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s4116.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s4117.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s4121.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s421.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s422.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s423.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s424.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s431.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s441.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s442.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s443.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s451.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s452.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s453.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s471.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s481.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s482.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-s491.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-va.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vag.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vas.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vbor.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vdotr.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vif.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vpv.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vpvpv.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vpvts.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vpvtv.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vsumr.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vtv.c: New test.
+	* gcc.dg/vect/tsvc/vect-tsvc-vtvtv.c: New test.
+
+2021-11-02  liuhongt  <hongtao.liu@intel.com>
+
+	PR testsuite/102944
+	* c-c++-common/Wstringop-overflow-2.c: Adjust target/xfail
+	selector.
+	* gcc.dg/Warray-bounds-48.c: Ditto.
+	* gcc.dg/Warray-bounds-51.c: Ditto.
+	* gcc.dg/Warray-parameter-3.c: Ditto.
+	* gcc.dg/Wstringop-overflow-14.c: Ditto.
+	* gcc.dg/Wstringop-overflow-21.c: Ditto.
+	* gcc.dg/Wstringop-overflow-68.c: Ditto
+	* gcc.dg/Wstringop-overflow-76.c: Ditto
+	* gcc.dg/Wzero-length-array-bounds-2.c: Ditto.
+	* lib/target-supports.exp (vect_slp_v4qi_store_unalign): New
+	efficient target.
+	(vect_slp_v4qi_store_unalign_1): Ditto.
+	(struct_4char_block_move): Ditto.
+	(struct_8char_block_move): Ditto.
+	(stryct_16char_block_move): Ditto.
+	(vect_slp_v2hi_store_align): Ditto.
+	(vect_slp_v2qi_store): Rename to ..
+	(vect_slp_v2qi_store_align): .. this.
+	(vect_slp_v4qi_store): Rename to ..
+	(vect_slp_v4qi_store_align): .. This.
+	(vect_slp_v8qi_store): Rename to ..
+	(vect_slp_v8qi_store_unalign_1): .. This.
+	(vect_slp_v16qi_store): Rename to ..
+	(vect_slp_v16qi_store_unalign_1): .. This.
+	(vect_slp_v2hi_store): Rename to ..
+	(vect_slp_v2hi_store_unalign): .. This.
+	(vect_slp_v4hi_store): Rename to ..
+	(vect_slp_v4hi_store_unalign): This.
+	(vect_slp_v2si_store): Rename to ..
+	(vect_slp_v2si_store_align): .. This.
+	(vect_slp_v4si_store): Rename to ..
+	(vect_slp_v4si_store_unalign): Ditto.
+	(check_vect_slp_aligned_store_usage): Rename to ..
+	(check_vect_slp_store_usage): .. this and adjust code to make
+	it an exact pattern match of corresponding testcase.
+
+2021-11-02  Roger Sayle  <roger@nextmovesoftware.com>
+	    Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/102986
+	* gcc.target/i386/sse2-v1ti-ashiftrt-1.c: New test case.
+	* gcc.target/i386/sse2-v1ti-ashiftrt-2.c: New test case.
+	* gcc.target/i386/sse2-v1ti-ashiftrt-3.c: New test case.
+	* gcc.target/i386/sse2-v1ti-shift-2.c: New test case.
+	* gcc.target/i386/sse2-v1ti-shift-3.c: New test case.
+
+2021-11-02  Jakub Jelinek  <jakub@redhat.com>
+
+	* g++.dg/gomp/loop-8.C: New test.
+
+2021-11-01  Martin Liska  <mliska@suse.cz>
+
+	* g++.dg/ipa/modref-1.C: Fix test-suite pattern scanning.
+
+2021-11-01  qing zhao  <qing.zhao@oracle.com>
+
+	* c-c++-common/pr102281.c: New test.
+	* gcc.target/i386/auto-init-2.c: Adjust testing case.
+	* gcc.target/i386/auto-init-4.c: Likewise.
+	* gcc.target/i386/auto-init-6.c: Likewise.
+	* gcc.target/aarch64/auto-init-6.c: Likewise.
+
+2021-11-01  Tamar Christina  <tamar.christina@arm.com>
+
+	* gcc.target/aarch64/vect-cse-codegen.c: New test.
+
+2021-11-01  Tamar Christina  <tamar.christina@arm.com>
+
+	PR testsuite/103000
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-double.c:
+	Force unroll.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-float.c: likewise
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-float.c:
+	Likewise
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-half-float.c:
+	Likewise.
+
+2021-11-01  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-c++-common/diagnostic-format-json-1.c: Add regexp to consume
+	"escape-source" attribute.
+	* c-c++-common/diagnostic-format-json-2.c: Likewise.
+	* c-c++-common/diagnostic-format-json-3.c: Likewise.
+	* c-c++-common/diagnostic-format-json-4.c: Likewise, twice.
+	* c-c++-common/diagnostic-format-json-5.c: Likewise.
+	* gcc.dg/cpp/warn-normalized-4-bytes.c: New test.
+	* gcc.dg/cpp/warn-normalized-4-unicode.c: New test.
+	* gcc.dg/encoding-issues-bytes.c: New test.
+	* gcc.dg/encoding-issues-unicode.c: New test.
+	* gfortran.dg/diagnostic-format-json-1.F90: Add regexp to consume
+	"escape-source" attribute.
+	* gfortran.dg/diagnostic-format-json-2.F90: Likewise.
+	* gfortran.dg/diagnostic-format-json-3.F90: Likewise.
+
+2021-11-01  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gcc.dg/pr103003.c: New.
+
+2021-10-31  Jan Hubicka  <hubicka@ucw.cz>
+
+	* g++.dg/ipa/modref-1.C: New test.
+
+2021-10-31  Iain Buclaw  <ibuclaw@gdcproject.org>
+
+	PR d/102959
+	* gdc.dg/torture/pr96435.d: Adjust for BigEndian.
+
+2021-10-30  Tobias Burnus  <tobias@codesourcery.com>
+
+	PR middle-end/102972
+	* c-c++-common/gomp/target-device-ancestor-3.c: Add non-API
+	routine test.
+	* gfortran.dg/gomp/order-6.f90: Add missing bind(C).
+	* c-c++-common/gomp/teams-3.c: New test.
+	* gfortran.dg/gomp/teams-3.f90: New test.
+	* gfortran.dg/gomp/teams-4.f90: New test.
+
+2021-10-30  Manfred Schwarb  <manfred99@gmx.ch>
+
+	* gfortran.dg/intrinsic_short-long.f90: New test.
+
+2021-10-30  Steve Kargl  <kargl@gcc.gnu.org>
+
+	PR fortran/99853
+	* gfortran.dg/pr99853.f90: New test.
+
+2021-10-30  Jakub Jelinek  <jakub@redhat.com>
+
+	* c-c++-common/gomp/loop-10.c: New test.
+
+2021-10-29  Tamar Christina  <tamar.christina@arm.com>
+
+	* g++.dg/vect/pr99149.cc: Update case.
+
+2021-10-29  Tobias Burnus  <tobias@codesourcery.com>
+
+	* c-c++-common/gomp/pragma-1.c: New test.
+	* c-c++-common/gomp/pragma-2.c: New test.
+
+2021-10-29  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gcc.dg/loop-unswitch-2.c: Adjust for threading changes.
+	* gcc.dg/old-style-asm-1.c: Same.
+	* gcc.dg/tree-ssa/phi_on_compare-1.c: Same.
+	* gcc.dg/tree-ssa/phi_on_compare-2.c: Same.
+	* gcc.dg/tree-ssa/phi_on_compare-3.c: Same.
+	* gcc.dg/tree-ssa/phi_on_compare-4.c: Same.
+	* gcc.dg/tree-ssa/pr20701.c: Same.
+	* gcc.dg/tree-ssa/pr21001.c: Same.
+	* gcc.dg/tree-ssa/pr21294.c: Same.
+	* gcc.dg/tree-ssa/pr21417.c: Same.
+	* gcc.dg/tree-ssa/pr21559.c: Same.
+	* gcc.dg/tree-ssa/pr21563.c: Same.
+	* gcc.dg/tree-ssa/pr49039.c: Same.
+	* gcc.dg/tree-ssa/pr59597.c: Same.
+	* gcc.dg/tree-ssa/pr61839_1.c: Same.
+	* gcc.dg/tree-ssa/pr61839_3.c: Same.
+	* gcc.dg/tree-ssa/pr66752-3.c: Same.
+	* gcc.dg/tree-ssa/pr68198.c: Same.
+	* gcc.dg/tree-ssa/pr77445-2.c: Same.
+	* gcc.dg/tree-ssa/pr77445.c: Same.
+	* gcc.dg/tree-ssa/ranger-threader-1.c: Same.
+	* gcc.dg/tree-ssa/ranger-threader-2.c: Same.
+	* gcc.dg/tree-ssa/ranger-threader-4.c: Same.
+	* gcc.dg/tree-ssa/ssa-dom-thread-1.c: Same.
+	* gcc.dg/tree-ssa/ssa-dom-thread-11.c: Same.
+	* gcc.dg/tree-ssa/ssa-dom-thread-12.c: Same.
+	* gcc.dg/tree-ssa/ssa-dom-thread-14.c: Same.
+	* gcc.dg/tree-ssa/ssa-dom-thread-16.c: Same.
+	* gcc.dg/tree-ssa/ssa-dom-thread-2b.c: Same.
+	* gcc.dg/tree-ssa/ssa-dom-thread-7.c: Same.
+	* gcc.dg/tree-ssa/ssa-thread-14.c: Same.
+	* gcc.dg/tree-ssa/ssa-thread-backedge.c: Same.
+	* gcc.dg/tree-ssa/ssa-vrp-thread-1.c: Same.
+	* gcc.dg/tree-ssa/vrp02.c: Same.
+	* gcc.dg/tree-ssa/vrp03.c: Same.
+	* gcc.dg/tree-ssa/vrp05.c: Same.
+	* gcc.dg/tree-ssa/vrp06.c: Same.
+	* gcc.dg/tree-ssa/vrp07.c: Same.
+	* gcc.dg/tree-ssa/vrp08.c: Same.
+	* gcc.dg/tree-ssa/vrp09.c: Same.
+	* gcc.dg/tree-ssa/vrp33.c: Same.
+	* gcc.dg/uninit-pred-9_b.c: Same.
+	* gcc.dg/uninit-pred-7_a.c: xfail.
+
+2021-10-29  Jeff Law  <jeffreyalaw@gmail.com>
+
+	* lib/multiline.exp (_build_multiline_regex): Use a better
+	regexp than .* to match up to EOL.
+
+2021-10-29  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gcc.dg/pr102983.c: New.
+
+2021-10-29  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/102977
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-int.c: Updated.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-long.c: Updated.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-short.c: Updated.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-int.c:
+	Updated.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-long.c:
+	Updated.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-short.c:
+	Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-double.c:
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-half-float.c:
+	Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-double.c:
+	Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-float.c:
+	Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-half-float.c:
+	Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-double.c:
+	Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-half-float.c:
+	Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-double.c:
+	Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-half-float.c:
+	Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-double.c: Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-half-float.c:
+	Updated.
+	* gcc.dg/vect/complex/fast-math-complex-add-double.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-add-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-add-half-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-add-pattern-double.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-add-pattern-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-add-pattern-half-float.c:
+	Updated.
+	* gcc.dg/vect/complex/fast-math-complex-mla-double.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-mla-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-mla-half-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-mls-double.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-mls-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-mls-half-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-mul-double.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-mul-float.c: Updated.
+	* gcc.dg/vect/complex/fast-math-complex-mul-half-float.c: Updated.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-byte.c: Updated.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-int.c: Updated.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-long.c: Updated.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-short.c: Updated.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-byte.c:
+	Updated.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-int.c:
+	Updated.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-long.c:
+	Updated.
+	* gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-short.c:
+	Updated.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-byte.c: Removed.
+	* gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-byte.c:
+	Removed.
+
+2021-10-29  Jakub Jelinek  <jakub@redhat.com>
+
+	* gcc.dg/Wstringop-overflow-62.c: Adjust expected diagnostics.
+
+2021-10-29  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/torture/fp-uint64-convert-double-1.c: Add
+	-fexcess-precision=standard.
+
+2021-10-29  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/102820
+	* g++.dg/cpp0x/dr2351.C: New test.
+
+2021-10-29  Xionghu Luo  <luoxhu@linux.ibm.com>
+
+	PR target/102868
+	* gcc.target/powerpc/pr102868.c: New test.
+
+2021-10-29  liuhongt  <hongtao.liu@intel.com>
+
+	* gcc.target/i386/pr102464-vrndscaleph.c: New test.
+
+2021-10-28  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/102951
+	* gcc.dg/tree-ssa/pr102951.c: New test.
+	* gcc.dg/Wstringop-overflow-62.c: Adjust expected diagnostics.
+
+2021-10-28  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gcc.dg/ifcvt-4.c: Adjust.
+
+2021-10-28  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gcc.dg/tree-ssa/evrp9.c: Adjust message scanned for.
+	* gcc.dg/tree-ssa/pr21458-2.c: Ditto.
+
+2021-10-28  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/102933
+	* g++.dg/cpp2a/nontype-class50.C: New test.
+	* g++.dg/cpp2a/nontype-class50a.C: New test.
+
+2021-10-28  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/102780
+	* g++.dg/cpp1z/fold13.C: New test.
+
+2021-10-28  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/84407
+	* gcc.dg/torture/fp-uint64-convert-double-1.c: New testcase.
+	* gcc.dg/torture/fp-uint64-convert-double-2.c: Likewise.
+
+2021-10-28  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gcc.dg/tree-ssa/vrp106.c: Adjust for threading.
+	* gcc.dg/tree-ssa/vrp113.c: Same.
+
+2021-10-28  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/57245
+	* gcc.dg/torture/fp-double-convert-float-1.c: New testcase.
+
+2021-10-28  Kewen Lin  <linkw@linux.ibm.com>
+
+	PR target/102767
+	* gcc.target/powerpc/ppc-fortran/pr102767.f90: New file.
+
+2021-10-28  Alexandre Oliva  <oliva@adacore.com>
+
+	* c-c++-common/torture/harden-comp.c: New.
+	* c-c++-common/torture/harden-cond.c: New.
+
+2021-10-28  Xionghu Luo  <luoxhu@linux.ibm.com>
+
+	* gcc.target/powerpc/builtins-1.c: Adjust.
+
+2021-10-28  Xionghu Luo  <luoxhu@linux.ibm.com>
+
+	PR target/94613
+	* gcc.target/powerpc/pr94613.c: New test.
+
+2021-10-28  Hongyu Wang  <hongyu.wang@intel.com>
+
+	* gcc.target/i386/avx512fp16-recip-1.c: New test.
+	* gcc.target/i386/avx512fp16-recip-2.c: Ditto.
+	* gcc.target/i386/pr102464.c: Add -fno-trapping-math.
+
+2021-10-27  Martin Sebor  <msebor@redhat.com>
+
+	* gcc.dg/Warray-bounds-90.c: Fix a typo.
+
+2021-10-27  Tobias Burnus  <tobias@codesourcery.com>
+
+	* gfortran.dg/PR93963.f90: Extend testcase by scan-tree-dump test.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/102897
+	* gcc.dg/pr102897.c: Add -Wno-psabi to dg-options.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	* c-c++-common/gomp/loop-8.c: New test.
+	* c-c++-common/gomp/loop-9.c: New test.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/102854
+	* g++.dg/gomp/loop-3.C: Don't expect some errors.
+	* g++.dg/gomp/loop-7.C: New test.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/102753
+	* g++.dg/cpp2a/consteval7.C: Expect diagnostics on quux.
+	* g++.dg/cpp2a/consteval24.C: New test.
+	* g++.dg/cpp23/consteval-if12.C: New test.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/102753
+	* g++.dg/cpp2a/consteval13.C: Don't expect errors.
+	* g++.dg/cpp2a/consteval20.C: New test.
+	* g++.dg/cpp2a/consteval21.C: New test.
+	* g++.dg/cpp2a/consteval22.C: New test.
+	* g++.dg/cpp2a/consteval23.C: New test.
+	* g++.dg/cpp23/consteval-if11.C: New test.
+
+2021-10-26  Martin Sebor  <msebor@redhat.com>
+
+	PR tree-optimization/102238
+	PR tree-optimization/102919
+	* gcc.dg/tree-ssa/builtin-sprintf-warn-23.c: Remove warnings.
+	* gcc.dg/Wrestrict-23.c: New test.
+
+2021-10-26  Martin Sebor  <msebor@redhat.com>
+
+	* gcc.dg/Wstringop-overflow-22.c: Correct typos.
+	* gcc.dg/Wstringop-overflow-81.c: New test.
+
+2021-10-26  Martin Sebor  <msebor@redhat.com>
+
+	PR middle-end/102453
+	* gcc.dg/Warray-bounds-90.c: New test.
+	* gcc.dg/Wstringop-overflow-77.c: New test.
+	* gcc.dg/Wstringop-overflow-78.c: New test.
+	* gcc.dg/Wstringop-overflow-79.c: New test.
+	* gcc.dg/Wstringop-overflow-80.c: New test.
+	* c-c++-common/gomp/atomic-4.c: Avoid an out-of-bounds access.
+
+2021-10-26  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/86551
+	* gfortran.dg/pr86551.f90: New test to verify that PR86551 remains
+	fixed.
+
+2021-10-26  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/102956
+	* gfortran.dg/pdt_32.f03: New test.
+
+2021-10-26  Vladimir N. Makarov  <vmakarov@redhat.com>
+
+	PR rtl-optimization/102842
+	* g++.target/arm/pr102842.C: New test.
+
+2021-10-26  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/102917
+	* gfortran.dg/pdt_4.f03: Adjust testcase.
+
+2021-10-26  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/102816
+	* gfortran.dg/pr102816.f90: New test.
+
+2021-10-26  Paul A. Clarke  <pc@us.ibm.com>
+
+	* gcc.target/powerpc/pr78102.c: Fix dg directives to require Power8
+	vector support.  Also, add -DNO_WARN_X86_INTRINSICS.
+
+2021-10-26  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/102617
+	* g++.dg/cpp23/init-stmt1.C: New test.
+	* g++.dg/cpp23/init-stmt2.C: New test.
+
+2021-10-26  Sandra Loosemore  <sandra@codesourcery.com>
+
+	PR testsuite/102910
+	* gfortran.dg/c-interop/cf-descriptor-5-c.c: Use a static buffer
+	instead of alloca.
+
+2021-10-26  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	PR testsuite/102834
+	* gcc.target/i386/avx512f-pr96891-3.c: Add -mstv -mno-stackrealign
+	to dg-options.
+
+2021-10-26  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	PR testsuite/102835
+	* gcc.target/i386/avx512fp16-trunchf.c: Allow for %esp instead of
+	%ebp.
+
+2021-10-26  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	PR testsuite/102836
+	* gcc.target/i386/pieces-memset-1.c: Add -mno-stackrealign to
+	dg-options.
+	* gcc.target/i386/pieces-memset-4.c: Likewise.
+	* gcc.target/i386/pieces-memset-7.c: Likewise.
+	* gcc.target/i386/pieces-memset-8.c: Likewise.
+	* gcc.target/i386/pieces-memset-41.c: Likewise.
+	* gcc.target/i386/pr90773-1.c: Likewise.
+
+2021-10-26  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* gcc.target/i386/pr100704-1.c: Add -fomit-frame-pointer to
+	dg-options.
+	* gcc.target/i386/pr100704-2.c: Likewise.
+
+2021-10-26  Kewen Lin  <linkw@linux.ibm.com>
+
+	* gcc.dg/pr102897.c: New test.
+
+2021-10-26  Tobias Burnus  <tobias@codesourcery.com>
+
+	PR fortran/102885
+	* gfortran.dg/lto/bind-c-char_0.f90: New test.
+
+2021-10-26  Roger Sayle  <roger@nextmovesoftware.com>
+
+	* gcc.target/i386/sse2-v1ti-shift.c: New test case.
+
+2021-10-26  Aldy Hernandez  <aldyh@redhat.com>
+
+	PR testsuite/102857
+	* gcc.dg/tree-ssa/ssa-dom-thread-7.c: Add -fdump-tree-vrp2-stats.
+	Tweak for aarch64.
+
+2021-10-26  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gcc.dg/tree-ssa/pr21090.c: Adjust for threading.
+	* gcc.dg/tree-ssa/ssa-thread-12.c: Removed.
+
+2021-10-26  Aldy Hernandez  <aldyh@redhat.com>
+
+	* gcc.dg/graphite/scop-dsyr2k-2.c: Adjust for jump threading changes.
+	* gcc.dg/graphite/scop-dsyr2k.c: Same.
+	* gcc.dg/graphite/scop-dsyrk-2.c: Same.
+	* gcc.dg/graphite/scop-dsyrk.c: Same.
+	* gcc.dg/tree-ssa/pr20701.c: Same.
+	* gcc.dg/tree-ssa/pr20702.c: Same.
+	* gcc.dg/tree-ssa/pr21086.c: Same.
+	* gcc.dg/tree-ssa/pr25382.c: Same.
+	* gcc.dg/tree-ssa/pr58480.c: Same.
+	* gcc.dg/tree-ssa/ssa-vrp-thread-1.c: Same.
+	* gcc.dg/tree-ssa/vrp08.c: Same.
+	* gcc.dg/tree-ssa/vrp55.c: Same.
+	* gcc.dg/tree-ssa/ssa-dom-thread-7.c: Same.
+	* gcc.dg/tree-ssa/ssa-dom-thread-4.c: Removed.
+	* gcc.dg/tree-ssa/ssa-thread-11.c: Removed.
+	* gcc.dg/uninit-pr89230-1.c: xfail.
+	* gcc.dg/tree-ssa/ssa-thread-backedge.c: New file.
+
+2021-10-25  Andrew MacLeod  <amacleod@redhat.com>
+
+	* gcc.dg/tree-ssa/vrp98.c: Disable evrp for vrp1 test.
+	* gcc.dg/tree-ssa/vrp98-1.c: New. Test for folding in evrp.
+
+2021-10-25  Roger Sayle  <roger@nextmovesoftware.com>
+
+	* gcc.target/bfin/ssashift-1.c: New test case.
+
+2021-10-25  Tamar Christina  <tamar.christina@arm.com>
+
+	PR target/102907
+	* gcc.target/aarch64/shrn-combine-1.c: Disable SVE.
+	* gcc.target/aarch64/shrn-combine-2.c: Likewise.
+	* gcc.target/aarch64/shrn-combine-3.c: Likewise.
+	* gcc.target/aarch64/shrn-combine-4.c: Likewise.
+	* gcc.target/aarch64/shrn-combine-5.c: Likewise.
+	* gcc.target/aarch64/shrn-combine-6.c: Likewise.
+	* gcc.target/aarch64/shrn-combine-7.c: Likewise.
+
+2021-10-25  Jim Wilson  <jimw@sifive.com>
+	    Kito Cheng  <kito.cheng@sifive.com>
+	    Jia-Wei Chen  <jiawei@iscas.ac.cn>
+	    Shi-Hua Liao  <shihua@iscas.ac.cn>
+
+	* gcc.target/riscv/zba-slliuw.c: Apply zbs to this testcase.
+	* gcc.target/riscv/zbs-bclr.c: New.
+	* gcc.target/riscv/zbs-bext.c: Ditto.
+	* gcc.target/riscv/zbs-binv.c: Ditto.
+	* gcc.target/riscv/zbs-bset.c: Ditto.
+
+2021-10-25  Jim Wilson  <jimw@sifive.com>
+
+	* gcc.target/riscv/zbb-li-rotr.c: New.
+
+2021-10-25  Jim Wilson  <jimw@sifive.com>
+	    Kito Cheng  <kito.cheng@sifive.com>
+	    Jia-Wei Chen  <jiawei@iscas.ac.cn>
+
+	* gcc.target/riscv/zbb-andn-orn-xnor-01.c: New.
+	* gcc.target/riscv/zbb-andn-orn-xnor-02.c: Ditto.
+	* gcc.target/riscv/zbb-min-max.c: Ditto.
+	* gcc.target/riscv/zbb-rol-ror-01.c: Ditto.
+	* gcc.target/riscv/zbb-rol-ror-02.c: Ditto.
+	* gcc.target/riscv/zbb-rol-ror-03.c: Ditto.
+	* gcc.target/riscv/zbbw.c: Ditto.
+
+2021-10-25  Jim Wilson  <jimw@sifive.com>
+	    Kito Cheng  <kito.cheng@sifive.com>
+	    Jia-Wei Chen  <jiawei@iscas.ac.cn>
+
+	* gcc.target/riscv/zba-adduw.c: New.
+	* gcc.target/riscv/zba-shNadd-01.c: Ditto.
+	* gcc.target/riscv/zba-shNadd-02.c: Ditto.
+	* gcc.target/riscv/zba-shNadd-03.c: Ditto.
+	* gcc.target/riscv/zba-slliuw.c: Ditto.
+	* gcc.target/riscv/zba-zextw.c: Ditto.
+
+2021-10-25  liuhongt  <hongtao.liu@intel.com>
+
+	PR target/102464
+	* gcc.target/i386/pr102464-sqrtph.c: New test.
+	* gcc.target/i386/pr102464-sqrtsh.c: New test.
+
+2021-10-25  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/102920
+	* gcc.dg/torture/pr102920.c: New testcase.
+
+2021-10-25  konglin1  <lingling.kong@intel.com>
+
+	* gcc.target/i386/avx512fp16-complex-fma.c: New test.
+
+2021-10-24  Roger Sayle  <roger@nextmovesoftware.com>
+
+	* gcc.target/bfin/20090914-3.c: Tweak test case.
+
+2021-10-23  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR fortran/9262
+	* gfortran.dg/bind-c-intent-out-2.f90: Moved to ...
+	* gfortran.dg/ubsan/bind-c-intent-out-2.f90
+
+2021-10-23  Roger Sayle  <roger@nextmovesoftware.com>
+
+	* gcc.target/i386/sse2-v1ti-logic.c: New test case.
+	* gcc.target/i386/sse2-v1ti-logic-2.c: New test case.
+
+2021-10-23  José Rui Faustino de Sousa  <jrfsousa@gmail.com>
+	    Sandra Loosemore  <sandra@codesourcery.com>
+
+	PR fortran/95196
+	* gfortran.dg/PR95196.f90: New.
+
+2021-10-22  Tobias Burnus  <tobias@codesourcery.com>
+
+	* gfortran.dg/associate_3.f03: Replace dg-excess-errors by
+	other dg-* to change XFAIL to PASS.
+	* gfortran.dg/binding_label_tests_4.f03: Likewise.
+	* gfortran.dg/block_4.f08: Likewise.
+	* gfortran.dg/charlen_04.f90: Likewise.
+	* gfortran.dg/charlen_05.f90: Likewise.
+	* gfortran.dg/charlen_06.f90: Likewise.
+	* gfortran.dg/charlen_13.f90: Likewise.
+	* gfortran.dg/coarray_9.f90: Likewise.
+	* gfortran.dg/coarray_collectives_3.f90: Likewise.
+	* gfortran.dg/data_invalid.f90: Likewise.
+	* gfortran.dg/do_4.f: Likewise.
+	* gfortran.dg/dollar_sym_1.f90: Likewise.
+	* gfortran.dg/dollar_sym_3.f: Likewise.
+	* gfortran.dg/fmt_tab_1.f90: Likewise.
+	* gfortran.dg/fmt_tab_2.f90: Likewise.
+	* gfortran.dg/forall_16.f90: Likewise.
+	* gfortran.dg/g77/970125-0.f: Likewise.
+	* gfortran.dg/gomp/unexpected-end.f90: Likewise.
+	* gfortran.dg/interface_operator_1.f90: Likewise.
+	* gfortran.dg/interface_operator_2.f90: Likewise.
+	* gfortran.dg/line_length_4.f90: Likewise.
+	* gfortran.dg/line_length_5.f90: Likewise.
+	* gfortran.dg/line_length_6.f90: Likewise.
+	* gfortran.dg/line_length_8.f90: Likewise.
+	* gfortran.dg/line_length_9.f90: Likewise.
+	* gfortran.dg/pr65045.f90: Likewise.
+	* gfortran.dg/pr69497.f90: Likewise.
+	* gfortran.dg/submodule_21.f08: Likewise.
+	* gfortran.dg/tab_continuation.f: Likewise.
+	* gfortran.dg/typebound_proc_2.f90: Likewise.
+	* gfortran.dg/warnings_are_errors_1.f90: Likewise.
+
+2021-10-22  Tobias Burnus  <tobias@codesourcery.com>
+
+	PR fortran/92621
+	* gfortran.dg/bind-c-intent-out-2.f90: New test.
+
+2021-10-22  José Rui Faustino de Sousa  <jrfsousa@gmail.com>
+	    Sandra Loosemore  <sandra@codesourcery.com>
+
+	PR fortran/94289
+	* gfortran.dg/PR94289.f90: New.
+
+2021-10-22  José Rui Faustino de Sousa  <jrfsousa@gmail.com>
+	    Sandra Loosemore  <sandra@codesourcery.com>
+
+	PR fortran/100906
+	* gfortran.dg/PR100906.f90: New.
+	* gfortran.dg/PR100906.c: New.
+
+2021-10-22  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/102893
+	* gcc.dg/tree-ssa/ssa-dce-9.c: New testcase.
+
+2021-10-22  Richard Biener  <rguenther@suse.de>
+
+	PR bootstrap/102681
+	* gcc.dg/tree-ssa/ssa-fre-97.c: New testcase.
+	* gcc.dg/ubsan/pr81981.c: XFAIL one case.
+
+2021-10-21  Uroš Bizjak  <ubizjak@gmail.com>
+
+	PR testsuite/102840
+	* gcc.target/i386/pr22076.c: Adjust to avoid compile time optimization.
+
+2021-10-21  Hongyu Wang  <hongyu.wang@intel.com>
+
+	* gcc.target/i386/avx512fp16-13.c: Adjust scan-assembler for
+	xmm/ymm load/store.
+
+2021-10-21  Martin Liska  <mliska@suse.cz>
+
+	PR debug/102585
+	PR bootstrap/102766
+	* gcc.dg/pr102585.c: New test.
+
+2021-10-21  Martin Jambor  <mjambor@suse.cz>
+
+	PR tree-optimization/102505
+	* g++.dg/torture/pr102505.C: New test.
+
+2021-10-21  Jakub Jelinek  <jakub@redhat.com>
+
+	* gfortran.dg/gomp/strictly-structured-block-1.f90: Use call do_work
+	instead of x = x + 1 in places where the latter could be a data race.
+	* gfortran.dg/gomp/strictly-structured-block-2.f90: Likewise.
+	* gfortran.dg/gomp/strictly-structured-block-3.f90: Likewise.
+
+2021-10-21  Hongyu Wang  <hongyu.wang@intel.com>
+
+	PR target/102812
+	* gcc.target/i386/pr102812.c: New test.
+
+2021-10-21  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/64888
+	* c-c++-common/ubsan/pr64888.c: New test.
+
+2021-10-21  Andrew Pinski  <apinski@marvell.com>
+
+	* gcc.dg/pr36902.c: Move buf to be a non-static variable.
+
+2021-10-21  Richard Biener  <rguenther@suse.de>
+
+	PR testsuite/102861
+	* gcc.dg/vect/bb-slp-16.c: Revert previous change, scan
+	the vect dump instead.
+
+2021-10-21  Chung-Lin Tang  <cltang@codesourcery.com>
+
+	* gfortran.dg/gomp/cancel-1.f90: Adjust testcase.
+	* gfortran.dg/gomp/nesting-3.f90: Adjust testcase.
+	* gfortran.dg/gomp/strictly-structured-block-1.f90: New test.
+	* gfortran.dg/gomp/strictly-structured-block-2.f90: New test.
+	* gfortran.dg/gomp/strictly-structured-block-3.f90: New test.
+
+2021-10-21  Sandra Loosemore  <sandra@codesourcery.com>
+
+	PR fortran/94070
+	* gfortran.dg/c-interop/shape-bindc.f90: New test.
+	* gfortran.dg/c-interop/shape-poly.f90: New test.
+	* gfortran.dg/c-interop/size-bindc.f90: New test.
+	* gfortran.dg/c-interop/size-poly.f90: New test.
+	* gfortran.dg/c-interop/ubound-bindc.f90: New test.
+	* gfortran.dg/c-interop/ubound-poly.f90: New test.
+
 2021-10-20  Tamar Christina  <tamar.christina@arm.com>
 
 	* gcc.target/aarch64/mvn-cmeq0-1.c: New test.
diff --git a/gcc/testsuite/c-c++-common/Wno-attributes-1.c b/gcc/testsuite/c-c++-common/Wno-attributes-1.c
new file mode 100644
index 0000000..aac1a69
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wno-attributes-1.c
@@ -0,0 +1,55 @@
+/* PR c++/101940 */
+/* { dg-do compile } */
+/* { dg-additional-options "-std=c++11" { target c++ } } */
+/* { dg-additional-options "-Wno-attributes=company::,yoyodyne::attr" } */
+/* { dg-additional-options "-Wno-attributes=c1::attr,c1::attr,c1::__attr__" } */
+/* { dg-additional-options "-Wno-attributes=c2::,c2::attr" } */
+/* { dg-additional-options "-Wno-attributes=c3::attr,c3::" } */
+/* { dg-additional-options "-Wno-attributes=x::" } */
+/* { dg-additional-options "-Wno-attributes=yoyodyne::attr_new" } */
+/* { dg-additional-options "-Wno-attributes=c4::__attr__" } */
+/* { dg-additional-options "-Wno-attributes=c5::attr" } */
+/* { dg-additional-options "-Wno-attributes=__c6__::attr" } */
+
+[[company::attr]] void f1();
+[[company::attr2]] void f2();
+
+[[yoyodyne::attr]] void f3();
+[[yoyodyne::__attr__]] void f3__();
+[[yoyodyne::attrx]] void f4(); /* { dg-warning "ignored" } */
+[[yoyodyne::__attrx__]] void f4__(); /* { dg-warning "ignored" } */
+
+[[c1::attr]] void f5();
+
+[[c2::attr]] void f6();
+[[c2::attrx]] void f7();
+[[c2::__attr__]] void f6__();
+[[c2::__attrx__]] void f7__();
+
+[[c3::attr]] void f8();
+[[c3::attrx]] void f9();
+
+[[x::x]] void f10();
+
+[[yoyodyne::attr_new]] void f11();
+[[yoyodyne::__attr_new__]] void f11__();
+[[yoyodyne::attr_mew]] void f12(); /* { dg-warning "ignored" } */
+[[yoyodyne::__attr_mew__]] void f12__(); /* { dg-warning "ignored" } */
+
+[[c4::attr]] void f13();
+[[c4::__attr__]] void f13__();
+[[c4::attrx]] void f14(); /* { dg-warning "ignored" } */
+
+[[c5::attr]] void f15();
+[[c5::__attr__]] void f15__();
+[[__c5__::attr]] void __f15();
+[[__c5__::__attr__]] void __f15__();
+[[c5::attrx]] void f15x(); /* { dg-warning "ignored" } */
+[[__c5__::attrx]] void f15x(); /* { dg-warning "ignored" } */
+
+[[c6::attr]] void f16();
+[[c6::__attr__]] void f16__();
+[[__c6__::attr]] void __f16();
+[[__c6__::__attr__]] void __f16__();
+[[c6::attrx]] void f16x(); /* { dg-warning "ignored" } */
+[[__c6__::attrx]] void f16x(); /* { dg-warning "ignored" } */
diff --git a/gcc/testsuite/c-c++-common/Wno-attributes-2.c b/gcc/testsuite/c-c++-common/Wno-attributes-2.c
new file mode 100644
index 0000000..4307c74
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wno-attributes-2.c
@@ -0,0 +1,56 @@
+/* PR c++/101940 */
+/* { dg-do compile } */
+/* { dg-additional-options "-std=c++11" { target c++ } } */
+
+#pragma GCC diagnostic ignored_attributes "company::,yoyodyne::attr"
+#pragma GCC diagnostic ignored_attributes "c1::attr,c1::attr,c1::__attr__"
+#pragma GCC diagnostic ignored_attributes "c2::,c2::attr"
+#pragma GCC diagnostic ignored_attributes "c3::attr,c3::"
+#pragma GCC diagnostic ignored_attributes "x::"
+#pragma GCC diagnostic ignored_attributes "yoyodyne::attr_new"
+#pragma GCC diagnostic ignored_attributes "c4::__attr__"
+#pragma GCC diagnostic ignored_attributes "c5::attr"
+#pragma GCC diagnostic ignored_attributes "__c6__::attr"
+
+[[company::attr]] void f1();
+[[company::attr2]] void f2();
+
+[[yoyodyne::attr]] void f3();
+[[yoyodyne::__attr__]] void f3__();
+[[yoyodyne::attrx]] void f4(); /* { dg-warning "ignored" } */
+[[yoyodyne::__attrx__]] void f4__(); /* { dg-warning "ignored" } */
+
+[[c1::attr]] void f5();
+
+[[c2::attr]] void f6();
+[[c2::attrx]] void f7();
+[[c2::__attr__]] void f6__();
+[[c2::__attrx__]] void f7__();
+
+[[c3::attr]] void f8();
+[[c3::attrx]] void f9();
+
+[[x::x]] void f10();
+
+[[yoyodyne::attr_new]] void f11();
+[[yoyodyne::__attr_new__]] void f11__();
+[[yoyodyne::attr_mew]] void f12(); /* { dg-warning "ignored" } */
+[[yoyodyne::__attr_mew__]] void f12__(); /* { dg-warning "ignored" } */
+
+[[c4::attr]] void f13();
+[[c4::__attr__]] void f13__();
+[[c4::attrx]] void f14(); /* { dg-warning "ignored" } */
+
+[[c5::attr]] void f15();
+[[c5::__attr__]] void f15__();
+[[__c5__::attr]] void __f15();
+[[__c5__::__attr__]] void __f15__();
+[[c5::attrx]] void f15x(); /* { dg-warning "ignored" } */
+[[__c5__::attrx]] void f15x(); /* { dg-warning "ignored" } */
+
+[[c6::attr]] void f16();
+[[c6::__attr__]] void f16__();
+[[__c6__::attr]] void __f16();
+[[__c6__::__attr__]] void __f16__();
+[[c6::attrx]] void f16x(); /* { dg-warning "ignored" } */
+[[__c6__::attrx]] void f16x(); /* { dg-warning "ignored" } */
diff --git a/gcc/testsuite/c-c++-common/Wno-attributes-3.c b/gcc/testsuite/c-c++-common/Wno-attributes-3.c
new file mode 100644
index 0000000..64370f2
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wno-attributes-3.c
@@ -0,0 +1,9 @@
+/* PR c++/101940 */
+/* { dg-do compile } */
+/* { dg-additional-options "-Wno-attributes=___::" } */
+/* { dg-additional-options "-Wno-attributes=c::____" } */
+/* { dg-additional-options "-Wno-attributes=____::____" } */
+/* { dg-additional-options "-Wno-attributes=c@::attr" } */
+/* { dg-additional-options "-Wno-attributes=c2::@tr" } */
+
+/* { dg-error "wrong argument to ignored attributes" "" { target *-*-* } 0 } */
diff --git a/gcc/testsuite/c-c++-common/Wstringop-overflow-2.c b/gcc/testsuite/c-c++-common/Wstringop-overflow-2.c
index ca38bda..e580261 100644
--- a/gcc/testsuite/c-c++-common/Wstringop-overflow-2.c
+++ b/gcc/testsuite/c-c++-common/Wstringop-overflow-2.c
@@ -190,7 +190,7 @@ void ga1__ (void)
   struct A1 a = { 1 };
   a.a[0] = 0;
   a.a[1] = 1;                    // { dg-warning "\\\[-Wstringop-overflow" }
-  a.a[2] = 2;                    // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v2qi_store } } }
+  a.a[2] = 2;                    // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v2qi_store_align } } }
   sink (&a);
 }
 
@@ -207,7 +207,7 @@ void ga1_0_ (void)
   struct A1 a = { 1, { } };
   a.a[0] = 0;
   a.a[1] = 1;                   // { dg-warning "\\\[-Wstringop-overflow" }
-  a.a[2] = 2;                   // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v2qi_store } } }
+  a.a[2] = 2;                   // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v2qi_store_align } } }
   sink (&a);
 }
 
@@ -221,10 +221,10 @@ void ga1_1 (void)
   a1_1.a[1] = 1;                // { dg-warning "\\\[-Wstringop-overflow" }
   a1_1.a[2] = 2;                // { dg-warning "\\\[-Wstringop-overflow" }
 
-  struct A1 a = { 0, { 1 } };   // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v4qi_store } } }
+  struct A1 a = { 0, { 1 } };   // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v4qi_store_align } } }
   a.a[0] = 0;
-  a.a[1] = 1;                   // { dg-warning "\\\[-Wstringop-overflow" "" { xfail { vect_slp_v4qi_store } } }
-  a.a[2] = 2;                   // { dg-warning "\\\[-Wstringop-overflow" "" { xfail { vect_slp_v4qi_store } } }
+  a.a[1] = 1;                   // { dg-warning "\\\[-Wstringop-overflow" "" { xfail { vect_slp_v4qi_store_align } } }
+  a.a[2] = 2;                   // { dg-warning "\\\[-Wstringop-overflow" "" { xfail { vect_slp_v4qi_store_align } } }
   sink (&a);
 }
 
@@ -289,7 +289,7 @@ void ga1i__ (void)
   struct A1i a = { 0 };
   a.a[0] = 0;
   a.a[1] = 1;                    // { dg-warning "\\\[-Wstringop-overflow" }
-  a.a[2] = 2;                    // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v2qi_store } } }
+  a.a[2] = 2;                    // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v2qi_store_align } } }
   sink (&a);
 }
 
@@ -306,7 +306,7 @@ void ga1i_0_ (void)
   struct A1 a = { 0, { } };
   a.a[0] = 0;
   a.a[1] = 1;                   // { dg-warning "\\\[-Wstringop-overflow" }
-  a.a[2] = 2;                   // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v2qi_store } } }
+  a.a[2] = 2;                   // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v2qi_store_align } } }
   sink (&a);
 }
 
@@ -320,10 +320,10 @@ void ga1i_1 (void)
   a1i_1.a[1] = 1;               // { dg-warning "\\\[-Wstringop-overflow" }
   a1i_1.a[2] = 2;               // { dg-warning "\\\[-Wstringop-overflow" }
 
-  struct A1 a = { 0, { 1 } };   // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { target { vect_slp_v4qi_store } } }
+  struct A1 a = { 0, { 1 } };   // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { target { vect_slp_v4qi_store_align } } }
   a.a[0] = 1;
-  a.a[1] = 2;                   // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v4qi_store } } }
-  a.a[2] = 3;                   // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v4qi_store } } }
+  a.a[1] = 2;                   // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v4qi_store_align } } }
+  a.a[2] = 3;                   // { dg-warning "\\\[-Wstringop-overflow" "pr102462" { xfail { vect_slp_v4qi_store_align } } }
   sink (&a);
 }
 
diff --git a/gcc/testsuite/c-c++-common/asan/null-deref-1.c b/gcc/testsuite/c-c++-common/asan/null-deref-1.c
index bae016d..c967b29 100644
--- a/gcc/testsuite/c-c++-common/asan/null-deref-1.c
+++ b/gcc/testsuite/c-c++-common/asan/null-deref-1.c
@@ -1,5 +1,5 @@
 /* { dg-do run } */
-/* { dg-options "-fno-omit-frame-pointer -fno-shrink-wrap" } */
+/* { dg-options "-fno-omit-frame-pointer -fno-shrink-wrap -fno-ipa-modref" } */
 /* { dg-additional-options "-mno-omit-leaf-frame-pointer" { target { i?86-*-* x86_64-*-* } } } */
 /* { dg-shouldfail "asan" } */
 
diff --git a/gcc/testsuite/c-c++-common/diagnostic-format-json-1.c b/gcc/testsuite/c-c++-common/diagnostic-format-json-1.c
index 740becb..af57eb6 100644
--- a/gcc/testsuite/c-c++-common/diagnostic-format-json-1.c
+++ b/gcc/testsuite/c-c++-common/diagnostic-format-json-1.c
@@ -9,6 +9,7 @@
 
 /* { dg-regexp "\"kind\": \"error\"" } */
 /* { dg-regexp "\"column-origin\": 1" } */
+/* { dg-regexp "\"escape-source\": false" } */
 /* { dg-regexp "\"message\": \"#error message\"" } */
 
 /* { dg-regexp "\"caret\": \{" } */
diff --git a/gcc/testsuite/c-c++-common/diagnostic-format-json-2.c b/gcc/testsuite/c-c++-common/diagnostic-format-json-2.c
index 2f24a6c..edb802e 100644
--- a/gcc/testsuite/c-c++-common/diagnostic-format-json-2.c
+++ b/gcc/testsuite/c-c++-common/diagnostic-format-json-2.c
@@ -9,6 +9,7 @@
 
 /* { dg-regexp "\"kind\": \"warning\"" } */
 /* { dg-regexp "\"column-origin\": 1" } */
+/* { dg-regexp "\"escape-source\": false" } */
 /* { dg-regexp "\"message\": \"#warning message\"" } */
 /* { dg-regexp "\"option\": \"-Wcpp\"" } */
 /* { dg-regexp "\"option_url\": \"https:\[^\n\r\"\]*#index-Wcpp\"" } */
diff --git a/gcc/testsuite/c-c++-common/diagnostic-format-json-3.c b/gcc/testsuite/c-c++-common/diagnostic-format-json-3.c
index afe96a9..bb7b8dc 100644
--- a/gcc/testsuite/c-c++-common/diagnostic-format-json-3.c
+++ b/gcc/testsuite/c-c++-common/diagnostic-format-json-3.c
@@ -9,6 +9,7 @@
 
 /* { dg-regexp "\"kind\": \"error\"" } */
 /* { dg-regexp "\"column-origin\": 1" } */
+/* { dg-regexp "\"escape-source\": false" } */
 /* { dg-regexp "\"message\": \"#warning message\"" } */
 /* { dg-regexp "\"option\": \"-Werror=cpp\"" } */
 /* { dg-regexp "\"option_url\": \"https:\[^\n\r\"\]*#index-Wcpp\"" } */
diff --git a/gcc/testsuite/c-c++-common/diagnostic-format-json-4.c b/gcc/testsuite/c-c++-common/diagnostic-format-json-4.c
index ae51091..8ac9072 100644
--- a/gcc/testsuite/c-c++-common/diagnostic-format-json-4.c
+++ b/gcc/testsuite/c-c++-common/diagnostic-format-json-4.c
@@ -19,6 +19,7 @@ int test (void)
 
 /* { dg-regexp "\"kind\": \"note\"" } */
 /* { dg-regexp "\"message\": \"...this statement, but the latter is misleadingly indented as if it were guarded by the 'if'\"" } */
+/* { dg-regexp "\"escape-source\": false" } */
 
 /* { dg-regexp "\"caret\": \{" } */
 /* { dg-regexp "\"file\": \"\[^\n\r\"\]*diagnostic-format-json-4.c\"" } */
@@ -39,6 +40,7 @@ int test (void)
 /* { dg-regexp "\"kind\": \"warning\"" } */
 /* { dg-regexp "\"column-origin\": 1" } */
 /* { dg-regexp "\"message\": \"this 'if' clause does not guard...\"" } */
+/* { dg-regexp "\"escape-source\": false" } */
 /* { dg-regexp "\"option\": \"-Wmisleading-indentation\"" } */
 /* { dg-regexp "\"option_url\": \"https:\[^\n\r\"\]*#index-Wmisleading-indentation\"" } */
 
diff --git a/gcc/testsuite/c-c++-common/diagnostic-format-json-5.c b/gcc/testsuite/c-c++-common/diagnostic-format-json-5.c
index e0e9ce4..8d2eb0c 100644
--- a/gcc/testsuite/c-c++-common/diagnostic-format-json-5.c
+++ b/gcc/testsuite/c-c++-common/diagnostic-format-json-5.c
@@ -14,6 +14,7 @@ int test (struct s *ptr)
 
 /* { dg-regexp "\"kind\": \"error\"" } */
 /* { dg-regexp "\"column-origin\": 1" } */
+/* { dg-regexp "\"escape-source\": false" } */
 /* { dg-regexp "\"message\": \".*\"" } */
 
 /* Verify fix-it hints.  */
diff --git a/gcc/testsuite/c-c++-common/goacc/combined-reduction.c b/gcc/testsuite/c-c++-common/goacc/combined-reduction.c
index ecf23f5..74ab05b 100644
--- a/gcc/testsuite/c-c++-common/goacc/combined-reduction.c
+++ b/gcc/testsuite/c-c++-common/goacc/combined-reduction.c
@@ -23,7 +23,7 @@ main ()
   return 0;
 }
 
-/* { dg-final { scan-tree-dump-times "omp target oacc_parallel reduction.+:v1. map.tofrom:v1" 1 "gimple" } } */
+/* { dg-final { scan-tree-dump-times "omp target oacc_parallel reduction.+:v1. firstprivate.n. map.tofrom:v1" 1 "gimple" } } */
 /* { dg-final { scan-tree-dump-times "acc loop reduction.+:v1. private.i." 1 "gimple" } } */
 /* { dg-final { scan-tree-dump-times "omp target oacc_kernels map.force_tofrom:n .len: 4.. map.force_tofrom:v1 .len: 4.." 1 "gimple" } } */
 /* { dg-final { scan-tree-dump-times "acc loop reduction.+:v1. private.i." 1 "gimple" } } */
diff --git a/gcc/testsuite/c-c++-common/goacc/firstprivate-mappings-1.c b/gcc/testsuite/c-c++-common/goacc/firstprivate-mappings-1.c
index 7987bea..5134ef6 100644
--- a/gcc/testsuite/c-c++-common/goacc/firstprivate-mappings-1.c
+++ b/gcc/testsuite/c-c++-common/goacc/firstprivate-mappings-1.c
@@ -419,12 +419,7 @@ vla (int array_li)
   copyout (array_so)
   /* The gimplifier has created an implicit 'firstprivate' clause for the array
      length.
-     { dg-final { scan-tree-dump {(?n)#pragma omp target oacc_parallel map\(from:array_so \[len: 4\]\) firstprivate\(array_li.[0-9]+\)} omplower { target { ! c++ } } } }
-     { dg-final { scan-tree-dump {(?n)#pragma omp target oacc_parallel map\(from:array_so \[len: 4\]\) firstprivate\(} omplower { target { c++ } } } }
-     (C++ computes an intermediate value, so can't scan for 'firstprivate(array_li)'.)  */
-  /* For C, non-LP64, the gimplifier has also created a mapping for the array
-     itself; PR90859.
-     { dg-final { scan-tree-dump {(?n)#pragma omp target oacc_parallel map\(from:array_so \[len: 4\]\) firstprivate\(array_li.[0-9]+\) map\(tofrom:\(\*array.[0-9]+\) \[len: D\.[0-9]+\]\) map\(firstprivate:array \[pointer assign, bias: 0\]\) \[} omplower { target { c && { ! lp64 } } } } } */
+     { dg-final { scan-tree-dump {(?n)#pragma omp target oacc_parallel firstprivate\(array_li.[0-9]+\) map\(from:array_so \[len: 4\]\) \[} omplower } } */
   {
     array_so = sizeof array;
   }
diff --git a/gcc/testsuite/c-c++-common/goacc/mdc-1.c b/gcc/testsuite/c-c++-common/goacc/mdc-1.c
index c2b8dc6..0a123be 100644
--- a/gcc/testsuite/c-c++-common/goacc/mdc-1.c
+++ b/gcc/testsuite/c-c++-common/goacc/mdc-1.c
@@ -45,7 +45,7 @@ t1 ()
 
 /* { dg-final { scan-tree-dump-times "pragma omp target oacc_enter_data map.to:s .len: 32.." 1 "omplower" } } */
 /* { dg-final { scan-tree-dump-times "pragma omp target oacc_data map.tofrom:.z .len: 40.. map.struct:s .len: 1.. map.alloc:s.a .len: 8.. map.tofrom:._1 .len: 40.. map.attach:s.a .bias: 0.." 1 "omplower" } } */
-/* { dg-final { scan-tree-dump-times "pragma omp target oacc_parallel map.attach:s.e .bias: 0.. map.tofrom:s .len: 32" 1 "omplower" } } */
+/* { dg-final { scan-tree-dump-times "pragma omp target oacc_parallel map.tofrom:s .len: 32.. map.attach:s.e .bias: 0.." 1 "omplower" } } */
 /* { dg-final { scan-tree-dump-times "pragma omp target oacc_enter_data map.attach:a .bias: 0.." 1 "omplower" } } */
 /* { dg-final { scan-tree-dump-times "pragma omp target oacc_exit_data map.detach:a .bias: 0.." 1 "omplower" } } */
 /* { dg-final { scan-tree-dump-times "pragma omp target oacc_enter_data map.to:a .len: 8.." 1 "omplower" } } */
diff --git a/gcc/testsuite/c-c++-common/gomp/atomic-4.c b/gcc/testsuite/c-c++-common/gomp/atomic-4.c
index 7f27370..5dd18d1 100644
--- a/gcc/testsuite/c-c++-common/gomp/atomic-4.c
+++ b/gcc/testsuite/c-c++-common/gomp/atomic-4.c
@@ -8,7 +8,7 @@ int *bar(void);
 void f1(void)
 {
   #pragma omp atomic
-    a[4] += 1;
+    a[3] += 1;
   #pragma omp atomic
     *p += 1;
   #pragma omp atomic
diff --git a/gcc/testsuite/c-c++-common/gomp/clauses-1.c b/gcc/testsuite/c-c++-common/gomp/clauses-1.c
index 742132f..3ff49e0 100644
--- a/gcc/testsuite/c-c++-common/gomp/clauses-1.c
+++ b/gcc/testsuite/c-c++-common/gomp/clauses-1.c
@@ -164,7 +164,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ll++;
   #pragma omp target teams \
     device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) \
-    shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) nowait depend(inout: dd[0]) \
+    shared(s) default(shared) reduction(+:r) num_teams(nte - 1:nte) thread_limit(tl) nowait depend(inout: dd[0]) \
     allocate (omp_default_mem_alloc:f) in_reduction(+:r2)
     ;
   #pragma omp target teams distribute \
@@ -175,7 +175,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ;
   #pragma omp target teams distribute parallel for \
     device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) \
-    shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) \
+    shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) \
     collapse(1) dist_schedule(static, 16) \
     if (parallel: i2) num_threads (nth) proc_bind(spread) \
     lastprivate (l) schedule(static, 4) nowait depend(inout: dd[0]) order(concurrent) \
@@ -194,7 +194,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ll++;
   #pragma omp target teams distribute simd \
     device(d) map (tofrom: m) if (i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) \
-    shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) \
+    shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) \
     collapse(1) dist_schedule(static, 16) order(concurrent) \
     safelen(8) simdlen(4) aligned(q: 32) nowait depend(inout: dd[0]) nontemporal(ntm) \
     allocate (omp_default_mem_alloc:f) in_reduction(+:r2)
@@ -236,7 +236,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ;
   #pragma omp target
   #pragma omp teams distribute parallel for \
-    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) \
+    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) \
     collapse(1) dist_schedule(static, 16) \
     if (parallel: i2) num_threads (nth) proc_bind(spread) \
     lastprivate (l) schedule(static, 4) order(concurrent) allocate (omp_default_mem_alloc: f)
@@ -254,7 +254,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ll++;
   #pragma omp target
   #pragma omp teams distribute simd \
-    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) \
+    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) \
     collapse(1) dist_schedule(static, 16) order(concurrent) \
     safelen(8) simdlen(4) aligned(q: 32) if(i3) nontemporal(ntm) \
     allocate (omp_default_mem_alloc: f)
@@ -268,7 +268,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   for (int i = 0; i < 64; i++)
     ll++;
   #pragma omp teams distribute parallel for \
-    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) \
+    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) \
     collapse(1) dist_schedule(static, 16) order(concurrent) \
     if (parallel: i2) num_threads (nth) proc_bind(spread) \
     lastprivate (l) schedule(static, 4) allocate (f)
@@ -284,7 +284,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   for (int i = 0; i < 64; i++)
     ll++;
   #pragma omp teams distribute parallel for simd \
-    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) \
+    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) \
     collapse(1) dist_schedule(static, 16) \
     if (parallel: i2) num_threads (nth) proc_bind(spread) \
     lastprivate (l) schedule(static, 4) order(concurrent) \
@@ -417,7 +417,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   for (l = 0; l < 64; l++)
     ll++;
   #pragma omp teams loop \
-    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) \
+    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) \
     collapse(1) lastprivate (l) bind(teams) allocate (f)
   for (l = 0; l < 64; ++l)
     ;
@@ -442,7 +442,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ;
   #pragma omp target teams loop \
     device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) \
-    shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) nowait depend(inout: dd[0]) \
+    shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) nowait depend(inout: dd[0]) \
     lastprivate (l) bind(teams) collapse(1) \
     allocate (omp_default_mem_alloc: f) in_reduction(+:r2)
   for (l = 0; l < 64; ++l)
diff --git a/gcc/testsuite/c-c++-common/gomp/loop-10.c b/gcc/testsuite/c-c++-common/gomp/loop-10.c
new file mode 100644
index 0000000..1229e61
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/gomp/loop-10.c
@@ -0,0 +1,35 @@
+int a, b;
+#pragma omp threadprivate (a, b)
+
+void
+foo (void)
+{
+  #pragma omp for				/* { dg-error "threadprivate iteration variable 'a'" } */
+  for (a = 0; a < 32; a++)
+    ;
+  #pragma omp parallel for collapse(2)		/* { dg-error "threadprivate iteration variable 'a'" "" { target c } } */
+  for (a = 0; a < 32; a++)			/* { dg-error "threadprivate iteration variable 'b'" "" { target c } .-1 } */
+    for (b = 0; b < 32; b++)			/* { dg-error "threadprivate iteration variable 'a'" "" { target c++ } .-1 } */
+      ;						/* { dg-error "threadprivate iteration variable 'b'" "" { target c++ } .-2 } */
+  #pragma omp simd				/* { dg-error "threadprivate iteration variable 'a'" } */
+  for (a = 0; a < 32; a++)
+    ;
+  #pragma omp taskloop				/* { dg-error "threadprivate iteration variable 'a'" } */
+  for (a = 0; a < 32; a++)
+    ;
+  #pragma omp loop bind(thread)			/* { dg-error "threadprivate iteration variable 'a'" } */
+  for (a = 0; a < 32; a++)
+    ;
+}
+
+void
+bar (void)
+{
+  #pragma omp distribute collapse(2)		/* { dg-error "threadprivate iteration variable 'a'" } */
+  for (a = 0; a < 32; a++)			/* { dg-error "threadprivate iteration variable 'b'" "" { target *-*-* } .-1 } */
+    for (b = 0; b < a; b++)
+      ;
+  #pragma omp distribute parallel for simd	/* { dg-error "threadprivate iteration variable 'a'" "" { target c } } */
+  for (a = 0; a < 32; a++)			/* { dg-error "threadprivate iteration variable 'a'" "" { target c++ } } */
+    ;
+}
diff --git a/gcc/testsuite/c-c++-common/gomp/loop-11.c b/gcc/testsuite/c-c++-common/gomp/loop-11.c
new file mode 100644
index 0000000..5076e65
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/gomp/loop-11.c
@@ -0,0 +1,5 @@
+/* PR tree-optimization/103208 */
+/* { dg-do compile } */
+/* { dg-additional-options "-O2 -fwrapv" } */
+
+#include "loop-8.c"
diff --git a/gcc/testsuite/c-c++-common/gomp/loop-8.c b/gcc/testsuite/c-c++-common/gomp/loop-8.c
new file mode 100644
index 0000000..d66bbcd
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/gomp/loop-8.c
@@ -0,0 +1,10 @@
+void
+foo (void)
+{
+  int a[1024];
+  int *p, *q;
+  #pragma omp parallel for collapse(2)
+  for (p = &a[0]; p < &a[512]; p++)
+    for (q = p + 64; q < p + 128; q++)
+      ;
+}
diff --git a/gcc/testsuite/c-c++-common/gomp/loop-9.c b/gcc/testsuite/c-c++-common/gomp/loop-9.c
new file mode 100644
index 0000000..a64ad98
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/gomp/loop-9.c
@@ -0,0 +1,38 @@
+int *qux (int *);
+
+void
+foo (void)
+{
+  int a[1024];
+  int *p;
+  short *q;
+  __PTRDIFF_TYPE__ r;
+  #pragma omp parallel for collapse(2)
+  for (p = &a[0]; p < &a[512]; p++)
+    for (q = (short *) p + 64; q < (short *) p + 128; q++)	/* { dg-error "outer iteration variable 'p' used in initializer expression has type other than 'short int ?\\\*'" } */
+      ;
+  #pragma omp parallel for collapse(2)
+  for (p = &a[0]; p < &a[512]; p++)
+    for (r = &a[32] - p; r < 32; r++)				/* { dg-error "initializer expression refers to iteration variable 'p'" } */
+      ;
+  #pragma omp parallel for collapse(2)
+  for (r = 0; r < 64; r++)
+    for (p = &a[0] + r; p < &a[32] + 3 * r; p++)		/* { dg-error "initializer expression refers to iteration variable 'r'" } */
+      ;
+}
+
+void
+bar (void)
+{
+  int a[1024];
+  int *p, *q, *r;
+  #pragma omp parallel for collapse(2)
+  for (p = &a[0]; p < &a[512]; p++)
+    for (q = p + (&a[16] - qux (p)); q < &a[32]; q++)		/* { dg-error "initializer expression refers to iteration variable 'p'" } */
+      ;
+  #pragma omp parallel for collapse(3)
+  for (p = &a[0]; p < &a[512]; p++)
+    for (q = &a[0]; q < &a[512]; q++)
+      for (r = p; r < q + 32; r++)				/* { dg-error "two different outer iteration variables 'p' and 'q' used in a single loop" } */
+	;
+}
diff --git a/gcc/testsuite/c-c++-common/gomp/num-teams-1.c b/gcc/testsuite/c-c++-common/gomp/num-teams-1.c
new file mode 100644
index 0000000..50cad85
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/gomp/num-teams-1.c
@@ -0,0 +1,48 @@
+int fn (int);
+
+void
+foo (void)
+{
+  #pragma omp teams num_teams (4 : 6)
+  ;
+  #pragma omp teams num_teams (7)
+  ;
+}
+
+void
+bar (void)
+{
+  #pragma omp target teams num_teams (5 : 19)
+  ;
+  #pragma omp target teams num_teams (21)
+  ;
+}
+
+void
+baz (void)
+{
+  #pragma omp teams num_teams (fn (1) : fn (2))
+  ;
+  #pragma omp teams num_teams (fn (3))
+  ;
+}
+
+void
+qux (void)
+{
+  #pragma omp target teams num_teams (fn (4) : fn (5))
+  ;
+  #pragma omp target teams num_teams (fn (6))
+  ;
+}
+
+void
+corge (void)
+{
+  #pragma omp target
+  #pragma omp teams num_teams (fn (7) : fn (8))
+  ;
+  #pragma omp target
+  #pragma omp teams num_teams (fn (9))
+  ;
+}
diff --git a/gcc/testsuite/c-c++-common/gomp/num-teams-2.c b/gcc/testsuite/c-c++-common/gomp/num-teams-2.c
new file mode 100644
index 0000000..242b994
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/gomp/num-teams-2.c
@@ -0,0 +1,27 @@
+int fn (int);
+
+void
+foo (int i)
+{
+  #pragma omp teams num_teams (6 : 4)		/* { dg-warning "'num_teams' lower bound '6' bigger than upper bound '4'" } */
+  ;
+  #pragma omp teams num_teams (-7)		/* { dg-warning "'num_teams' value must be positive" } */
+  ;
+  #pragma omp teams num_teams (i : -7)		/* { dg-warning "'num_teams' value must be positive" } */
+  ;
+  #pragma omp teams num_teams (-7 : 8)		/* { dg-warning "'num_teams' value must be positive" } */
+  ;
+}
+
+void
+bar (int i)
+{
+  #pragma omp target teams num_teams (6 : 4)	/* { dg-warning "'num_teams' lower bound '6' bigger than upper bound '4'" } */
+  ;
+  #pragma omp target teams num_teams (-7)	/* { dg-warning "'num_teams' value must be positive" } */
+  ;
+  #pragma omp target teams num_teams (i : -7)	/* { dg-warning "'num_teams' value must be positive" } */
+  ;
+  #pragma omp target teams num_teams (-7 : 8)	/* { dg-warning "'num_teams' value must be positive" } */
+  ;
+}
diff --git a/gcc/testsuite/c-c++-common/gomp/pragma-1.c b/gcc/testsuite/c-c++-common/gomp/pragma-1.c
new file mode 100644
index 0000000..e330f17
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/gomp/pragma-1.c
@@ -0,0 +1,50 @@
+/* { dg-additional-options "-fdump-tree-original" }  */
+/* PR c++/51484  */
+
+#define TEST(T) { \
+  int fail = 0, trial; \
+  for (int trial = 0; trial < TRIALS && fail == 0; trial++) { \
+    _Pragma("omp target teams num_teams(1) thread_limit(1024)") \
+     {T} \
+  } \
+}
+
+#define TRIALS (1)
+#define N (1024*3)
+
+int main(void) {
+
+  double C[N], D[N];
+  double S[N];
+  double p[2];
+  int i; 
+  for (i = 0; i < N; i++)
+  {C[i] = 1; D[i] = i;}
+
+  int max_threads = 224;
+
+#define PARALLEL(X) TEST({ \
+_Pragma("omp parallel if(threads[0] > 1) num_threads(threads[0])") \
+{ \
+_Pragma("omp for ordered") \
+  X  \
+_Pragma("omp for schedule(auto) ordered") \
+  X  \
+} \
+})
+
+  for (int t = 0; t <= max_threads; t += max_threads) {
+    int threads[1]; threads[0] = t;
+    S[0] = 0;
+    PARALLEL(
+    for (int i = 0; i < N; i++) { \
+      _Pragma("omp ordered") \
+      S[0] += C[i] + D[i]; \
+    })
+  }
+  return 0;
+}
+
+/* On expansion, the _Pragma were wrongly placed, ensure the order is now correct: */
+/* { dg-final { scan-tree-dump "#pragma omp target.*#pragma omp teams num_teams\\(1\\) thread_limit\\(1024\\).*#pragma omp parallel num_threads\\(threads\\\[0\\\]\\) if\\(threads\\\[0\\\] > 1\\).*#pragma omp for ordered.*#pragma omp ordered.*#pragma omp for ordered schedule\\(auto\\).*#pragma omp ordered" "original" } } */
+
diff --git a/gcc/testsuite/c-c++-common/gomp/pragma-2.c b/gcc/testsuite/c-c++-common/gomp/pragma-2.c
new file mode 100644
index 0000000..5358f87
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/gomp/pragma-2.c
@@ -0,0 +1,50 @@
+/* { dg-additional-options "-fdump-tree-original -save-temps" }  */
+/* PR c++/51484  */
+
+#define TEST(T) { \
+  int fail = 0, trial; \
+  for (int trial = 0; trial < TRIALS && fail == 0; trial++) { \
+    _Pragma("omp target teams num_teams(1) thread_limit(1024)") \
+     {T} \
+  } \
+}
+
+#define TRIALS (1)
+#define N (1024*3)
+
+int main(void) {
+
+  double C[N], D[N];
+  double S[N];
+  double p[2];
+  int i; 
+  for (i = 0; i < N; i++)
+  {C[i] = 1; D[i] = i;}
+
+  int max_threads = 224;
+
+#define PARALLEL(X) TEST({ \
+_Pragma("omp parallel if(threads[0] > 1) num_threads(threads[0])") \
+{ \
+_Pragma("omp for ordered") \
+  X  \
+_Pragma("omp for schedule(auto) ordered") \
+  X  \
+} \
+})
+
+  for (int t = 0; t <= max_threads; t += max_threads) {
+    int threads[1]; threads[0] = t;
+    S[0] = 0;
+    PARALLEL(
+    for (int i = 0; i < N; i++) { \
+      _Pragma("omp ordered") \
+      S[0] += C[i] + D[i]; \
+    })
+  }
+  return 0;
+}
+
+/* On expansion, the _Pragma were wrongly placed, ensure the order is now correct: */
+/* { dg-final { scan-tree-dump "#pragma omp target.*#pragma omp teams num_teams\\(1\\) thread_limit\\(1024\\).*#pragma omp parallel num_threads\\(threads\\\[0\\\]\\) if\\(threads\\\[0\\\] > 1\\).*#pragma omp for ordered.*#pragma omp ordered.*#pragma omp for ordered schedule\\(auto\\).*#pragma omp ordered" "original" } } */
+
diff --git a/gcc/testsuite/c-c++-common/gomp/target-device-ancestor-3.c b/gcc/testsuite/c-c++-common/gomp/target-device-ancestor-3.c
index 5e3a478..ea6e5a0 100644
--- a/gcc/testsuite/c-c++-common/gomp/target-device-ancestor-3.c
+++ b/gcc/testsuite/c-c++-common/gomp/target-device-ancestor-3.c
@@ -3,6 +3,7 @@ extern "C" {
 #endif
 
 int omp_get_num_teams (void);
+int bar (void);
 
 #ifdef __cplusplus
 }
@@ -22,6 +23,7 @@ foo (void)
 
   #pragma omp target device (ancestor: 1)
     {
+      a = bar (); /* OK */
       a = omp_get_num_teams (); /* { dg-error "OpenMP runtime API call '\[^\n\r]*omp_get_num_teams\[^\n\r]*' in a region with 'device\\(ancestor\\)' clause" }  */
     }
 
diff --git a/gcc/testsuite/c-c++-common/gomp/target-implicit-map-1.c b/gcc/testsuite/c-c++-common/gomp/target-implicit-map-1.c
new file mode 100644
index 0000000..52944fd
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/gomp/target-implicit-map-1.c
@@ -0,0 +1,39 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-fdump-tree-gimple" } */
+#ifdef __cplusplus
+extern "C"
+#else
+extern
+#endif
+void abort (void);
+
+int
+main (void)
+{
+  #define N 5
+  int array[N][N];
+
+  for (int i = 0; i < N; i++)
+    {
+      #pragma omp target enter data map(alloc: array[i:1][0:N])
+
+      #pragma omp target
+      for (int j = 0; j < N; j++)
+	array[i][j] = i * 10 + j;
+
+      #pragma omp target exit data map(from: array[i:1][0:N])
+    }
+
+  for (int i = 0; i < N; i++)
+    for (int j = 0; j < N; j++)
+      if (array[i][j] != i + j)
+	abort ();
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump {#pragma omp target enter data map\(alloc:array\[[^]]+\]\[0\] \[len: [0-9]+\]\)} "gimple" } } */
+
+/* { dg-final { scan-tree-dump {#pragma omp target num_teams.* firstprivate\(i\) map\(tofrom:array \[len: [0-9]+\]\[implicit\]\)} "gimple" } } */
+
+/* { dg-final { scan-tree-dump {#pragma omp target exit data map\(from:array\[[^]]+\]\[0\] \[len: [0-9]+\]\)} "gimple" } } */
diff --git a/gcc/testsuite/c-c++-common/gomp/teams-3.c b/gcc/testsuite/c-c++-common/gomp/teams-3.c
new file mode 100644
index 0000000..7f8b47f
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/gomp/teams-3.c
@@ -0,0 +1,64 @@
+/* PR middle-end/102972 */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* From omp.h  */
+extern int omp_get_num_teams (void);
+extern void omp_set_num_teams (int);
+extern int omp_get_team_size (int);
+extern int omp_get_team_num (void);
+extern int omp_get_max_teams (void);
+extern void omp_set_teams_thread_limit (int);
+extern int omp_get_teams_thread_limit (void);
+extern int omp_is_initial_device (void);
+extern int omp_get_num_threads (void);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+
+void valid ()
+{
+  #pragma omp teams
+  {
+    #pragma omp distribute
+    for (int i = 0; i < 64; i++)
+      ;
+
+    int n = omp_get_num_teams ();
+    if (n >= omp_get_team_num ())
+      __builtin_abort ();
+
+    #pragma omp parallel for
+    for (int i = 0; i < 64; i++)
+      if (!omp_is_initial_device () || omp_get_num_threads () < 0)
+	__builtin_abort ();
+
+    #pragma omp loop
+    for (int i = 0; i < 64; i++)
+      ;
+  }
+}
+
+void invalid_nest ()
+{
+  #pragma omp teams
+  {
+    #pragma distribute parallel for simd
+    for (int i = 0; i < 64; i++)
+      ;
+
+    int n = 0;
+    n += omp_get_team_size (0);  /* { dg-error "OpenMP runtime API call '\[^\n\r]*omp_get_team_size\[^\n\r]*' strictly nested in a 'teams' region" }  */
+    n += omp_get_num_teams ();
+    n += omp_get_team_num ();
+    omp_set_num_teams (n);  /* { dg-error "OpenMP runtime API call '\[^\n\r]*omp_set_num_teams\[^\n\r]*' strictly nested in a 'teams' region" }  */
+    n += omp_get_max_teams ();  /* { dg-error "OpenMP runtime API call '\[^\n\r]*omp_get_max_teams\[^\n\r]*' strictly nested in a 'teams' region" }  */
+    n += omp_get_teams_thread_limit ();  /* { dg-error "OpenMP runtime API call '\[^\n\r]*omp_get_teams_thread_limit\[^\n\r]*' strictly nested in a 'teams' region" }  */
+    omp_set_teams_thread_limit (n);  /* { dg-error "OpenMP runtime API call '\[^\n\r]*omp_set_teams_thread_limit\[^\n\r]*' strictly nested in a 'teams' region" }  */
+  }
+}
diff --git a/gcc/testsuite/c-c++-common/pr102281.c b/gcc/testsuite/c-c++-common/pr102281.c
new file mode 100644
index 0000000..a961451
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/pr102281.c
@@ -0,0 +1,17 @@
+/* PR102281  */
+/* { dg-do compile } */
+/* { dg-options "-ftrivial-auto-var-init=zero -Wno-psabi" } */
+long long var1;
+float var2;
+typedef long long V __attribute__((__vector_size__(2 * sizeof(long long))));
+typedef float W __attribute__((__vector_size__(4 * sizeof(float)))); 
+
+V foo (void)
+{
+  return (V) {var1}; 
+}
+
+W bar (void)
+{
+  return (W) {var2};
+}
diff --git a/gcc/testsuite/c-c++-common/tm/inline-asm.c b/gcc/testsuite/c-c++-common/tm/inline-asm.c
index 7389260..1762668 100644
--- a/gcc/testsuite/c-c++-common/tm/inline-asm.c
+++ b/gcc/testsuite/c-c++-common/tm/inline-asm.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-fgnu-tm -O1" } */
+/* { dg-options "-fgnu-tm -O1 -fno-ipa-modref -fno-ipa-pure-const" } */
 
 static inline void
 inline_death ()
diff --git a/gcc/testsuite/c-c++-common/torture/builtin-convertvector-2.c b/gcc/testsuite/c-c++-common/torture/builtin-convertvector-2.c
new file mode 100644
index 0000000..d88f6a7
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/torture/builtin-convertvector-2.c
@@ -0,0 +1,26 @@
+/* { dg-do run } */
+/* PR target/101529 */
+
+typedef unsigned char __attribute__((__vector_size__ (1))) W;
+typedef unsigned char __attribute__((__vector_size__ (8))) V;
+typedef unsigned short __attribute__((__vector_size__ (16))) U;
+
+unsigned short us;
+
+/* aarch64 used to miscompile foo to just return 0. */
+W
+foo (unsigned char uc)
+{
+  V v = __builtin_convertvector ((U){ } >= us, V);
+  return __builtin_shufflevector ((W){ }, v, 4) & uc;
+}
+
+int
+main (void)
+{
+  W x = foo (5);
+  if (x[0] != 5)
+    __builtin_abort();
+  return 0;
+}
+
diff --git a/gcc/testsuite/c-c++-common/torture/builtin-shufflevector-2.c b/gcc/testsuite/c-c++-common/torture/builtin-shufflevector-2.c
new file mode 100644
index 0000000..7c4999e
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/torture/builtin-shufflevector-2.c
@@ -0,0 +1,26 @@
+/* { dg-do run}  */
+/* PR target/101529 */
+typedef unsigned char C;
+typedef unsigned char __attribute__((__vector_size__ (8))) V;
+typedef unsigned char __attribute__((__vector_size__ (32))) U;
+
+C c;
+
+/* aarch64 used to miscompile foo to just return a vector of 0s */
+V
+foo (V v)
+{
+  v |= __builtin_shufflevector (c * v, (U) (0 == (U){ }),
+				0, 1, 8, 32, 8, 20, 36, 36);
+  return v;
+}
+
+int
+main (void)
+{
+  V v = foo ((V) { });
+  for (unsigned i = 0; i < sizeof (v); i++)
+    if (v[i] != (i >= 2 ? 0xff : 0))
+      __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/c-c++-common/torture/harden-comp.c b/gcc/testsuite/c-c++-common/torture/harden-comp.c
new file mode 100644
index 0000000..1ee0b36
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/torture/harden-comp.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-fharden-compares -fdump-tree-hardcmp -ffat-lto-objects" } */
+
+int
+f (int i, int j)
+{
+  return i < j;
+}
+
+/* { dg-final { scan-tree-dump-times "Splitting block" 1 "hardcmp" } } */
+/* { dg-final { scan-tree-dump-times "Adding reversed compare" 1 "hardcmp" } } */
+/* { dg-final { scan-tree-dump-times "__builtin_trap" 1 "hardcmp" } } */
+/* { dg-final { scan-tree-dump-times "_\[0-9\]* = i_\[0-9\]*\[(\]D\[)\] < j_\[0-9\]*\[(\]D\[)\];" 1 "hardcmp" } } */
+/* { dg-final { scan-tree-dump-times "_\[0-9\]* = i_\[0-9\]* >= j_\[0-9\]*;" 1 "hardcmp" } } */
diff --git a/gcc/testsuite/c-c++-common/torture/harden-cond.c b/gcc/testsuite/c-c++-common/torture/harden-cond.c
new file mode 100644
index 0000000..86de8e15
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/torture/harden-cond.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-fharden-conditional-branches -fdump-tree-hardcbr -ffat-lto-objects" } */
+
+extern int f1 (void);
+extern int f2 (void);
+
+
+int
+f (int i, int j)
+{
+  return (i < j) ? f1 () : f2 ();
+}
+
+/* { dg-final { scan-tree-dump-times "Splitting edge" 2 "hardcbr" } } */
+/* { dg-final { scan-tree-dump-times "Adding reversed compare" 2 "hardcbr" } } */
+/* { dg-final { scan-tree-dump-times "__builtin_trap" 2 "hardcbr" } } */
+/* { dg-final { scan-tree-dump-times "if \[(\]i_\[0-9\]*\[(\]D\[)\] < j_\[0-9\]*\[(\]D\[)\]\[)\]" 1 "hardcbr" } } */
+/* { dg-final { scan-tree-dump-times "if \[(\]i_\[0-9\]* >= j_\[0-9\]*\[)\]" 2 "hardcbr" } } */
diff --git a/gcc/testsuite/c-c++-common/tsan/free_race.c b/gcc/testsuite/c-c++-common/tsan/free_race.c
index 258f7b7..390ec30 100644
--- a/gcc/testsuite/c-c++-common/tsan/free_race.c
+++ b/gcc/testsuite/c-c++-common/tsan/free_race.c
@@ -1,4 +1,5 @@
 /* { dg-shouldfail "tsan" } */
+/* { dg-additional-options "-fno-ipa-modref" } */
 
 #include <stdlib.h>
 
diff --git a/gcc/testsuite/c-c++-common/tsan/free_race2.c b/gcc/testsuite/c-c++-common/tsan/free_race2.c
index 3971180..89d12c3 100644
--- a/gcc/testsuite/c-c++-common/tsan/free_race2.c
+++ b/gcc/testsuite/c-c++-common/tsan/free_race2.c
@@ -1,4 +1,5 @@
 /* { dg-shouldfail "tsan" } */
+/* { dg-additional-options "-fno-ipa-modref" } */
 
 #include <stdlib.h>
 
diff --git a/gcc/testsuite/g++.dg/abi/abi-tag16.C b/gcc/testsuite/g++.dg/abi/abi-tag16.C
index d4fa142..3027d79 100644
--- a/gcc/testsuite/g++.dg/abi/abi-tag16.C
+++ b/gcc/testsuite/g++.dg/abi/abi-tag16.C
@@ -1,4 +1,4 @@
-// { dg-options -Wabi=9 }
+// { dg-options "-Wabi=9 -fno-implicit-constexpr" }
 // { dg-final { scan-assembler "_ZGVZN1N1FEvE4NameB5cxx11" } }
 namespace std {
   __extension__ inline namespace __cxx11 __attribute__((abi_tag("cxx11"))) {
diff --git a/gcc/testsuite/g++.dg/abi/abi-tag18a.C b/gcc/testsuite/g++.dg/abi/abi-tag18a.C
index 95e192a..c6fb160 100644
--- a/gcc/testsuite/g++.dg/abi/abi-tag18a.C
+++ b/gcc/testsuite/g++.dg/abi/abi-tag18a.C
@@ -1,5 +1,5 @@
 // { dg-skip-if "PR 70349" { hppa*-*-hpux* && { ! lp64 } } }
-// { dg-options -fabi-version=9 }
+// { dg-options "-fabi-version=9 -fno-implicit-constexpr" }
 // { dg-final { scan-assembler "_Z1fB7__test1v" } }
 // { dg-final { scan-assembler "_ZZ1fB7__test1vEN1T1gB7__test2Ev" } }
 // { dg-final { scan-assembler "_ZZZ1fB7__test1vEN1T1gEvE1x" } }
diff --git a/gcc/testsuite/g++.dg/abi/guard4.C b/gcc/testsuite/g++.dg/abi/guard4.C
index 537c905..71e6744 100644
--- a/gcc/testsuite/g++.dg/abi/guard4.C
+++ b/gcc/testsuite/g++.dg/abi/guard4.C
@@ -3,9 +3,10 @@
 
 namespace x {
   struct s {
-    s() {}
+    s();
     static int a;
   };
+  s::s() {}
   // { dg-final { scan-assembler {.weak[^\n]*_ZGVN1x1bE} } }
   struct s __attribute__((weak)) b = s();
 }
diff --git a/gcc/testsuite/g++.dg/abi/lambda-defarg1.C b/gcc/testsuite/g++.dg/abi/lambda-defarg1.C
index 8c53858..79e4fa6 100644
--- a/gcc/testsuite/g++.dg/abi/lambda-defarg1.C
+++ b/gcc/testsuite/g++.dg/abi/lambda-defarg1.C
@@ -1,5 +1,6 @@
 // PR c++/91241
 // { dg-do compile { target c++11 } }
+// { dg-additional-options -fkeep-inline-functions }
 
 struct A {
   int *b(const int & = []() -> int { return 0; }(),
diff --git a/gcc/testsuite/g++.dg/abi/mangle26.C b/gcc/testsuite/g++.dg/abi/mangle26.C
index 5d16095..2041d77 100644
--- a/gcc/testsuite/g++.dg/abi/mangle26.C
+++ b/gcc/testsuite/g++.dg/abi/mangle26.C
@@ -1,7 +1,7 @@
 // Test of std mangling
 
 // { dg-do compile }
-// { dg-options "-fno-inline" }
+// { dg-options "-fno-inline -fno-implicit-constexpr" }
 
 namespace std {
   struct A {
diff --git a/gcc/testsuite/g++.dg/asan/asan_test.C b/gcc/testsuite/g++.dg/asan/asan_test.C
index 49933ab..15c338b 100644
--- a/gcc/testsuite/g++.dg/asan/asan_test.C
+++ b/gcc/testsuite/g++.dg/asan/asan_test.C
@@ -2,7 +2,7 @@
 // { dg-skip-if "" { *-*-* } { "*" } { "-O2" } }
 // { dg-skip-if "" { *-*-* } { "-flto" } { "" } }
 // { dg-additional-sources "asan_globals_test-wrapper.cc" }
-// { dg-options "-std=c++11 -fsanitize=address -fno-builtin -Wall -Werror -Wno-alloc-size-larger-than -Wno-array-bounds -Wno-stringop-overflow -g -DASAN_UAR=0 -DASAN_HAS_EXCEPTIONS=1 -DASAN_HAS_BLACKLIST=0 -DSANITIZER_USE_DEJAGNU_GTEST=1 -lasan -lpthread" }
+// { dg-options "-std=c++11 -fsanitize=address -fno-builtin -Wall -Werror -Wno-alloc-size-larger-than -Wno-array-bounds -Wno-stringop-overflow -Wno-stringop-overread -g -DASAN_UAR=0 -DASAN_HAS_EXCEPTIONS=1 -DASAN_HAS_BLACKLIST=0 -DSANITIZER_USE_DEJAGNU_GTEST=1 -lasan -lpthread" }
 // { dg-additional-options "-ldl" { target { ! *-*-freebsd* } } }
 // { dg-additional-options "-DASAN_NEEDS_SEGV=1" { target { ! arm*-*-* } } }
 // { dg-additional-options "-DASAN_LOW_MEMORY=1 -DASAN_NEEDS_SEGV=0" { target arm*-*-* } }
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-diag3.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-diag3.C
index 1c43569..c167bb1 100644
--- a/gcc/testsuite/g++.dg/cpp0x/constexpr-diag3.C
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-diag3.C
@@ -13,7 +13,7 @@ int main()
 
 // --------------------
 
-struct complex 			// { dg-message "no .constexpr. constructor" }
+struct complex 			// { dg-message "no .constexpr. constructor" "" { target { ! implicit_constexpr } } }
 {
   complex(double r, double i) : re(r), im(i) { }
   constexpr double real() const { return re; } // { dg-error "not a literal type" "" { target c++11_only } }
@@ -24,23 +24,23 @@ private:
   double im;
 };
 
-constexpr complex co1(0, 1);	   // { dg-error "19:the type .const complex. of .constexpr. variable .co1. is not literal" }
-constexpr double dd2 = co1.real(); // { dg-error "|in .constexpr. expansion of " }
+constexpr complex co1(0, 1);	   // { dg-error "19:the type .const complex. of .constexpr. variable .co1. is not literal" "" { target { ! implicit_constexpr } } }
+constexpr double dd2 = co1.real(); // { dg-error "|in .constexpr. expansion of " "" { target { ! implicit_constexpr } } }
 
 // --------------------
 
-struct base		       // { dg-message "no .constexpr. constructor" }
+struct base		       // { dg-message "no .constexpr. constructor" "" { target { ! implicit_constexpr } } }
 {
   int _M_i;
   base() : _M_i(5) { }
 };
 
-struct derived : public base	// { dg-message "base class" }
+struct derived : public base	// { dg-message "base class" "" { target { ! implicit_constexpr } } }
 {
-  constexpr derived(): base() { } // { dg-error "non-.constexpr. function" }
+  constexpr derived(): base() { } // { dg-error "non-.constexpr. function" "" { target { ! implicit_constexpr } } }
 };
 
-constexpr derived obj;		// { dg-error "not literal" }
+constexpr derived obj;		// { dg-error "not literal" "" { target { ! implicit_constexpr } } }
 
 // --------------------
 
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-ex1.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-ex1.C
index e5e58bd..1d5c58b 100644
--- a/gcc/testsuite/g++.dg/cpp0x/constexpr-ex1.C
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-ex1.C
@@ -87,7 +87,7 @@ struct resource {
   }
 };
 constexpr resource f(resource d)
-{ return d; }                  // { dg-error "non-.constexpr." }
-constexpr resource d = f(9);   // { dg-message ".constexpr." }
+{ return d; }                  // { dg-error "non-.constexpr." "" { target { ! implicit_constexpr } } }
+constexpr resource d = f(9);   // { dg-message ".constexpr." "" { target { ! implicit_constexpr } } }
 
 // 4.4 floating-point constant expressions
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-ice5.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-ice5.C
index 51b328e..e934421 100644
--- a/gcc/testsuite/g++.dg/cpp0x/constexpr-ice5.C
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-ice5.C
@@ -9,5 +9,5 @@ struct A
 struct B
 {
   A a[1];
-  constexpr B() : a() {} // { dg-error "non-constant|non-.constexpr." }
+  constexpr B() : a() {} // { dg-error "non-constant|non-.constexpr." "" { target { ! implicit_constexpr } } }
 };
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-incomplete2.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-incomplete2.C
index a04f1d5..c018ede 100644
--- a/gcc/testsuite/g++.dg/cpp0x/constexpr-incomplete2.C
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-incomplete2.C
@@ -28,4 +28,4 @@ struct D
   C<D> c;
 };
 
-constexpr D d {};		// { dg-error "non-.constexpr. function" }
+constexpr D d {};		// { dg-error "non-.constexpr. function" "" { target { ! implicit_constexpr } } }
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-memfn1.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-memfn1.C
index d58e2ec..3725528 100644
--- a/gcc/testsuite/g++.dg/cpp0x/constexpr-memfn1.C
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-memfn1.C
@@ -13,6 +13,6 @@ constexpr X X::g(X x) { return x; }
 struct Y
 {
   Y() { }
-  constexpr Y f(Y y) { return y; }  // { dg-error "constexpr" }
-  static constexpr Y g(Y y) { return y; } // { dg-error "constexpr" }
+  constexpr Y f(Y y) { return y; }  // { dg-error "constexpr" "" { target { ! implicit_constexpr } } }
+  static constexpr Y g(Y y) { return y; } // { dg-error "constexpr" "" { target { ! implicit_constexpr } } }
 };
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-neg3.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-neg3.C
index 55bb838..89559d7 100644
--- a/gcc/testsuite/g++.dg/cpp0x/constexpr-neg3.C
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-neg3.C
@@ -9,7 +9,7 @@ struct A
 template<typename> struct B
 {
   A a;
-  constexpr int bar() { return a.foo(); } // { dg-error "foo" }
+  constexpr int bar() { return a.foo(); } // { dg-error "foo" "" { target { ! implicit_constexpr } } }
 };
 
-constexpr int i = B<void>().bar(); // { dg-error "bar" }
+constexpr int i = B<void>().bar(); // { dg-error "bar" "" { target { ! implicit_constexpr } } }
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-specialization.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-specialization.C
index 64d8f4e..acda6e0 100644
--- a/gcc/testsuite/g++.dg/cpp0x/constexpr-specialization.C
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-specialization.C
@@ -3,10 +3,10 @@
 
 template<typename T> constexpr int foo(T);
 template<> int foo(int);
-template<> int foo(int);            // { dg-message "previous declaration 'int foo" }
-template<> constexpr int foo(int);  // { dg-error "redeclaration 'constexpr int foo" }
+template<> int foo(int);            // { dg-message "previous declaration 'int foo" "" { target { ! implicit_constexpr } } }
+template<> constexpr int foo(int);  // { dg-error "redeclaration 'constexpr int foo" "" { target { ! implicit_constexpr } } }
 
 template<typename T> int bar(T);
 template<> constexpr int bar(int);
-template<> constexpr int bar(int);  // { dg-message "previous declaration 'constexpr int bar" }
-template<> int bar(int);            // { dg-error "redeclaration 'int bar" }
+template<> constexpr int bar(int);  // { dg-message "previous declaration 'constexpr int bar" "" { target { ! implicit_constexpr } } }
+template<> int bar(int);            // { dg-error "redeclaration 'int bar" "" { target { ! implicit_constexpr } } }
diff --git a/gcc/testsuite/g++.dg/cpp0x/dr2351.C b/gcc/testsuite/g++.dg/cpp0x/dr2351.C
new file mode 100644
index 0000000..5116dfe
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/dr2351.C
@@ -0,0 +1,51 @@
+// DR2351
+// { dg-do compile { target c++11 } }
+
+void
+foo ()
+{
+  void{};
+  void();
+}
+
+template <class ...T>
+void
+bar (T... t)
+{
+  void{t...};
+  void(t...);
+}
+
+void
+baz ()
+{
+  bar ();
+}
+
+template <class ...T>
+void
+qux (T... t)
+{
+  void{t...};	// { dg-error "compound literal of non-object type" }
+}
+
+void
+corge ()
+{
+  qux (1, 2);
+}
+
+template <class ...T>
+void
+garply (T... t)
+{
+  void{t..., t..., t...};
+  void(t..., t..., t...);
+}
+
+template <class ...T>
+void
+grault (T... t)
+{
+  void{t..., 1};	// { dg-error "compound literal of non-object type" }
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/inh-ctor19.C b/gcc/testsuite/g++.dg/cpp0x/inh-ctor19.C
index 7e2d58b..3476f96 100644
--- a/gcc/testsuite/g++.dg/cpp0x/inh-ctor19.C
+++ b/gcc/testsuite/g++.dg/cpp0x/inh-ctor19.C
@@ -11,4 +11,4 @@ struct B : A
   using A::A;
 };
 
-constexpr B b;  // { dg-error "literal" }
+constexpr B b;  // { dg-error "literal" "" { target { ! implicit_constexpr } } }
diff --git a/gcc/testsuite/g++.dg/cpp0x/inh-ctor30.C b/gcc/testsuite/g++.dg/cpp0x/inh-ctor30.C
index 04c5863..c978f0c 100644
--- a/gcc/testsuite/g++.dg/cpp0x/inh-ctor30.C
+++ b/gcc/testsuite/g++.dg/cpp0x/inh-ctor30.C
@@ -1,6 +1,7 @@
 // PR c++/81860
 // { dg-do compile { target c++11 } }
 // { dg-final { scan-assembler "_ZN1AIjEC\[12\]Ev" } }
+// { dg-additional-options -fno-implicit-constexpr }
 
 template <typename T>
 struct A
diff --git a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-eh2.C b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-eh2.C
index 3fb50df..eddd3c9 100644
--- a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-eh2.C
+++ b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-eh2.C
@@ -7,7 +7,7 @@
 
 int main( void )
 {
-  std::set_unexpected( []{ throw 0; } );
+  std::set_unexpected( []{ throw 0; } ); // { dg-warning "deprecated" }
   try
     {
       []() throw( int ) { throw nullptr; }();	// { dg-warning "deprecated" }
diff --git a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle3.C b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle3.C
index 5f17a21..2120d8c 100644
--- a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle3.C
+++ b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle3.C
@@ -1,6 +1,7 @@
 // PR c++/51818
 // { dg-do compile { target c++11 } }
 // { dg-final { scan-assembler "_ZN1AC1IN3foo3barMUlvE_EEET_" } }
+// { dg-additional-options -fno-implicit-constexpr }
 
 struct A
 {
diff --git a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle5.C b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle5.C
index dd95894..23df3a2 100644
--- a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle5.C
+++ b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-mangle5.C
@@ -1,4 +1,5 @@
 // { dg-do compile { target c++11 } }
+// { dg-additional-options -fkeep-inline-functions }
 // { dg-final { scan-assembler "_ZZN1AIiEC4IiEET_S2_Ed_NKUlvE_clEv" } }
 
 template <class T> struct A
diff --git a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-switch.C b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-switch.C
index d05c976..e417967 100644
--- a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-switch.C
+++ b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-switch.C
@@ -16,11 +16,11 @@ main ()
 	      break;		// { dg-error "break" }
 	    }
 	  };
-	  l = []()
+	  l = []()		// { dg-warning "statement will never be executed" }
 	    {
 	    case 3:		// { dg-error "case" }
 	      break;		// { dg-error "break" }
-	    };		// { dg-warning "statement will never be executed" }
+	    };
 	}
     }
 }
diff --git a/gcc/testsuite/g++.dg/cpp0x/noexcept06.C b/gcc/testsuite/g++.dg/cpp0x/noexcept06.C
index ad9edec..ea15223 100644
--- a/gcc/testsuite/g++.dg/cpp0x/noexcept06.C
+++ b/gcc/testsuite/g++.dg/cpp0x/noexcept06.C
@@ -23,7 +23,7 @@ void f() noexcept
 
 int main()
 {
-  std::set_unexpected (my_unexpected);
+  std::set_unexpected (my_unexpected); // { dg-warning "deprecated" }
   std::set_terminate (my_terminate);
   f();
   return 1;
diff --git a/gcc/testsuite/g++.dg/cpp0x/noexcept07.C b/gcc/testsuite/g++.dg/cpp0x/noexcept07.C
index de16e01..dbcc23e 100644
--- a/gcc/testsuite/g++.dg/cpp0x/noexcept07.C
+++ b/gcc/testsuite/g++.dg/cpp0x/noexcept07.C
@@ -18,7 +18,7 @@ void f() throw()
 
 int main()
 {
-  std::set_unexpected (my_unexpected);
+  std::set_unexpected (my_unexpected); // { dg-warning "deprecated" }
   f();
   return 1;
 }
diff --git a/gcc/testsuite/g++.dg/cpp1y/auto-fn12.C b/gcc/testsuite/g++.dg/cpp1y/auto-fn12.C
index 224f278..9888579 100644
--- a/gcc/testsuite/g++.dg/cpp1y/auto-fn12.C
+++ b/gcc/testsuite/g++.dg/cpp1y/auto-fn12.C
@@ -1,5 +1,6 @@
 // { dg-do compile { target c++14 } }
 // { dg-final { scan-assembler "_ZN1AIiEcvDaEv" } }
+// { dg-additional-options -fno-implicit-constexpr }
 
 template <class T>
 struct A {
diff --git a/gcc/testsuite/g++.dg/cpp1y/constexpr-89285-2.C b/gcc/testsuite/g++.dg/cpp1y/constexpr-89285-2.C
index 656bc9c..ea44daa 100644
--- a/gcc/testsuite/g++.dg/cpp1y/constexpr-89285-2.C
+++ b/gcc/testsuite/g++.dg/cpp1y/constexpr-89285-2.C
@@ -10,7 +10,7 @@ struct B {
     int *c = &x->a;
     while (*c)
       c = reinterpret_cast<int *>((reinterpret_cast<char *>(c) + *c));
-    *c = reinterpret_cast<char *>(this) - reinterpret_cast<char *>(c);
+    *c = reinterpret_cast<char *>(this) - reinterpret_cast<char *>(c); // { dg-error "reinterpret_cast" }
   }
 };
 struct C : A {
diff --git a/gcc/testsuite/g++.dg/cpp1y/constexpr-89285.C b/gcc/testsuite/g++.dg/cpp1y/constexpr-89285.C
index 3809e1f..26aab9b 100644
--- a/gcc/testsuite/g++.dg/cpp1y/constexpr-89285.C
+++ b/gcc/testsuite/g++.dg/cpp1y/constexpr-89285.C
@@ -17,4 +17,4 @@ struct C : A {
   B bar {this};
 };
 
-constexpr C foo {};	// { dg-message "expansion of" }
+constexpr C foo {};		// { dg-message "" }
diff --git a/gcc/testsuite/g++.dg/cpp1y/constexpr-loop5.C b/gcc/testsuite/g++.dg/cpp1y/constexpr-loop5.C
index 02f372d..150aadf 100644
--- a/gcc/testsuite/g++.dg/cpp1y/constexpr-loop5.C
+++ b/gcc/testsuite/g++.dg/cpp1y/constexpr-loop5.C
@@ -9,11 +9,11 @@ template<typename>
 constexpr int count()
 {
     auto item = thing {};
-    for(; (item.foo(), false);); // { dg-error "foo" }
+    for(; (item.foo(), false);); // { dg-error "foo" "" { target { ! implicit_constexpr } } }
     return 0;
 }
 
 int main()
 {
-    static_assert( count<int>() == 0, "" ); // { dg-error "" }
+    static_assert( count<int>() == 0, "" ); // { dg-error "" "" { target { ! implicit_constexpr } } }
 }
diff --git a/gcc/testsuite/g++.dg/cpp1y/var-templ69.C b/gcc/testsuite/g++.dg/cpp1y/var-templ69.C
new file mode 100644
index 0000000..420d617
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/var-templ69.C
@@ -0,0 +1,5 @@
+// PR c++/100652
+// { dg-do compile { target c++14 } }
+
+template<class...> int var;
+template<class... Ts> char var<bool, Ts>; // { dg-error "parameter packs not expanded" }
diff --git a/gcc/testsuite/g++.dg/cpp1z/constexpr-lambda7.C b/gcc/testsuite/g++.dg/cpp1z/constexpr-lambda7.C
index 474ce88..faac00c 100644
--- a/gcc/testsuite/g++.dg/cpp1z/constexpr-lambda7.C
+++ b/gcc/testsuite/g++.dg/cpp1z/constexpr-lambda7.C
@@ -8,5 +8,5 @@ struct NonLiteral {
   int n;
 };
 
-static_assert( ID (NonLiteral{3}).n == 3); // { dg-error "non-literal" }
+static_assert( ID (NonLiteral{3}).n == 3); // { dg-error "non-literal" "" { target { ! implicit_constexpr } } }
 // { dg-prune-output "static assertion" }
diff --git a/gcc/testsuite/g++.dg/cpp1z/fold13.C b/gcc/testsuite/g++.dg/cpp1z/fold13.C
new file mode 100644
index 0000000..9d7554f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1z/fold13.C
@@ -0,0 +1,29 @@
+// { dg-do compile { target c++17 } }
+// Verify constexpr evaluation of a large left fold logical expression
+// isn't quadratic in the size of the expanded expression.
+
+template<int> struct S { static constexpr bool value = true; };
+
+template<class T, T...> struct integer_sequence { };
+
+template<class T, T N>
+using make_integer_sequence
+#if __has_builtin(__make_integer_seq)
+  = __make_integer_seq<integer_sequence, T, N>;
+#else
+  = integer_sequence<T, __integer_pack(N)...>;
+#endif
+
+template<int... Is>
+constexpr bool f_impl(integer_sequence<int, Is...>) {
+  return (... && S<Is>::value);
+}
+
+static_assert(f_impl(make_integer_sequence<int, 1024>()));
+
+template<int... Is>
+constexpr bool g_impl(integer_sequence<int, Is...>) {
+  return (... || !S<Is>::value);
+}
+
+static_assert(!g_impl(make_integer_sequence<int, 1024>()));
diff --git a/gcc/testsuite/g++.dg/cpp1z/inh-ctor23.C b/gcc/testsuite/g++.dg/cpp1z/inh-ctor23.C
index fe2c2eb..c806b0e 100644
--- a/gcc/testsuite/g++.dg/cpp1z/inh-ctor23.C
+++ b/gcc/testsuite/g++.dg/cpp1z/inh-ctor23.C
@@ -1,6 +1,6 @@
 // Testcase from P0136
 // { dg-do compile { target c++11 } }
-// { dg-options "-fnew-inheriting-ctors -fdump-tree-gimple -O2 -fno-inline" }
+// { dg-options "-fnew-inheriting-ctors -fdump-tree-gimple -O2 -fno-inline -fno-ipa-modref" }
 
 struct W { W(int); };
 struct V: W { using W::W; };
diff --git a/gcc/testsuite/g++.dg/cpp23/consteval-if11.C b/gcc/testsuite/g++.dg/cpp23/consteval-if11.C
new file mode 100644
index 0000000..a22736c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp23/consteval-if11.C
@@ -0,0 +1,27 @@
+// PR c++/102753
+// { dg-do compile { target c++20 } }
+// { dg-options "" }
+
+struct S {
+  constexpr S () : s (0) {}
+  consteval int foo () { return 1; }
+  virtual consteval int bar () { return 2; }
+  int s;
+};
+
+consteval int foo () { return 42; }
+
+constexpr int
+bar ()
+{
+  if consteval {	// { dg-warning "'if consteval' only available with" "" { target c++20_only } }
+    int (*fn1) () = foo;
+    int (S::*fn2) () = &S::foo;
+    int (S::*fn3) () = &S::bar;
+    S s;
+    return fn1 () + (s.*fn2) () + (s.*fn3) ();
+  }
+  return 0;
+}
+
+static_assert (bar () == 45);
diff --git a/gcc/testsuite/g++.dg/cpp23/consteval-if12.C b/gcc/testsuite/g++.dg/cpp23/consteval-if12.C
new file mode 100644
index 0000000..7a47680
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp23/consteval-if12.C
@@ -0,0 +1,34 @@
+// PR c++/102753
+// { dg-do compile { target c++20 } }
+// { dg-options "" }
+
+struct S {
+  constexpr S () : s (0) {}
+  consteval int foo () { return 1; }
+  virtual consteval int bar () { return 2; }
+  int s;
+};
+
+consteval int foo () { return 42; }
+consteval auto baz () { return foo; }
+consteval auto qux () { return &S::foo; }
+consteval auto corge () { return &S::bar; }
+
+constexpr int
+bar ()
+{
+  S s;
+  if consteval {			// { dg-warning "'if consteval' only available with" "" { target c++20_only } }
+    constexpr auto fn1 = foo;		// { dg-error "immediate evaluation returns address of immediate function" }
+    constexpr auto fn2 = &foo;		// { dg-error "immediate evaluation returns address of immediate function" }
+    constexpr auto fn3 = &S::foo;	// { dg-error "immediate evaluation returns address of immediate function" }
+    constexpr auto fn4 = &S::bar;	// { dg-error "immediate evaluation returns address of immediate function" }
+    constexpr auto fn5 = baz ();	// { dg-error "immediate evaluation returns address of immediate function" }
+    constexpr auto fn6 = qux ();	// { dg-error "immediate evaluation returns address of immediate function" }
+    constexpr auto fn7 = corge ();	// { dg-error "immediate evaluation returns address of immediate function" }
+    return fn1 () + fn2 () + (s.*fn3) () + (s.*fn4) () + fn5 () + (s.*fn6) () + (s.*fn7) ();
+  }
+  return 0;
+}
+
+auto a = bar ();
diff --git a/gcc/testsuite/g++.dg/cpp23/init-stmt1.C b/gcc/testsuite/g++.dg/cpp23/init-stmt1.C
new file mode 100644
index 0000000..29e3256a
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp23/init-stmt1.C
@@ -0,0 +1,31 @@
+// PR c++/102617
+// P2360R0: Extend init-statement to allow alias-declaration
+// { dg-do compile { target c++20 } }
+// Test valid use.
+
+int v[10];
+
+void
+g ()
+{
+  for (using T = int; (T) false;) // { dg-error "only available with" "" { target c++20_only } }
+    ;
+  for (using T = int; T e : v) // { dg-error "only available with" "" { target c++20_only } }
+    (void) e;
+  if (using T = int; true) // { dg-error "only available with" "" { target c++20_only } }
+    {
+      T x = 0;
+      (void) x;
+    }
+  if constexpr (using T = int; true) // { dg-error "only available with" "" { target c++20_only } }
+    {
+      T x = 0;
+      (void) x;
+    }
+  switch (using T = int; 42) // { dg-error "only available with" "" { target c++20_only } }
+    case 42:
+      {
+	T x = 0;
+	(void) x;
+      }
+}
diff --git a/gcc/testsuite/g++.dg/cpp23/init-stmt2.C b/gcc/testsuite/g++.dg/cpp23/init-stmt2.C
new file mode 100644
index 0000000..ca6201b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp23/init-stmt2.C
@@ -0,0 +1,25 @@
+// PR c++/102617
+// P2360R0: Extend init-statement to allow alias-declaration
+// { dg-do compile { target c++23 } }
+// Test invalid use.
+
+int v[10];
+namespace N { using X = int; }
+
+void
+g ()
+{
+  for (using N::X; false;) // { dg-error "expected" }
+    ;
+  for (using N::X; int e : v) // { dg-error "expected" }
+    (void) e;
+  for (using T = int; using U = int; int e : v) // { dg-error "" }
+    ;
+  if (using N::X; false) // { dg-error "expected" }
+    {}
+  switch (using N::X; 0) // { dg-error "expected" }
+    ;
+ if (using T = int;) // { dg-error "expected" }
+  {
+  }
+}
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-pr85846.C b/gcc/testsuite/g++.dg/cpp2a/concepts-pr85846.C
new file mode 100644
index 0000000..7fda002
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-pr85846.C
@@ -0,0 +1,12 @@
+// PR c++/85846
+// { dg-do compile { target c++20 } }
+
+template<int=0>
+concept A = true;
+
+bool i(A<>);
+
+template<class=int>
+concept B = true;
+
+bool j(B<>);
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-pr98394.C b/gcc/testsuite/g++.dg/cpp2a/concepts-pr98394.C
new file mode 100644
index 0000000..c8407cd
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-pr98394.C
@@ -0,0 +1,14 @@
+// PR c++/98394
+// { dg-do compile { target c++20 } }
+
+template<int...>
+concept C = true;
+
+template<int, int>
+concept D = true;
+
+int main() {
+  if (C<1>); // { dg-bogus "does not constrain a type" }
+  if (D<1>); // { dg-error "wrong number of template arguments" }
+	     // { dg-bogus "does not constrain a type" "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/g++.dg/cpp2a/consteval13.C b/gcc/testsuite/g++.dg/cpp2a/consteval13.C
index a2e1750..09ea6b7 100644
--- a/gcc/testsuite/g++.dg/cpp2a/consteval13.C
+++ b/gcc/testsuite/g++.dg/cpp2a/consteval13.C
@@ -10,8 +10,8 @@ void
 foo ()
 {
    auto qux = [] (fnptr a = quux ()) consteval { return a (); };
-   constexpr auto c = qux (baz);	// { dg-error "28:taking address of an immediate function" }
-   constexpr auto d = qux (bar);	// { dg-error "28:taking address of an immediate function" }
+   constexpr auto c = qux (baz);
+   constexpr auto d = qux (bar);
    static_assert (c == 1);
    static_assert (d == 42);
 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/consteval20.C b/gcc/testsuite/g++.dg/cpp2a/consteval20.C
new file mode 100644
index 0000000..2c35963
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/consteval20.C
@@ -0,0 +1,24 @@
+// PR c++/102753
+// { dg-do compile { target c++20 } }
+
+struct S {
+  consteval int foo () const { return 42; }
+};
+
+constexpr S s;
+
+int
+bar ()
+{
+  return (s.*&S::foo) ();		// { dg-error "taking address of an immediate function" }
+}
+
+constexpr auto a = &S::foo;		// { dg-error "taking address of an immediate function" }
+
+consteval int
+baz ()
+{
+  return (s.*&S::foo) ();
+}
+
+static_assert (baz () == 42);
diff --git a/gcc/testsuite/g++.dg/cpp2a/consteval21.C b/gcc/testsuite/g++.dg/cpp2a/consteval21.C
new file mode 100644
index 0000000..06ec705
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/consteval21.C
@@ -0,0 +1,35 @@
+// PR c++/102753
+// { dg-do compile { target c++20 } }
+
+struct S {
+  constexpr S () : s (0) {}
+  consteval int foo () { return 1; }
+  virtual consteval int bar () { return 2; }
+  int s;
+};
+
+consteval int foo () { return 42; }
+
+consteval int
+bar (int (*fn) () = &foo)
+{
+  return fn ();
+}
+
+consteval int
+baz (int (S::*fn) () = &S::foo)
+{
+  S s;
+  return (s.*fn) ();
+}
+
+consteval int
+qux (int (S::*fn) () = &S::bar)
+{
+  S s;
+  return (s.*fn) ();
+}
+
+static_assert (bar () == 42);
+static_assert (baz () == 1);
+static_assert (qux () == 2);
diff --git a/gcc/testsuite/g++.dg/cpp2a/consteval22.C b/gcc/testsuite/g++.dg/cpp2a/consteval22.C
new file mode 100644
index 0000000..5c36371
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/consteval22.C
@@ -0,0 +1,34 @@
+// PR c++/102753
+// { dg-do compile { target c++20 } }
+
+struct S {
+  constexpr S () : s (0) {}
+  consteval int foo () { return 1; }
+  virtual consteval int bar () { return 2; }
+  int s;
+};
+typedef int (S::*P) ();
+
+consteval P
+foo ()
+{
+  return &S::foo;
+}
+
+consteval P
+bar ()
+{
+  return &S::bar;
+}
+
+consteval int
+baz ()
+{
+  S s;
+  return (s.*(foo ())) () + (s.*(bar ())) ();
+}
+
+static_assert (baz () == 3);
+
+constexpr P a = foo ();		// { dg-error "immediate evaluation returns address of immediate function" }
+constexpr P b = bar ();		// { dg-error "immediate evaluation returns address of immediate function" }
diff --git a/gcc/testsuite/g++.dg/cpp2a/consteval23.C b/gcc/testsuite/g++.dg/cpp2a/consteval23.C
new file mode 100644
index 0000000..4c7e844
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/consteval23.C
@@ -0,0 +1,13 @@
+// PR c++/102753
+// { dg-do compile { target c++20 } }
+
+consteval int foo () { return 42; }
+
+consteval int
+bar (int (*fn) () = foo)
+{
+  return fn ();
+}
+
+static_assert (bar () == 42);
+static_assert (bar (foo) == 42);
diff --git a/gcc/testsuite/g++.dg/cpp2a/consteval24.C b/gcc/testsuite/g++.dg/cpp2a/consteval24.C
new file mode 100644
index 0000000..6d0c63c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/consteval24.C
@@ -0,0 +1,30 @@
+// PR c++/102753
+// { dg-do compile { target c++20 } }
+
+struct S {
+  constexpr S () : s (0) {}
+  consteval int foo () { return 1; }
+  virtual consteval int bar () { return 2; }
+  int s;
+};
+
+consteval int foo () { return 42; }
+consteval auto baz () { return foo; }
+consteval auto qux () { return &S::foo; }
+consteval auto corge () { return &S::bar; }
+
+consteval int
+bar ()
+{
+  S s;
+  constexpr auto fn1 = foo;		// { dg-error "immediate evaluation returns address of immediate function" }
+  constexpr auto fn2 = &foo;		// { dg-error "immediate evaluation returns address of immediate function" }
+  constexpr auto fn3 = &S::foo;		// { dg-error "immediate evaluation returns address of immediate function" }
+  constexpr auto fn4 = &S::bar;		// { dg-error "immediate evaluation returns address of immediate function" }
+  constexpr auto fn5 = baz ();		// { dg-error "immediate evaluation returns address of immediate function" }
+  constexpr auto fn6 = qux ();		// { dg-error "immediate evaluation returns address of immediate function" }
+  constexpr auto fn7 = corge ();	// { dg-error "immediate evaluation returns address of immediate function" }
+  return fn1 () + fn2 () + (s.*fn3) () + (s.*fn4) () + fn5 () + (s.*fn6) () + (s.*fn7) ();
+}
+
+auto a = bar ();
diff --git a/gcc/testsuite/g++.dg/cpp2a/consteval7.C b/gcc/testsuite/g++.dg/cpp2a/consteval7.C
index 10e4ea4..23f3d25 100644
--- a/gcc/testsuite/g++.dg/cpp2a/consteval7.C
+++ b/gcc/testsuite/g++.dg/cpp2a/consteval7.C
@@ -7,7 +7,7 @@ constexpr auto a = bar ();	// { dg-error "immediate evaluation returns address o
 struct S { int b; int (*c) (); };
 consteval S baz () { return { 5, foo }; }
 consteval int qux () { S s = baz (); return s.b + s.c (); }
-consteval int quux () { constexpr S s = baz (); return s.b + s.c (); }
+consteval int quux () { constexpr S s = baz (); return s.b + s.c (); }	// { dg-error "immediate evaluation returns address of immediate function 'consteval int foo\\(\\)'" }
 constexpr auto d = baz ();	// { dg-error "immediate evaluation returns address of immediate function 'consteval int foo\\(\\)'" }
 constexpr auto e = qux ();
 constexpr auto f = quux ();
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dtor3.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dtor3.C
index 69fe9e2..a68a6b4 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dtor3.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dtor3.C
@@ -7,16 +7,16 @@ struct S
   constexpr ~S () {}
   int s;
 };
-struct T	// { dg-message "'T' is not literal because" }
-{		// { dg-message "'T' does not have 'constexpr' destructor" "" { target *-*-* } .-1 }
+struct T	// { dg-message "'T' is not literal because" "" { target { ! implicit_constexpr } } }
+{		// { dg-message "'T' does not have 'constexpr' destructor" "" { target { ! implicit_constexpr } } .-1 }
   constexpr T () : t (0) {}
-  ~T () {}	// { dg-message "defaulted destructor calls non-'constexpr' 'T::~T\\(\\)'" }
+  ~T () {}	// { dg-message "defaulted destructor calls non-'constexpr' 'T::~T\\(\\)'" "" { target { ! implicit_constexpr } } }
   int t;
 };
 struct U : public S
 {
   constexpr U () : u (0) {}
-  constexpr ~U () = default;	// { dg-error "explicitly defaulted function 'constexpr U::~U\\(\\)' cannot be declared 'constexpr' because the implicit declaration is not 'constexpr'" }
+  constexpr ~U () = default;	// { dg-error "explicitly defaulted function 'constexpr U::~U\\(\\)' cannot be declared 'constexpr' because the implicit declaration is not 'constexpr'" "" { target { ! implicit_constexpr } } }
   int u;
   T t;
 };
@@ -100,11 +100,11 @@ struct W8
 struct X : public T
 {
   constexpr X () : x (0) {}
-  constexpr ~X () = default;	// { dg-error "explicitly defaulted function 'constexpr X::~X\\(\\)' cannot be declared 'constexpr' because the implicit declaration is not 'constexpr'" }
+  constexpr ~X () = default;	// { dg-error "explicitly defaulted function 'constexpr X::~X\\(\\)' cannot be declared 'constexpr' because the implicit declaration is not 'constexpr'" "" { target { ! implicit_constexpr } } }
   int x;
 };
 constexpr S s;
-constexpr T t;	// { dg-error "the type 'const T' of 'constexpr' variable 't' is not literal" }
+constexpr T t;	// { dg-error "the type 'const T' of 'constexpr' variable 't' is not literal" "" { target { ! implicit_constexpr } } }
 constexpr W0 w1;
 constexpr W0 w2 = 12;
 constexpr W1 w3 = 5;	// { dg-message "in 'constexpr' expansion of" }
@@ -167,19 +167,19 @@ constexpr int x5 = f5 ();	// { dg-message "in 'constexpr' expansion of" }
 void
 f6 ()
 {
-  constexpr T t2;	// { dg-error "the type 'const T' of 'constexpr' variable 't2' is not literal" }
+  constexpr T t2;	// { dg-error "the type 'const T' of 'constexpr' variable 't2' is not literal" "" { target { ! implicit_constexpr } } }
 }
 
 constexpr int
 f7 ()
 {
-  constexpr T t3;	// { dg-error "the type 'const T' of 'constexpr' variable 't3' is not literal" }
+  constexpr T t3;	// { dg-error "the type 'const T' of 'constexpr' variable 't3' is not literal" "" { target { ! implicit_constexpr } } }
   return 0;
 }
 
 constexpr int
 f8 ()
 {
-  T t4;			// { dg-error "variable 't4' of non-literal type 'T' in 'constexpr' function only available with" "" { target c++20_down } }
+  T t4;			// { dg-error "variable 't4' of non-literal type 'T' in 'constexpr' function only available with" "" { target { c++20_down && { ! implicit_constexpr } } } }
   return 0;
 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-new13.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-new13.C
index 6762c24..7eed50c 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-new13.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-new13.C
@@ -9,7 +9,7 @@ struct A {
 
 struct B : A {
   constexpr B () : b (0) {}
-  virtual int foo () { return 0 + b * 4; }	// { dg-message "declared here" }
+  virtual int foo () { return 0 + b * 4; }	// { dg-message "declared here" "" { target { ! implicit_constexpr } } }
   int b;
 };
 
@@ -18,7 +18,7 @@ foo ()
 {
   A *a = new B ();
   a->a = 4;
-  int r = a->foo ();	// { dg-error "call to non-.constexpr. function" }
+  int r = a->foo ();	// { dg-error "call to non-.constexpr. function" "" { target { ! implicit_constexpr } } }
   delete a;
   return r;
 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-virtual20.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-virtual20.C
new file mode 100644
index 0000000..3c411fa
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-virtual20.C
@@ -0,0 +1,22 @@
+// Test for constexpr call through vbase thunk.
+// { dg-do compile { target c++20 } }
+
+class Rep {
+public:
+  constexpr virtual int foo() { return 1; }
+};
+
+class VBase {
+public:
+  constexpr virtual int foo() { return 2; }
+};
+
+class Main : public Rep, virtual public VBase {
+public:
+  constexpr virtual int foo() { return 5; }
+};
+
+int main() {
+  Main m;
+  static_assert(static_cast<VBase*>(&m)->foo() == 5); // { dg-error "" }
+}
diff --git a/gcc/testsuite/g++.dg/cpp2a/constinit11.C b/gcc/testsuite/g++.dg/cpp2a/constinit11.C
index e8b3bcb..b090277 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constinit11.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constinit11.C
@@ -39,9 +39,9 @@ constinit thread_local const int &r2 = gl;
 constinit const int &r3 // { dg-error "variable .r3. does not have a constant initializer" }
   = foo (); // { dg-error "call to non-.constexpr. function" }
 constinit const literal &r4 = 42;
-constinit const nonliteral &r5 // { dg-error "variable .r5. does not have a constant initializer" }
-  = 42; // { dg-error "call to non-.constexpr. function" }
-constinit const int &r6 = nonliteral(2).m; // { dg-error "variable .r6. does not have a constant initializer|call to non-.constexpr. function" }
+constinit const nonliteral &r5 // { dg-error "variable .r5. does not have a constant initializer" "" { target { ! implicit_constexpr } } }
+  = 42; // { dg-error "call to non-.constexpr. function" "" { target { ! implicit_constexpr } } }
+constinit const int &r6 = nonliteral(2).m; // { dg-error "variable .r6. does not have a constant initializer|call to non-.constexpr. function" "" { target { ! implicit_constexpr } } }
 
 constinit pod p1;
 constinit pod p2 = { 42 };
@@ -64,8 +64,8 @@ constinit thread_local literal l11{};
 pod S::p;
 constinit pod S::pc(S::p); // { dg-error "variable .S::pc. does not have a constant initializer|not usable" }
 
-constinit const nonliteral S::n(42); // { dg-error "variable .S::n. does not have a constant initializer|call to non-.constexpr. function" }
-constinit int n1 = nonliteral{42}.m; // { dg-error "variable .n1. does not have a constant initializer|temporary of non-literal type" }
+constinit const nonliteral S::n(42); // { dg-error "variable .S::n. does not have a constant initializer|call to non-.constexpr. function" "" { target { ! implicit_constexpr } } }
+constinit int n1 = nonliteral{42}.m; // { dg-error "variable .n1. does not have a constant initializer|temporary of non-literal type" "" { target { ! implicit_constexpr } } }
 constinit int n2 = literal{42}.m;
 
 void
diff --git a/gcc/testsuite/g++.dg/cpp2a/constinit12.C b/gcc/testsuite/g++.dg/cpp2a/constinit12.C
index cc6569b..faa1b43 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constinit12.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constinit12.C
@@ -8,7 +8,7 @@ struct S {
 template <class T>
 struct U {
   T m;
-  constexpr U(int i) : m(i) { } // { dg-error "call to non-.constexpr. function" }
+  constexpr U(int i) : m(i) { } // { dg-error "call to non-.constexpr. function" "" { target { ! implicit_constexpr } } }
 };
 
-constinit U<S> u(42); // { dg-error "does not have a constant initializer|called in a constant expression" }
+constinit U<S> u(42); // { dg-error "does not have a constant initializer|called in a constant expression" "" { target { ! implicit_constexpr } } }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constinit14.C b/gcc/testsuite/g++.dg/cpp2a/constinit14.C
index 86a058b..06c4cb4 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constinit14.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constinit14.C
@@ -2,7 +2,7 @@
 // { dg-do compile { target c++20 } }
 
 struct Value {
-  Value() : v{new int{42}} {}
+  Value() : v{new int{42}} {}	// { dg-error "result of 'operator new'" "" { target implicit_constexpr } }
   int* v;
 };
 
diff --git a/gcc/testsuite/g++.dg/cpp2a/constinit15.C b/gcc/testsuite/g++.dg/cpp2a/constinit15.C
index 29e8b51..b621558 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constinit15.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constinit15.C
@@ -6,7 +6,7 @@ struct B {
 };
 
 struct A {
-    constinit static inline B b1{}; // { dg-error "does not have a constant initializer|call to non-.constexpr. function" }
+    constinit static inline B b1{}; // { dg-error "does not have a constant initializer|call to non-.constexpr. function" "" { target { ! implicit_constexpr } } }
 };
 
 int main() {
diff --git a/gcc/testsuite/g++.dg/cpp2a/nontype-class50.C b/gcc/testsuite/g++.dg/cpp2a/nontype-class50.C
new file mode 100644
index 0000000..1c2786a
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/nontype-class50.C
@@ -0,0 +1,13 @@
+// PR c++/102933
+// { dg-do compile { target c++20 } }
+
+template<class T> struct X { T t; };
+
+template<X> void f();
+
+template<class T>
+void g() {
+  f<X{T{0}}>();
+}
+
+template void g<int>();
diff --git a/gcc/testsuite/g++.dg/cpp2a/nontype-class50a.C b/gcc/testsuite/g++.dg/cpp2a/nontype-class50a.C
new file mode 100644
index 0000000..eb8a6ad
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/nontype-class50a.C
@@ -0,0 +1,5 @@
+// PR c++/102933
+// { dg-do compile { target c++20 } }
+// { dg-additional-options "-fconcepts-ts" }
+
+#include "nontype-class50.C"
diff --git a/gcc/testsuite/g++.dg/cpp2a/spaceship-constexpr1.C b/gcc/testsuite/g++.dg/cpp2a/spaceship-constexpr1.C
index 6ccc7e8..dff5927 100644
--- a/gcc/testsuite/g++.dg/cpp2a/spaceship-constexpr1.C
+++ b/gcc/testsuite/g++.dg/cpp2a/spaceship-constexpr1.C
@@ -9,7 +9,7 @@ struct A
 struct B
 {
   A a;
-  bool operator==(const B&) const = default; // { dg-error "A::operator==" }
+  bool operator==(const B&) const = default; // { dg-error "A::operator==" "" { target { ! implicit_constexpr } } }
 };
 
-constexpr bool x = B() == B();	// { dg-error "non-.constexpr" }
+constexpr bool x = B() == B();	// { dg-error "non-.constexpr" "" { target { ! implicit_constexpr } } }
diff --git a/gcc/testsuite/g++.dg/cpp2a/spaceship-eq3.C b/gcc/testsuite/g++.dg/cpp2a/spaceship-eq3.C
index d31faef..7a517a8 100644
--- a/gcc/testsuite/g++.dg/cpp2a/spaceship-eq3.C
+++ b/gcc/testsuite/g++.dg/cpp2a/spaceship-eq3.C
@@ -12,5 +12,5 @@ struct D
 };
 
 constexpr D d{A()};
-static_assert (d == d);		// { dg-error "non-constant|constexpr" }
-static_assert (!(d != d));	// { dg-error "non-constant|constexpr" }
+static_assert (d == d);		// { dg-error "constant|constexpr" }
+static_assert (!(d != d));	// { dg-error "constant|constexpr" }
diff --git a/gcc/testsuite/g++.dg/cpp2a/udlit-class-nttp-neg2.C b/gcc/testsuite/g++.dg/cpp2a/udlit-class-nttp-neg2.C
index 62cb86a..ab7c012 100644
--- a/gcc/testsuite/g++.dg/cpp2a/udlit-class-nttp-neg2.C
+++ b/gcc/testsuite/g++.dg/cpp2a/udlit-class-nttp-neg2.C
@@ -9,5 +9,5 @@ struct non_literal_class {
   // auto operator<=> (const non_literal_fixed_string&) = default;
 };
 
-template <non_literal_class> // { dg-error "11:is not a valid type for a template non-type parameter because it is not structural" }
-int operator"" _udl();       // { dg-error "5:literal operator template .int operator\"\"_udl\\(\\). has invalid parameter list" }
+template <non_literal_class> // { dg-error "11:is not a valid type for a template non-type parameter because it is not structural" "" { target { ! implicit_constexpr } } }
+int operator"" _udl();       // { dg-error "5:literal operator template .int operator\"\"_udl\\(\\). has invalid parameter list" "" { target { ! implicit_constexpr } } }
diff --git a/gcc/testsuite/g++.dg/debug/dwarf2/auto1.C b/gcc/testsuite/g++.dg/debug/dwarf2/auto1.C
index 5daf3cd..3e5867d 100644
--- a/gcc/testsuite/g++.dg/debug/dwarf2/auto1.C
+++ b/gcc/testsuite/g++.dg/debug/dwarf2/auto1.C
@@ -1,6 +1,6 @@
 // PR c++/53756
 // { dg-do compile { target c++14 } }
-// { dg-options "-gdwarf-2 -dA -fno-debug-types-section" }
+// { dg-options "-gdwarf-2 -dA -fno-debug-types-section -fno-inline" }
 // We're looking for something like
 
 // .uleb128 0x3    # (DIE (0x33) DW_TAG_subprogram)
diff --git a/gcc/testsuite/g++.dg/debug/dwarf2/cdtor-1.C b/gcc/testsuite/g++.dg/debug/dwarf2/cdtor-1.C
index c0d3d22..b211c02 100644
--- a/gcc/testsuite/g++.dg/debug/dwarf2/cdtor-1.C
+++ b/gcc/testsuite/g++.dg/debug/dwarf2/cdtor-1.C
@@ -1,5 +1,5 @@
 // origin PR debug/49047
-// { dg-options "-gdwarf-2 -dA -fno-merge-debug-strings" }
+// { dg-options "-gdwarf-2 -dA -fno-merge-debug-strings -fno-implicit-constexpr" }
 // { dg-do compile }
 
 struct K
diff --git a/gcc/testsuite/g++.dg/debug/dwarf2/lambda1.C b/gcc/testsuite/g++.dg/debug/dwarf2/lambda1.C
index bd3ce5d..b2566c7 100644
--- a/gcc/testsuite/g++.dg/debug/dwarf2/lambda1.C
+++ b/gcc/testsuite/g++.dg/debug/dwarf2/lambda1.C
@@ -1,6 +1,6 @@
 // PR c++/43912
 // { dg-do compile { target c++11 } }
-// { dg-options "-gdwarf-2 -dA -fno-merge-debug-strings -gno-strict-dwarf" }
+// { dg-options "-gdwarf-2 -dA -fno-merge-debug-strings -gno-strict-dwarf -fno-inline" }
 
 // Check for the local alias variables that point to the members of the closure.
 // { dg-final { scan-assembler-times "DW_TAG_variable\[^.\]*\.ascii \"j.0\"" 4 { xfail { powerpc-ibm-aix* } } } }
diff --git a/gcc/testsuite/g++.dg/debug/dwarf2/pr101378.C b/gcc/testsuite/g++.dg/debug/dwarf2/pr101378.C
new file mode 100644
index 0000000..ec5e44d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/debug/dwarf2/pr101378.C
@@ -0,0 +1,13 @@
+// PR debug/101378
+// { dg-do compile { target c++11 } }
+// { dg-options "-gdwarf-5 -dA" }
+// { dg-final { scan-assembler-times "0\[^0-9x\\r\\n\]* DW_AT_data_member_location" 1 } }
+// { dg-final { scan-assembler-times "1\[^0-9x\\r\\n\]* DW_AT_data_member_location" 1 } }
+// { dg-final { scan-assembler-times "2\[^0-9x\\r\\n\]* DW_AT_data_member_location" 1 } }
+// { dg-final { scan-assembler-not "-1\[^0-9x\\r\\n\]* DW_AT_data_member_location" } }
+
+struct E {};
+struct S
+{
+  [[no_unique_address]] E e, f, g;
+} s;
diff --git a/gcc/testsuite/g++.dg/debug/dwarf2/pr54508.C b/gcc/testsuite/g++.dg/debug/dwarf2/pr54508.C
index e7a6aa4..0a3721e 100644
--- a/gcc/testsuite/g++.dg/debug/dwarf2/pr54508.C
+++ b/gcc/testsuite/g++.dg/debug/dwarf2/pr54508.C
@@ -1,6 +1,6 @@
 // PR debug/54508
 // { dg-do compile }
-// { dg-options "-gdwarf-2 -g2 -dA -fno-merge-debug-strings" }
+// { dg-options "-gdwarf-2 -g2 -dA -fno-merge-debug-strings -fno-inline" }
 
 // { dg-final { scan-assembler "\"cbase\\\\0\"\[ \t\]+\[#;/!|@\]+ +DW_AT_name" } }
 // { dg-final { scan-assembler "\"OPCODE\\\\0\"\[ \t\]+\[#;/!|@\]+ +DW_AT_name" } }
diff --git a/gcc/testsuite/g++.dg/debug/dwarf2/pubnames-2.C b/gcc/testsuite/g++.dg/debug/dwarf2/pubnames-2.C
index af0f6f1..1fb5004 100644
--- a/gcc/testsuite/g++.dg/debug/dwarf2/pubnames-2.C
+++ b/gcc/testsuite/g++.dg/debug/dwarf2/pubnames-2.C
@@ -1,6 +1,6 @@
 // { dg-do compile { target c++11 } }
 // { dg-skip-if "" { powerpc-ibm-aix* } }
-// { dg-options "-gpubnames -gdwarf-4 -fno-debug-types-section -dA" }
+// { dg-options "-gpubnames -gdwarf-4 -fno-debug-types-section -dA -fno-inline" }
 // { dg-final { scan-assembler-times "\.section\[\t \]\[^\n\]*debug_pubnames" 1 } }
 // { dg-final { scan-assembler "\"\\(anonymous namespace\\)\\\\0\"+\[ \t\]+\[#;/|@!]+\[ \t\]+external name" } }
 // { dg-final { scan-assembler "\"one\\\\0\"+\[ \t\]+\[#;/|@!]+\[ \t\]+external name" } }
diff --git a/gcc/testsuite/g++.dg/debug/dwarf2/pubnames-3.C b/gcc/testsuite/g++.dg/debug/dwarf2/pubnames-3.C
index 63b870b..37e04fb 100644
--- a/gcc/testsuite/g++.dg/debug/dwarf2/pubnames-3.C
+++ b/gcc/testsuite/g++.dg/debug/dwarf2/pubnames-3.C
@@ -1,6 +1,6 @@
 // { dg-do compile { target c++11 } }
 // { dg-skip-if "" { powerpc-ibm-aix* } }
-// { dg-options "-gpubnames -gdwarf-4 -fdebug-types-section -dA" }
+// { dg-options "-gpubnames -gdwarf-4 -fdebug-types-section -dA -fno-inline" }
 // { dg-final { scan-assembler-times "\.section\[\t \]\[^\n\]*debug_pubnames" 1 } }
 // { dg-final { scan-assembler "\"\\(anonymous namespace\\)\\\\0\"+\[ \t\]+\[#;/|@!]+\[ \t\]+external name" } }
 // { dg-final { scan-assembler "\"one\\\\0\"+\[ \t\]+\[#;/|@!]+\[ \t\]+external name" } }
diff --git a/gcc/testsuite/g++.dg/eh/forced3.C b/gcc/testsuite/g++.dg/eh/forced3.C
index 9e92daf..51cbc8d 100644
--- a/gcc/testsuite/g++.dg/eh/forced3.C
+++ b/gcc/testsuite/g++.dg/eh/forced3.C
@@ -53,7 +53,7 @@ doit () throw()
 
 int main()
 { 
-  std::set_unexpected (handle_unexpected);
+  std::set_unexpected (handle_unexpected); // { dg-warning "deprecated" "" { target c++11 } }
   doit ();
   abort ();
 }
diff --git a/gcc/testsuite/g++.dg/eh/unexpected1.C b/gcc/testsuite/g++.dg/eh/unexpected1.C
index cd5585f..973ebea 100644
--- a/gcc/testsuite/g++.dg/eh/unexpected1.C
+++ b/gcc/testsuite/g++.dg/eh/unexpected1.C
@@ -30,7 +30,7 @@ doit () throw (Two)			// { dg-warning "deprecated" "" { target { c++11 } } }
 
 int main ()
 {
-  std::set_unexpected (handle_unexpected);
+  std::set_unexpected (handle_unexpected); // { dg-warning "deprecated" "" { target { c++11 } } }
 
   try
   {
diff --git a/gcc/testsuite/g++.dg/ext/complex10.C b/gcc/testsuite/g++.dg/ext/complex10.C
new file mode 100644
index 0000000..67dfb0c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/complex10.C
@@ -0,0 +1,5 @@
+// PR c++/103114
+// { dg-do compile }
+// { dg-options "-frounding-math" }
+
+_Complex double d = 10.1i;
diff --git a/gcc/testsuite/g++.dg/ext/flexary37.C b/gcc/testsuite/g++.dg/ext/flexary37.C
index ceb5053..5cd48c1 100644
--- a/gcc/testsuite/g++.dg/ext/flexary37.C
+++ b/gcc/testsuite/g++.dg/ext/flexary37.C
@@ -12,4 +12,4 @@ public:
 
 struct y { // { dg-error "unknown array size in delete" }
     int a; C b[];
-} y = { 1, { { 2, 3 } } }; // { dg-error "unknown array size in delete" }
+} y = { 1, { { 2, 3 } } };
diff --git a/gcc/testsuite/g++.dg/ext/is_literal_type3.C b/gcc/testsuite/g++.dg/ext/is_literal_type3.C
index 22d8494..4fede87 100644
--- a/gcc/testsuite/g++.dg/ext/is_literal_type3.C
+++ b/gcc/testsuite/g++.dg/ext/is_literal_type3.C
@@ -6,7 +6,11 @@ struct S {
   int n;
 };
 
+#if __cpp_implicit_constexpr
+static_assert(__is_literal_type(S), "");
+#else
 static_assert(!__is_literal_type(S), "");
+#endif
 
 #ifdef __cpp_constexpr_dynamic_alloc
 struct T {
diff --git a/gcc/testsuite/g++.dg/ext/visibility/template7.C b/gcc/testsuite/g++.dg/ext/visibility/template7.C
index 5197fb1..f1490fa 100644
--- a/gcc/testsuite/g++.dg/ext/visibility/template7.C
+++ b/gcc/testsuite/g++.dg/ext/visibility/template7.C
@@ -1,6 +1,6 @@
 // PR c++/35688
 // { dg-require-visibility "" }
-// { dg-options "-fvisibility=hidden" }
+// { dg-options "-fvisibility=hidden -fno-inline" }
 
 // { dg-final { scan-hidden "_ZN1s6vectorI1AEC1Ev" } }
 // { dg-final { scan-hidden "_ZN1s3fooI1AEEvT_" } }
diff --git a/gcc/testsuite/g++.dg/gcov/gcov-12.C b/gcc/testsuite/g++.dg/gcov/gcov-12.C
index c4708e4..9f2b29b 100644
--- a/gcc/testsuite/g++.dg/gcov/gcov-12.C
+++ b/gcc/testsuite/g++.dg/gcov/gcov-12.C
@@ -1,5 +1,5 @@
 /* PR 51113 */
-/* { dg-options "-fprofile-arcs -ftest-coverage -fpic" } */
+/* { dg-options "-fprofile-arcs -ftest-coverage -fpic -fno-implicit-constexpr" } */
 /* { dg-do run { target native } } */
 /* { dg-additional-sources "gcovpart-12b.C" } */
 
diff --git a/gcc/testsuite/g++.dg/gcov/gcov-2.C b/gcc/testsuite/g++.dg/gcov/gcov-2.C
index 2b4cdd8..05db15d 100644
--- a/gcc/testsuite/g++.dg/gcov/gcov-2.C
+++ b/gcc/testsuite/g++.dg/gcov/gcov-2.C
@@ -1,6 +1,6 @@
 /* Verify line coverage counts for simple member functions. */
 
-/* { dg-options "-fprofile-arcs -ftest-coverage" } */
+/* { dg-options "-fprofile-arcs -ftest-coverage -fno-implicit-constexpr" } */
 /* { dg-do run { target native } } */
 
 class C {
diff --git a/gcc/testsuite/g++.dg/goacc/firstprivate-mappings-1.C b/gcc/testsuite/g++.dg/goacc/firstprivate-mappings-1.C
index 1b1badb..99a3bd4 100644
--- a/gcc/testsuite/g++.dg/goacc/firstprivate-mappings-1.C
+++ b/gcc/testsuite/g++.dg/goacc/firstprivate-mappings-1.C
@@ -416,7 +416,7 @@ vla (int &array_li)
   copyout (array_so)
   /* The gimplifier has created an implicit 'firstprivate' clause for the array
      length.
-     { dg-final { scan-tree-dump {(?n)#pragma omp target oacc_parallel map\(from:array_so \[len: 4\]\) firstprivate\(} omplower } }
+     { dg-final { scan-tree-dump {(?n)#pragma omp target oacc_parallel firstprivate\([^)]+\) map\(from:array_so \[len: 4\]\)} omplower } }
      (C++ computes an intermediate value, so can't scan for 'firstprivate(array_li)'.)  */
   {
     array_so = sizeof array;
diff --git a/gcc/testsuite/g++.dg/gomp/attrs-1.C b/gcc/testsuite/g++.dg/gomp/attrs-1.C
index 2a5f2cf..319ad32 100644
--- a/gcc/testsuite/g++.dg/gomp/attrs-1.C
+++ b/gcc/testsuite/g++.dg/gomp/attrs-1.C
@@ -211,7 +211,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ll++;
   [[omp::sequence (directive (target teams
     device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp)
-    shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) nowait depend(inout: dd[0])
+    shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) nowait depend(inout: dd[0])
     allocate (omp_default_mem_alloc:f) in_reduction(+:r2)))]]
     ;
   [[omp::sequence (directive (target
@@ -226,7 +226,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ;
   [[omp::directive (target teams distribute parallel for
     device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp)
-    shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl)
+    shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl)
     collapse(1) dist_schedule(static, 16)
     if (parallel: i2) num_threads (nth) proc_bind(spread)
     lastprivate (l) schedule(static, 4) nowait depend(inout: dd[0]) order(concurrent)
@@ -245,7 +245,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ll++;
   [[omp::directive (target teams distribute simd
     device(d) map (tofrom: m) if (i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp)
-    shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl)
+    shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl)
     collapse(1) dist_schedule(static, 16) order(concurrent)
     safelen(8) simdlen(4) aligned(q: 32) nowait depend(inout: dd[0]) nontemporal(ntm)
     allocate (omp_default_mem_alloc:f) in_reduction(+:r2))]]
@@ -309,7 +309,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   for (int i = 0; i < 64; i++)
     ;
   [[omp::directive (teams
-    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl)
+    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl)
     allocate (omp_default_mem_alloc: f))]]
     ;
   [[omp::sequence (omp::directive (target),
@@ -322,7 +322,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ll++;
   [[omp::sequence (directive (target),
     directive (teams distribute parallel for simd
-    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl)
+    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl)
     collapse(1) dist_schedule(static, 16)
     if (parallel: i2) num_threads (nth) proc_bind(spread)
     lastprivate (l) schedule(static, 4) order(concurrent)
@@ -339,7 +339,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   for (int i = 0; i < 64; i++)
     ll++;
   [[omp::directive (teams distribute parallel for
-    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl)
+    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl)
     collapse(1) dist_schedule(static, 16)
     if (parallel: i2) num_threads (nth) proc_bind(spread)
     lastprivate (l) schedule(static, 4) copyin(t) allocate (f))]]
@@ -353,7 +353,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   for (int i = 0; i < 64; i++)
     ll++;
   [[omp::directive (teams distribute parallel for simd
-    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl)
+    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl)
     collapse(1) dist_schedule(static, 16)
     if (parallel: i2) num_threads (nth) proc_bind(spread)
     lastprivate (l) schedule(static, 4)
@@ -371,7 +371,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   for (int i = 0; i < 64; i++)
     ll++;
   [[omp::directive (teams distribute simd
-    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl)
+    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl)
     collapse(1) dist_schedule(static, 16) order(concurrent)
     safelen(8) simdlen(4) aligned(q: 32) if(i3) nontemporal(ntm) allocate(f))]]
   for (int i = 0; i < 64; i++)
@@ -507,7 +507,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   for (l = 0; l < 64; ++l)
     ;
   [[omp::directive (teams loop
-    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl)
+    private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte - 1:nte) thread_limit(tl)
     collapse(1) lastprivate (l) order(concurrent) allocate (f))]]
   for (l = 0; l < 64; ++l)
     ;
@@ -534,7 +534,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ;
   [[omp::directive (target teams loop
     device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp)
-    shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) nowait depend(inout: dd[0])
+    shared(s) default(shared) reduction(+:r) num_teams(nte - 1 : nte) thread_limit(tl) nowait depend(inout: dd[0])
     lastprivate (l) order(concurrent) collapse(1)
     allocate (omp_default_mem_alloc: f) in_reduction(+:r2))]]
   for (l = 0; l < 64; ++l)
diff --git a/gcc/testsuite/g++.dg/gomp/attrs-2.C b/gcc/testsuite/g++.dg/gomp/attrs-2.C
index c00be7f..955b2dd 100644
--- a/gcc/testsuite/g++.dg/gomp/attrs-2.C
+++ b/gcc/testsuite/g++.dg/gomp/attrs-2.C
@@ -220,7 +220,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ;
   [[omp::sequence (omp::directive (target teams distribute,
     device(d),map (tofrom: m),if (target: i1),private (p),firstprivate (f),defaultmap(tofrom: scalar),is_device_ptr (idp),
-    shared(s),default(shared),reduction(+:r),num_teams(nte),thread_limit(tl),order(concurrent),
+    shared(s),default(shared),reduction(+:r),num_teams(nte-1:nte),thread_limit(tl),order(concurrent),
     collapse(1),dist_schedule(static, 16),nowait depend(inout: dd[0]),allocate (omp_default_mem_alloc:f),in_reduction(+:r2)))]]
   for (int i = 0; i < 64; i++)
     ;
@@ -235,7 +235,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ll++;
   [[omp::directive (target teams distribute parallel for simd,
     device(d),map (tofrom: m),if (target: i1),private (p),firstprivate (f),defaultmap(tofrom: scalar),is_device_ptr (idp),
-    shared(s),default(shared),reduction(+:r),num_teams(nte),thread_limit(tl),
+    shared(s),default(shared),reduction(+:r),num_teams(2:nte),thread_limit(tl),
     collapse(1),dist_schedule(static, 16),
     if (parallel: i2),num_threads (nth),proc_bind(spread),
     lastprivate (l),schedule(static, 4),order(concurrent),
@@ -304,7 +304,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   [[omp::directive (taskwait)]];
   [[omp::sequence (directive (target, nowait,depend(inout: dd[0]),in_reduction(+:r2)),
     directive (teams distribute,
-    private(p),firstprivate (f),shared(s),default(shared),reduction(+:r),num_teams(nte),thread_limit(tl),
+    private(p),firstprivate(f),shared(s),default(shared),reduction(+:r),num_teams(nte - 1 : nte),thread_limit(tl),
     collapse(1),dist_schedule(static, 16),allocate (omp_default_mem_alloc: f),order(concurrent)))]]
   for (int i = 0; i < 64; i++)
     ;
@@ -314,7 +314,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ;
   [[omp::sequence (omp::directive (target),
     omp::directive (teams distribute parallel for,
-    private(p),firstprivate (f),shared(s),default(shared),reduction(+:r),num_teams(nte),thread_limit(tl),
+    private(p),firstprivate(f),shared(s),default(shared),reduction(+:r),num_teams(16:nte),thread_limit(tl),
     collapse(1),dist_schedule(static, 16),
     if (parallel: i2),num_threads (nth),proc_bind(spread),
     lastprivate (l),schedule(static, 4),order(concurrent),allocate (omp_default_mem_alloc: f)))]]
@@ -332,7 +332,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ll++;
   [[omp::sequence (directive (target),
     directive (teams distribute simd,
-    private(p),firstprivate (f),shared(s),default(shared),reduction(+:r),num_teams(nte),thread_limit(tl),
+    private(p),firstprivate(f),shared(s),default(shared),reduction(+:r),num_teams(nte-1:nte),thread_limit(tl),
     collapse(1),dist_schedule(static, 16),order(concurrent),
     safelen(8),simdlen(4),aligned(q: 32),if(i3),nontemporal(ntm),
     allocate (omp_default_mem_alloc: f)))]]
@@ -346,7 +346,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   for (int i = 0; i < 64; i++)
     ll++;
   [[omp::directive (teams distribute parallel for,
-    private(p),firstprivate (f),shared(s),default(shared),reduction(+:r),num_teams(nte),thread_limit(tl),
+    private(p),firstprivate (f),shared(s),default(shared),reduction(+:r),num_teams(nte-1:nte),thread_limit(tl),
     collapse(1),dist_schedule(static, 16),order(concurrent),
     if (parallel: i2),num_threads (nth),proc_bind(spread),
     lastprivate (l),schedule(static, 4),allocate (f))]]
@@ -362,7 +362,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   for (int i = 0; i < 64; i++)
     ll++;
   [[omp::directive (teams distribute parallel for simd,
-    private(p),firstprivate (f),shared(s),default(shared),reduction(+:r),num_teams(nte),thread_limit(tl),
+    private(p),firstprivate (f),shared(s),default(shared),reduction(+:r),num_teams(nte-1:nte),thread_limit(tl),
     collapse(1),dist_schedule(static, 16),
     if (parallel: i2),num_threads (nth),proc_bind(spread),
     lastprivate (l),schedule(static, 4),order(concurrent),
@@ -502,7 +502,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
   for (l = 0; l < 64; l++)
     ll++;
   [[omp::directive (teams loop,
-    private(p),firstprivate (f),shared(s),default(shared),reduction(+:r),num_teams(nte),thread_limit(tl),
+    private(p),firstprivate (f),shared(s),default(shared),reduction(+:r),num_teams(nte-1:nte),thread_limit(tl),
     collapse(1),lastprivate (l),bind(teams),allocate (f))]]
   for (l = 0; l < 64; ++l)
     ;
@@ -527,7 +527,7 @@ bar (int d, int m, int i1, int i2, int i3, int p, int *idp, int s,
     ;
   [[omp::directive (target teams loop,
     device(d),map (tofrom: m),if (target: i1),private (p),firstprivate (f),defaultmap(tofrom: scalar),is_device_ptr (idp),
-    shared(s),default(shared),reduction(+:r),num_teams(nte),thread_limit(tl),nowait,depend(inout: dd[0]),
+    shared(s),default(shared),reduction(+:r),num_teams(nte-1:nte),thread_limit(tl),nowait,depend(inout: dd[0]),
     lastprivate (l),bind(teams),collapse(1),
     allocate (omp_default_mem_alloc: f),in_reduction(+:r2))]]
   for (l = 0; l < 64; ++l)
diff --git a/gcc/testsuite/g++.dg/gomp/loop-3.C b/gcc/testsuite/g++.dg/gomp/loop-3.C
index 3806e1f..d9b8465 100644
--- a/gcc/testsuite/g++.dg/gomp/loop-3.C
+++ b/gcc/testsuite/g++.dg/gomp/loop-3.C
@@ -116,7 +116,7 @@ f1 (I<int> &x, I<int> &y, I<int> &u, I<int> &v)
     for (j = x; j < y; j++)
       ;
   #pragma omp for collapse(2)
-  for (i = x; i < y; i = i + 2) /* { dg-error "initializer expression refers to iteration variable" } */
+  for (i = x; i < y; i = i + 2)
     for (j = i; j < v; j += 2)
       ;
   #pragma omp for collapse(2)
@@ -128,11 +128,11 @@ f1 (I<int> &x, I<int> &y, I<int> &u, I<int> &v)
     for (j = baz (&i); j < v; j += 2) /* { dg-error "initializer expression refers to iteration variable" } */
       ;
   #pragma omp for collapse(2)
-  for (i = x; i < y; i++) /* { dg-error "condition expression refers to iteration variable" } */
+  for (i = x; i < y; i++)
     for (j = v; j > i; j--)
       ;
   #pragma omp for collapse(2)
-  for (i = x; i < y; i++) /* { dg-error "condition expression refers to iteration variable" } */
+  for (i = x; i < y; i++)
     for (j = x; j < i; j++)
       ;
   #pragma omp for collapse(2)
@@ -234,7 +234,7 @@ f2 (I<int> &x, I<int> &y, I<int> &u, I<int> &v)
     for (I<int> j = u; j < y; j += 2)
       ;
   #pragma omp for collapse(2)
-  for (I<int> i = x; i < y; i = i + 2) /* { dg-error "initializer expression refers to iteration variable" } */
+  for (I<int> i = x; i < y; i = i + 2)
     for (I<int> j = i; j < v; j += 2)
       ;
   #pragma omp for collapse(2)
@@ -246,11 +246,11 @@ f2 (I<int> &x, I<int> &y, I<int> &u, I<int> &v)
     for (I<int> j = baz (&i); j < v; j += 2) /* { dg-error "initializer expression refers to iteration variable" } */
       ;
   #pragma omp for collapse(2)
-  for (I<int> i = x; i < y; i++) /* { dg-error "condition expression refers to iteration variable" } */
+  for (I<int> i = x; i < y; i++)
     for (I<int> j = v; j > i; j--)
       ;
   #pragma omp for collapse(2)
-  for (I<int> i = x; i < y; i++) /* { dg-error "condition expression refers to iteration variable" } */
+  for (I<int> i = x; i < y; i++)
     for (I<int> j = x; j < i; j++)
       ;
   #pragma omp for collapse(2)
diff --git a/gcc/testsuite/g++.dg/gomp/loop-7.C b/gcc/testsuite/g++.dg/gomp/loop-7.C
new file mode 100644
index 0000000..9466c1c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/gomp/loop-7.C
@@ -0,0 +1,22 @@
+// PR c++/102854
+// { dg-do compile }
+
+template <typename T>
+void
+foo (T N, T M)
+{
+  #pragma omp parallel for collapse(2)
+  for (T i = 0; i < N; ++i)
+    for (T k = i; k < M; ++k)
+      ;
+  #pragma omp parallel for collapse(2)
+  for (T i = 0; i < N; ++i)
+    for (T k = i; k < 2 * i; ++k)
+      ;
+}
+
+void
+bar ()
+{
+  foo (5, 10);
+}
diff --git a/gcc/testsuite/g++.dg/gomp/loop-8.C b/gcc/testsuite/g++.dg/gomp/loop-8.C
new file mode 100644
index 0000000..41a1817
--- /dev/null
+++ b/gcc/testsuite/g++.dg/gomp/loop-8.C
@@ -0,0 +1,128 @@
+typedef __PTRDIFF_TYPE__ ptrdiff_t;
+
+template <typename T>
+class I
+{
+public:
+  typedef ptrdiff_t difference_type;
+  I ();
+  ~I ();
+  I (T *);
+  I (const I &);
+  T &operator * ();
+  T *operator -> ();
+  T &operator [] (const difference_type &) const;
+  I &operator = (const I &);
+  I &operator ++ ();
+  I operator ++ (int);
+  I &operator -- ();
+  I operator -- (int);
+  I &operator += (const difference_type &);
+  I &operator -= (const difference_type &);
+  I operator + (const difference_type &) const;
+  I operator - (const difference_type &) const;
+  template <typename S> friend bool operator == (I<S> &, I<S> &);
+  template <typename S> friend bool operator == (const I<S> &, const I<S> &);
+  template <typename S> friend bool operator < (I<S> &, I<S> &);
+  template <typename S> friend bool operator < (const I<S> &, const I<S> &);
+  template <typename S> friend bool operator <= (I<S> &, I<S> &);
+  template <typename S> friend bool operator <= (const I<S> &, const I<S> &);
+  template <typename S> friend bool operator > (I<S> &, I<S> &);
+  template <typename S> friend bool operator > (const I<S> &, const I<S> &);
+  template <typename S> friend bool operator >= (I<S> &, I<S> &);
+  template <typename S> friend bool operator >= (const I<S> &, const I<S> &);
+  template <typename S> friend typename I<S>::difference_type operator - (I<S> &, I<S> &);
+  template <typename S> friend typename I<S>::difference_type operator - (const I<S> &, const I<S> &);
+  template <typename S> friend I<S> operator + (typename I<S>::difference_type , const I<S> &);
+private:
+  T *p;
+};
+
+template <typename T> bool operator == (I<T> &, I<T> &);
+template <typename T> bool operator == (const I<T> &, const I<T> &);
+template <typename T> bool operator != (I<T> &, I<T> &);
+template <typename T> bool operator != (const I<T> &, const I<T> &);
+template <typename T> bool operator < (I<T> &, I<T> &);
+template <typename T> bool operator < (const I<T> &, const I<T> &);
+template <typename T> bool operator <= (I<T> &, I<T> &);
+template <typename T> bool operator <= (const I<T> &, const I<T> &);
+template <typename T> bool operator > (I<T> &, I<T> &);
+template <typename T> bool operator > (const I<T> &, const I<T> &);
+template <typename T> bool operator >= (I<T> &, I<T> &);
+template <typename T> bool operator >= (const I<T> &, const I<T> &);
+template <typename T> typename I<T>::difference_type operator - (I<T> &, I<T> &);
+template <typename T> typename I<T>::difference_type operator - (const I<T> &, const I<T> &);
+template <typename T> I<T> operator + (typename I<T>::difference_type, const I<T> &);
+
+extern I<int> i, j;
+#pragma omp threadprivate (i, j)
+extern I<int> k, l;
+#pragma omp threadprivate (k, l)
+I<int> k, l;
+
+void
+f1 (I<int> &x, I<int> &y)
+{
+  #pragma omp for collapse(2)
+  for (i = x; i < y; i++)	// { dg-error "expected iteration declaration or initialization" }
+    for (j = x; j < y; j++)
+      ;
+}
+
+void
+f2 (I<int> &x, I<int> &y)
+{
+  #pragma omp for collapse(2)
+  for (k = x; k < y; k++)	// { dg-error "expected iteration declaration or initialization" }
+    for (l = x; l < y; l++)
+      ;
+}
+
+template <int N>
+void
+f3 (I<int> &x, I<int> &y)
+{
+  #pragma omp for collapse(2)
+  for (i = x; i < y; i++)	// { dg-error "'i' is predetermined 'threadprivate' for 'private'" }
+    for (j = x; j < y; j++)	// { dg-error "'j' is predetermined 'threadprivate' for 'private'" }
+      ;
+}
+
+template <int N>
+void
+f4 (I<int> &x, I<int> &y)
+{
+  #pragma omp for collapse(2)
+  for (k = x; k < y; k++)	// { dg-error "'k' is predetermined 'threadprivate' for 'private'" }
+    for (l = x; l < y; l++)	// { dg-error "'l' is predetermined 'threadprivate' for 'private'" }
+      ;
+}
+
+template <typename T>
+void
+f5 (I<T> &x, I<T> &y)
+{
+  #pragma omp for collapse(2)	// { dg-error "expected iteration declaration or initialization" }
+  for (i = x; i < y; i++)	// { dg-error "'i' is predetermined 'threadprivate' for 'private'" }
+    for (j = x; j < y; j++)	// { dg-error "'j' is predetermined 'threadprivate' for 'private'" }
+      ;
+}
+
+template <typename T>
+void
+f6 (I<T> &x, I<T> &y)
+{
+  #pragma omp for collapse(2)	// { dg-error "expected iteration declaration or initialization" }
+  for (k = x; k < y; k++)	// { dg-error "'k' is predetermined 'threadprivate' for 'private'" }
+    for (l = x; l < y; l++)	// { dg-error "'l' is predetermined 'threadprivate' for 'private'" }
+      ;
+}
+
+void
+test (I<int> &x, I<int> &y)
+{
+  f3<0> (x, y);
+  f4<0> (x, y);
+  f5 (x, y);
+  f6 (x, y);
+}
diff --git a/gcc/testsuite/g++.dg/gomp/num-teams-1.C b/gcc/testsuite/g++.dg/gomp/num-teams-1.C
new file mode 100644
index 0000000..5b36ffb
--- /dev/null
+++ b/gcc/testsuite/g++.dg/gomp/num-teams-1.C
@@ -0,0 +1,122 @@
+int fn1 (int);
+template <typename T>
+T fn2 (T);
+
+template <int N>
+void
+f1 ()
+{
+  #pragma omp teams num_teams (4 : 6)
+  ;
+  #pragma omp teams num_teams (7)
+  ;
+}
+
+template <int N>
+void
+f2 ()
+{
+  #pragma omp target teams num_teams (5 : 19)
+  ;
+  #pragma omp target teams num_teams (21)
+  ;
+}
+
+template <int N>
+void
+f3 ()
+{
+  #pragma omp teams num_teams (fn1 (1) : fn1 (2))
+  ;
+  #pragma omp teams num_teams (fn1 (3))
+  ;
+}
+
+template <int N>
+void
+f4 ()
+{
+  #pragma omp target teams num_teams (fn1 (4) : fn1 (5))
+  ;
+  #pragma omp target teams num_teams (fn1 (6))
+  ;
+}
+
+template <int N>
+void
+f5 ()
+{
+  #pragma omp target
+  #pragma omp teams num_teams (fn1 (7) : fn1 (8))
+  ;
+  #pragma omp target
+  #pragma omp teams num_teams (fn1 (9))
+  ;
+}
+
+template <typename T, T N4, T N6, T N7>
+void
+f1 ()
+{
+  #pragma omp teams num_teams (N4 : N6)
+  ;
+  #pragma omp teams num_teams (N7)
+  ;
+}
+
+template <typename T, T N5, T N19, T N21>
+void
+f2 ()
+{
+  #pragma omp target teams num_teams (N5 : N19)
+  ;
+  #pragma omp target teams num_teams (N21)
+  ;
+}
+
+template <typename T, T N1, T N2, T N3>
+void
+f3 ()
+{
+  #pragma omp teams num_teams (fn2 (N1) : fn2 (N2))
+  ;
+  #pragma omp teams num_teams (fn2 (N3))
+  ;
+}
+
+template <typename T, T N4, T N5, T N6>
+void
+f4 ()
+{
+  #pragma omp target teams num_teams (fn2 (N4) : fn2 (N5))
+  ;
+  #pragma omp target teams num_teams (fn2 (N6))
+  ;
+}
+
+template <typename T, T N7, T N8, T N9>
+void
+f5 ()
+{
+  #pragma omp target
+  #pragma omp teams num_teams (fn2 (N7) : fn2 (N8))
+  ;
+  #pragma omp target
+  #pragma omp teams num_teams (fn2 (N9))
+  ;
+}
+
+void
+test ()
+{
+  f1<0> ();
+  f2<0> ();
+  f3<0> ();
+  f4<0> ();
+  f5<0> ();
+  f1<int, 4, 6, 7> ();
+  f2<int, 5, 19, 21> ();
+  f3<int, 1, 2, 3> ();
+  f4<int, 4, 5, 6> ();
+  f5<int, 7, 8, 9> ();
+}
diff --git a/gcc/testsuite/g++.dg/gomp/num-teams-2.C b/gcc/testsuite/g++.dg/gomp/num-teams-2.C
new file mode 100644
index 0000000..8b8933c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/gomp/num-teams-2.C
@@ -0,0 +1,64 @@
+template <int N>
+void
+foo (int i)
+{
+  #pragma omp teams num_teams (6 : 4)		// { dg-warning "'num_teams' lower bound '6' bigger than upper bound '4'" }
+  ;
+  #pragma omp teams num_teams (-7)		// { dg-warning "'num_teams' value must be positive" }
+  ;
+  #pragma omp teams num_teams (i : -7)		// { dg-warning "'num_teams' value must be positive" }
+  ;
+  #pragma omp teams num_teams (-7 : 8)		// { dg-warning "'num_teams' value must be positive" }
+  ;
+}
+
+template <int N>
+void
+bar (int i)
+{
+  #pragma omp target teams num_teams (6 : 4)	// { dg-warning "'num_teams' lower bound '6' bigger than upper bound '4'" }
+  ;
+  #pragma omp target teams num_teams (-7)	// { dg-warning "'num_teams' value must be positive" }
+  ;
+  #pragma omp target teams num_teams (i : -7)	// { dg-warning "'num_teams' value must be positive" }
+  ;
+  #pragma omp target teams num_teams (-7 : 8)	// { dg-warning "'num_teams' value must be positive" }
+  ;
+}
+
+template <typename T, T NM7, T N4, T N6, T N8>
+void
+baz (T i)
+{
+  #pragma omp teams num_teams (N6 : N4)		// { dg-warning "'num_teams' lower bound '6' bigger than upper bound '4'" }
+  ;
+  #pragma omp teams num_teams (NM7)		// { dg-warning "'num_teams' value must be positive" }
+  ;
+  #pragma omp teams num_teams (i : NM7)		// { dg-warning "'num_teams' value must be positive" }
+  ;
+  #pragma omp teams num_teams (NM7 : N8)	// { dg-warning "'num_teams' value must be positive" }
+  ;
+}
+
+template <typename T, T NM7, T N4, T N6, T N8>
+void
+qux (T i)
+{
+  #pragma omp target teams num_teams (N6 : N4)	// { dg-warning "'num_teams' lower bound '6' bigger than upper bound '4'" }
+  ;
+  #pragma omp target teams num_teams (NM7)	// { dg-warning "'num_teams' value must be positive" }
+  ;
+  #pragma omp target teams num_teams (i : NM7)	// { dg-warning "'num_teams' value must be positive" }
+  ;
+  #pragma omp target teams num_teams (NM7 : N8)	// { dg-warning "'num_teams' value must be positive" }
+  ;
+}
+
+void
+test ()
+{
+  foo<0> (5);
+  bar<0> (5);
+  baz<int, -7, 4, 6, 8> (5);
+  qux<int, -7, 4, 6, 8> (5);
+}
diff --git a/gcc/testsuite/g++.dg/init/array41a.C b/gcc/testsuite/g++.dg/init/array41a.C
new file mode 100644
index 0000000..aa9fdc6
--- /dev/null
+++ b/gcc/testsuite/g++.dg/init/array41a.C
@@ -0,0 +1,27 @@
+// PR c++/70690
+// { dg-do run { target c++11 } }
+
+struct A {
+  constexpr A() {}
+};
+
+struct APadded : public A {
+  char pad[63];
+};
+
+int f();
+int i = f();
+APadded cache[50];
+APadded *p = cache;
+
+int f()
+{
+  cache[0].pad[0] = 42;
+  return 1;
+}
+
+int main()
+{
+  if (cache[0].pad[0] != 42)
+    __builtin_abort();
+}
diff --git a/gcc/testsuite/g++.dg/init/bitfield6.C b/gcc/testsuite/g++.dg/init/bitfield6.C
new file mode 100644
index 0000000..70854ee
--- /dev/null
+++ b/gcc/testsuite/g++.dg/init/bitfield6.C
@@ -0,0 +1,20 @@
+// { dg-do compile { target c++11 } }
+
+struct A
+{
+  int : 8;
+  int i = 42;
+};
+
+constexpr A a;
+
+struct B : A {
+};
+
+constexpr B b;
+
+struct C {
+  int : 0;
+};
+
+constexpr C c;
diff --git a/gcc/testsuite/g++.dg/ipa/devirt-35.C b/gcc/testsuite/g++.dg/ipa/devirt-35.C
index 87f72b8..ca9ccb7 100644
--- a/gcc/testsuite/g++.dg/ipa/devirt-35.C
+++ b/gcc/testsuite/g++.dg/ipa/devirt-35.C
@@ -15,7 +15,6 @@ m(struct B *b)
                   //  test2 may change the type of A by placement new.
                   // C++ standard is bit imprecise about this.
 }
-/* { dg-final { scan-tree-dump "converting indirect call to function virtual int B::t"  "fre1"  } } */
-/* { dg-final { scan-ipa-dump "to virtual int B::t"  "devirt"  } } */
+/* { dg-final { scan-tree-dump "converting indirect call to function virtual int B::t"  "fre1" { target { ! implicit_constexpr } } } } */
+/* { dg-final { scan-ipa-dump "to virtual int B::t"  "devirt" { target { ! implicit_constexpr } } } } */
 /* { dg-final { scan-ipa-dump "1 speculatively devirtualized"  "devirt"  } } */
-
diff --git a/gcc/testsuite/g++.dg/ipa/devirt-36.C b/gcc/testsuite/g++.dg/ipa/devirt-36.C
index 067a2bb..60b8301 100644
--- a/gcc/testsuite/g++.dg/ipa/devirt-36.C
+++ b/gcc/testsuite/g++.dg/ipa/devirt-36.C
@@ -18,7 +18,7 @@ m(struct B *b)
                   //  test2 may change the type of A by placement new.
                   // C++ standard is bit imprecise about this.
 }
-/* { dg-final { scan-tree-dump "converting indirect call to function virtual int B::t"  "fre1"  } } */
-/* { dg-final { scan-ipa-dump "to virtual int B::t"  "devirt"  } } */
+/* { dg-final { scan-tree-dump "converting indirect call to function virtual int B::t"  "fre1" { target { ! implicit_constexpr } } } } */
+/* { dg-final { scan-ipa-dump "to virtual int B::t"  "devirt" { target { ! implicit_constexpr } } } } */
 /* { dg-final { scan-ipa-dump "1 speculatively devirtualized"  "devirt"  } } */
 
diff --git a/gcc/testsuite/g++.dg/ipa/devirt-37.C b/gcc/testsuite/g++.dg/ipa/devirt-37.C
index b7f52a0..df5ab90 100644
--- a/gcc/testsuite/g++.dg/ipa/devirt-37.C
+++ b/gcc/testsuite/g++.dg/ipa/devirt-37.C
@@ -33,4 +33,4 @@ t()
 /* { dg-final { scan-tree-dump "No dynamic type change found."  "fre3"  } } */
 /* { dg-final { scan-tree-dump "Checking vtbl store:"  "fre3"  } } */
 /* { dg-final { scan-tree-dump "Function call may change dynamic type:extcall"  "fre3"  } } */
-/* { dg-final { scan-tree-dump "converting indirect call to function virtual void"  "fre3"  } } */
+/* { dg-final { scan-tree-dump "converting indirect call to function virtual void"  "fre3" { target { ! implicit_constexpr } } } } */
diff --git a/gcc/testsuite/g++.dg/ipa/devirt-44.C b/gcc/testsuite/g++.dg/ipa/devirt-44.C
index 5de7614..ed211e1 100644
--- a/gcc/testsuite/g++.dg/ipa/devirt-44.C
+++ b/gcc/testsuite/g++.dg/ipa/devirt-44.C
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O3 -fno-ipa-cp -fdump-ipa-inline-details -fno-early-inlining" } */
+/* { dg-options "-O3 -fno-ipa-cp -fdump-ipa-inline-details -fno-early-inlining -fno-implicit-constexpr" } */
 struct A {
   virtual int foo () {return 1;}
   void wrapfoo () {foo();}
diff --git a/gcc/testsuite/g++.dg/ipa/imm-devirt-1.C b/gcc/testsuite/g++.dg/ipa/imm-devirt-1.C
index 00ac61e7..fa31cfb 100644
--- a/gcc/testsuite/g++.dg/ipa/imm-devirt-1.C
+++ b/gcc/testsuite/g++.dg/ipa/imm-devirt-1.C
@@ -60,7 +60,7 @@ int main (int argc, char *argv[])
 
 /* middleman_2 gets early inlined and the virtual call should get turned to
    a direct call.  */
-/* { dg-final { scan-tree-dump "Inlining int middleman_1" "einline"  } } */
-/* { dg-final { scan-tree-dump "Inlining int middleman_2" "einline"  } } */
+/* { dg-final { scan-tree-dump "Inlining int middleman_1" "einline" { target { ! implicit_constexpr } } } } */
+/* { dg-final { scan-tree-dump "Inlining int middleman_2" "einline" { target { ! implicit_constexpr } } } } */
 /* { dg-final { scan-tree-dump "B::foo \\(" "einline"  } } */
 /* { dg-final { scan-tree-dump-times "OBJ_TYPE_REF" 2 "einline"  } } */
diff --git a/gcc/testsuite/g++.dg/ipa/ipa-icf-4.C b/gcc/testsuite/g++.dg/ipa/ipa-icf-4.C
index 0b49c6f..b4e1958 100644
--- a/gcc/testsuite/g++.dg/ipa/ipa-icf-4.C
+++ b/gcc/testsuite/g++.dg/ipa/ipa-icf-4.C
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-ipa-icf-optimized-missed -fno-inline" } */
+/* { dg-options "-O2 -fdump-ipa-icf-optimized-missed -fno-inline -fno-ipa-modref" } */
 
 namespace {
 struct A
diff --git a/gcc/testsuite/g++.dg/ipa/modref-1.C b/gcc/testsuite/g++.dg/ipa/modref-1.C
new file mode 100644
index 0000000..b492381
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ipa/modref-1.C
@@ -0,0 +1,35 @@
+/* { dg-do link } */
+/* { dg-options "-O2 -fdump-tree-local-pure-const1 -fdump-tree-modref1 -std=gnu++2a"  } */
+namespace {
+struct B {
+	int b;
+	struct B *bptr;
+	B() {b=1; }
+	B(B &src)
+	{
+		b=src.b;
+		bptr=0;
+	}
+	__attribute__ ((noinline))
+	static struct B genB()
+	{
+		struct B b;
+		b.b=2;
+		b.bptr = 0;
+		return b;
+	}
+};
+}
+void linker_error ();
+int main()
+{
+	struct B b1 = B::genB();
+	b1.b = 1;
+	struct B b2 = B::genB();
+	if (b1.b != 1 || b2.bptr == &b2)
+		linker_error ();
+	return 0;
+}
+/* { dg-final { scan-tree-dump "Function found to be const: static {anonymous}::B {anonymous}::B::genB" "local-pure-const1"  } } */
+/* { dg-final { scan-tree-dump "Retslot flags: not_returned_directly" "modref1" } } */
+  
diff --git a/gcc/testsuite/g++.dg/ipa/pr103099.C b/gcc/testsuite/g++.dg/ipa/pr103099.C
new file mode 100644
index 0000000..5fb137d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ipa/pr103099.C
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -g" } */
+
+void pthread_mutex_unlock(int *);
+int __gthread_mutex_unlock___mutex, unlock___trans_tmp_1;
+struct Object {
+  void _change_notify() {}
+  bool _is_queued_for_deletion;
+};
+struct ClassDB {
+  template <class N, class M> static int bind_method(N, M);
+};
+struct CanvasItemMaterial : Object {
+  bool particles_animation;
+  void set_particles_animation(bool);
+};
+void CanvasItemMaterial::set_particles_animation(bool p_particles_anim) {
+  particles_animation = p_particles_anim;
+  if (unlock___trans_tmp_1)
+    pthread_mutex_unlock(&__gthread_mutex_unlock___mutex);
+  _change_notify();
+}
+void CanvasItemMaterial_bind_methods() {
+  ClassDB::bind_method("", &CanvasItemMaterial::set_particles_animation);
+}
diff --git a/gcc/testsuite/g++.dg/lookup/builtin5.C b/gcc/testsuite/g++.dg/lookup/builtin5.C
index 1bd67dc..652e3f5 100644
--- a/gcc/testsuite/g++.dg/lookup/builtin5.C
+++ b/gcc/testsuite/g++.dg/lookup/builtin5.C
@@ -1,5 +1,5 @@
 // PR c++/37276
-
+// { dg-additional-options -fno-inline }
 // { dg-final { scan-assembler "_ZSt5atanhd" } }
 
 namespace std
diff --git a/gcc/testsuite/g++.dg/lto/inline-crossmodule-1_0.C b/gcc/testsuite/g++.dg/lto/inline-crossmodule-1_0.C
index adbc43e..0294dcc 100644
--- a/gcc/testsuite/g++.dg/lto/inline-crossmodule-1_0.C
+++ b/gcc/testsuite/g++.dg/lto/inline-crossmodule-1_0.C
@@ -1,5 +1,5 @@
 // { dg-lto-do link }
-/* { dg-lto-options { "-O2 -fno-early-inlining -flto -fdump-ipa-inline-details" } } */
+/* { dg-lto-options { "-O2 -fno-early-inlining -fno-implicit-constexpr -flto -fdump-ipa-inline-details" } } */
 #include "inline-crossmodule-1.h"
 int a::key ()
 {
diff --git a/gcc/testsuite/g++.dg/modules/enum-1_a.C b/gcc/testsuite/g++.dg/modules/enum-1_a.C
index 53e2ac8..24cad28 100644
--- a/gcc/testsuite/g++.dg/modules/enum-1_a.C
+++ b/gcc/testsuite/g++.dg/modules/enum-1_a.C
@@ -1,5 +1,5 @@
 // { dg-module-do run }
-// { dg-additional-options "-fmodules-ts -fdump-lang-module-uid" }
+// { dg-additional-options "-fmodules-ts -fdump-lang-module-uid -fno-implicit-constexpr" }
 export module enUm;
 // { dg-module-cmi "enUm" }
 
diff --git a/gcc/testsuite/g++.dg/modules/fn-inline-1_c.C b/gcc/testsuite/g++.dg/modules/fn-inline-1_c.C
index 55a7aaa..2b8bbdc 100644
--- a/gcc/testsuite/g++.dg/modules/fn-inline-1_c.C
+++ b/gcc/testsuite/g++.dg/modules/fn-inline-1_c.C
@@ -1,4 +1,4 @@
-// { dg-additional-options "-fmodules-ts" }
+// { dg-additional-options "-fmodules-ts -fno-inline" }
 import bob;
 
 int main ()
diff --git a/gcc/testsuite/g++.dg/modules/pmf-1_a.H b/gcc/testsuite/g++.dg/modules/pmf-1_a.H
index c597db1..a7b3fc3 100644
--- a/gcc/testsuite/g++.dg/modules/pmf-1_a.H
+++ b/gcc/testsuite/g++.dg/modules/pmf-1_a.H
@@ -1,4 +1,4 @@
-// { dg-additional-options -fmodule-header }
+// { dg-additional-options "-fmodule-header -fno-implicit-constexpr" }
 // { dg-module-cmi {} }
 
 #include "pmf-1.h"
diff --git a/gcc/testsuite/g++.dg/modules/pmf-1_b.C b/gcc/testsuite/g++.dg/modules/pmf-1_b.C
index 0b08610..cb2a35b 100644
--- a/gcc/testsuite/g++.dg/modules/pmf-1_b.C
+++ b/gcc/testsuite/g++.dg/modules/pmf-1_b.C
@@ -1,4 +1,4 @@
-// { dg-additional-options "-fmodules-ts -fno-module-lazy -fdump-lang-module-alias" }
+// { dg-additional-options "-fmodules-ts -fno-module-lazy -fdump-lang-module-alias -fno-implicit-constexpr" }
 
 #include "pmf-1.h"
 import "pmf-1_a.H";
diff --git a/gcc/testsuite/g++.dg/modules/used-1_c.C b/gcc/testsuite/g++.dg/modules/used-1_c.C
index 0d1514e..b51a19f 100644
--- a/gcc/testsuite/g++.dg/modules/used-1_c.C
+++ b/gcc/testsuite/g++.dg/modules/used-1_c.C
@@ -1,4 +1,4 @@
-// { dg-additional-options -fmodules-ts }
+// { dg-additional-options "-fmodules-ts -fno-inline" }
 
 import "used-1_b.H";
 
diff --git a/gcc/testsuite/g++.dg/opt/pr100469.C b/gcc/testsuite/g++.dg/opt/pr100469.C
new file mode 100644
index 0000000..9eb37e7
--- /dev/null
+++ b/gcc/testsuite/g++.dg/opt/pr100469.C
@@ -0,0 +1,32 @@
+// PR debug/100469
+// { dg-do compile }
+// { dg-options "-O2 -fcompare-debug -fno-tree-dse -fno-tree-forwprop -fno-tree-tail-merge --param=sccvn-max-alias-queries-per-access=0" }
+
+struct S
+{
+  long m;
+  S (const S &s)
+  {
+    m = s.m;
+  }
+  S (long l)
+  {
+    m = l;
+  }
+  bool operatorX (const S &s)
+  {
+    return m >= s.m;
+  }
+};
+
+static inline S
+bar (S a, S b)
+{
+  return a.operatorX (b) ? a : b;
+}
+
+S
+foo (S s)
+{
+  return bar (s, (S) 0);
+}
diff --git a/gcc/testsuite/g++.dg/opt/pr102970.C b/gcc/testsuite/g++.dg/opt/pr102970.C
new file mode 100644
index 0000000..2acb3c2
--- /dev/null
+++ b/gcc/testsuite/g++.dg/opt/pr102970.C
@@ -0,0 +1,60 @@
+// { dg-do run }
+// { dg-require-effective-target c++14 }
+// { dg-options "-O2 -funroll-loops -fno-tree-vectorize" }
+
+#include <vector>
+#include <algorithm>
+#include <cassert>
+
+struct box {
+   box(int xmin, int xmax, int ymin, int ymax) noexcept
+      : m_xmin(xmin),
+        m_xmax(xmax),
+        m_ymin(ymin),
+        m_ymax(ymax) {
+   }
+
+   box(box const & o) noexcept
+      : m_xmin(o.m_xmin),
+        m_xmax(o.m_xmax),
+        m_ymin(o.m_ymin),
+        m_ymax(o.m_ymax) { }
+
+   int m_xmin;
+   int m_xmax;
+   int m_ymin;
+   int m_ymax;
+};
+
+
+int main() {
+    std::vector<box> vRects{ // requires 18 elements
+        { 900, 11, 22, 33 },
+        { 901, 11, 22, 33 },
+        { 902, 11, 22, 33 },
+        { 903, 11, 22, 33 },
+        { 704, 11, 22, 33 },
+        { 705, 11, 22, 33 },
+        { 706, 11, 22, 33 },
+        { 707, 11, 22, 33 },
+        { 808, 11, 22, 33 },
+        { 809, 11, 22, 33 },
+        { 810, 11, 22, 33 },
+        { 811, 11, 22, 33 },
+        { 812, 11, 22, 33 },
+        { 813, 11, 22, 33 },
+        { 814, 11, 22, 33 },
+        { 815, 11, 22, 33 },
+        { 816, 11, 22, 33 },
+        { 817, 11, 22, 33 },
+        { 818, 11, 22, 33 },
+    };
+
+    std::stable_sort(vRects.begin(), vRects.end(),
+		     [](auto const &r1, auto const &r2) -> bool
+      {
+        if (r2.m_xmax==0||r2.m_ymin==0||r2.m_ymax==0){__builtin_abort();}
+        return r1.m_xmin < r2.m_xmin;
+      });
+  return 0;
+}
diff --git a/gcc/testsuite/g++.dg/pr102955.C b/gcc/testsuite/g++.dg/pr102955.C
new file mode 100644
index 0000000..de9689e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pr102955.C
@@ -0,0 +1,14 @@
+/* PR debug/102955 */
+/* { dg-do compile } */
+/* { dg-options "-g -gtoggle" } */
+
+#pragma GCC optimize "0"
+struct j
+{
+  explicit j ();
+  ~j ();
+};
+void g (void)
+{
+  new j();
+}
diff --git a/gcc/testsuite/g++.dg/pr103007.C b/gcc/testsuite/g++.dg/pr103007.C
new file mode 100644
index 0000000..1631a85
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pr103007.C
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3" } */
+
+typedef float MushMeshVector[4];
+struct MushMeshQuaternionPair {
+  void VectorRotate(MushMeshVector &);
+  MushMeshVector m_first;
+  MushMeshVector m_second;
+};
+void 
+MushMeshQuaternionPair::
+VectorRotate(MushMeshVector &ioVec)  {
+  ioVec[2] = (2 - m_first[1] + m_first[3] * 0);
+  ioVec[3] = (m_first[3] + m_first[1] - m_first[2] * 0);
+  float c = ioVec[2], d = ioVec[3];
+  ioVec[2] = (0 - d * m_second[1]);
+  ioVec[3] = (2 - c * m_second[1]);
+}
+
diff --git a/gcc/testsuite/g++.dg/pr103038.C b/gcc/testsuite/g++.dg/pr103038.C
new file mode 100644
index 0000000..bb7183a
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pr103038.C
@@ -0,0 +1,5 @@
+/* { dg-do compile } */
+/* { dg-options "-ftrivial-auto-var-init=pattern" } */
+
+struct S;
+void test() { int(S::*PtrMem); }
diff --git a/gcc/testsuite/g++.dg/pr103122.C b/gcc/testsuite/g++.dg/pr103122.C
new file mode 100644
index 0000000..3465ead
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pr103122.C
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+unsigned a;
+int b;
+short c;
+void d(long) {
+  for (bool e = (bool)c - 1; e < (bool)b - 1; e += 0)
+    ;
+  if (a) {
+    for (char f = 0; f < 7; f = 7)
+      for (int g = 0; g < c; g += 10)
+        ;
+    d(-!c);
+  }
+}
diff --git a/gcc/testsuite/g++.dg/tls/thread_local11.C b/gcc/testsuite/g++.dg/tls/thread_local11.C
index 273ee03..7e83a45 100644
--- a/gcc/testsuite/g++.dg/tls/thread_local11.C
+++ b/gcc/testsuite/g++.dg/tls/thread_local11.C
@@ -2,7 +2,7 @@
 // { dg-do compile { target c++11 } }
 // { dg-add-options tls }
 // { dg-require-effective-target tls_runtime }
-// { dg-additional-options "-fdump-tree-gimple" }
+// { dg-additional-options "-fdump-tree-gimple -fno-implicit-constexpr" }
 // { dg-final { scan-tree-dump-times "_ZTW2s1" 2 "gimple" } }
 // { dg-final { scan-tree-dump-times "_ZTW2s2" 2 "gimple" } }
 // { dg-final { scan-tree-dump-times "_ZTW2s3" 2 "gimple" } }
diff --git a/gcc/testsuite/g++.dg/tls/thread_local11a.C b/gcc/testsuite/g++.dg/tls/thread_local11a.C
index d8c4a6d..a628398 100644
--- a/gcc/testsuite/g++.dg/tls/thread_local11a.C
+++ b/gcc/testsuite/g++.dg/tls/thread_local11a.C
@@ -3,7 +3,7 @@
 // { dg-add-options tls }
 // { dg-require-alias "" }
 // { dg-require-effective-target tls_runtime }
-// { dg-additional-options "-fdump-tree-gimple" }
+// { dg-additional-options "-fdump-tree-gimple -fno-implicit-constexpr" }
 // { dg-final { scan-tree-dump-times "_ZTH2s1" 1 "gimple" } }
 // { dg-final { scan-tree-dump-times "_ZTH2s2" 1 "gimple" } }
 // { dg-final { scan-tree-dump-times "_ZTH2s3" 1 "gimple" } }
diff --git a/gcc/testsuite/g++.dg/tm/pr46653.C b/gcc/testsuite/g++.dg/tm/pr46653.C
index f8f3a1d..a4649eb 100644
--- a/gcc/testsuite/g++.dg/tm/pr46653.C
+++ b/gcc/testsuite/g++.dg/tm/pr46653.C
@@ -1,5 +1,5 @@
 // { dg-do compile }
-// { dg-options "-fgnu-tm -O" }
+// { dg-options "-fgnu-tm -O -fno-implicit-constexpr" }
 
 class shared_count
 {
diff --git a/gcc/testsuite/g++.dg/torture/pr102505.C b/gcc/testsuite/g++.dg/torture/pr102505.C
new file mode 100644
index 0000000..a846751
--- /dev/null
+++ b/gcc/testsuite/g++.dg/torture/pr102505.C
@@ -0,0 +1,15 @@
+struct D  { int i;  int pad alignas(16); };
+struct B : virtual D
+{
+  int j =84;
+  int k =84;
+};
+
+struct C: B { };
+
+int main()
+{
+  C c;
+  if (c.j != 84 || c.k != 84)
+    __builtin_abort();
+}
diff --git a/gcc/testsuite/g++.dg/torture/pr103040.C b/gcc/testsuite/g++.dg/torture/pr103040.C
new file mode 100644
index 0000000..d634895
--- /dev/null
+++ b/gcc/testsuite/g++.dg/torture/pr103040.C
@@ -0,0 +1,37 @@
+// { dg-do run }
+// { dg-additional-options "-fno-early-inlining" }
+struct S101273
+{
+    int x;
+    S101273* impl;
+    S101273(int x)
+    {
+        this->x = x;
+        this->impl = this;
+    }
+    S101273(const S101273 &o)
+    {
+	this->x = o.x;
+	this->impl = this;
+    }
+    ~S101273() { }
+};
+
+S101273 makeS101273()
+{
+    return S101273(2);
+}
+
+S101273 nrvo101273()
+{
+    S101273 ret = makeS101273();
+    return ret;
+}
+
+int main()
+{
+    auto nrvo = nrvo101273();
+    if(&nrvo != nrvo.impl) __builtin_abort ();
+
+    return 0;
+}
diff --git a/gcc/testsuite/g++.dg/torture/pr103073.C b/gcc/testsuite/g++.dg/torture/pr103073.C
new file mode 100644
index 0000000..02b1eee
--- /dev/null
+++ b/gcc/testsuite/g++.dg/torture/pr103073.C
@@ -0,0 +1,8 @@
+// { dg-do compile }
+int a;
+void b(bool c[], char d[], bool g[][55][21]) {
+  for (signed e = 0; e < 11; e += 3)
+    for (unsigned f = c[0] + 1; f < d[0]; f += 3)
+      a = g[0][e][f + 2];
+}
+
diff --git a/gcc/testsuite/g++.dg/tree-ssa/pr31146-2.C b/gcc/testsuite/g++.dg/tree-ssa/pr31146-2.C
index 9fb5dc1..fc26857 100644
--- a/gcc/testsuite/g++.dg/tree-ssa/pr31146-2.C
+++ b/gcc/testsuite/g++.dg/tree-ssa/pr31146-2.C
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O -fcheck-new -fno-tree-vrp -fdump-tree-forwprop1" } */
+/* { dg-options "-O -fcheck-new -fno-tree-vrp -fdump-tree-forwprop1 -fno-thread-jumps" } */
 
 #include <new>
 
diff --git a/gcc/testsuite/g++.dg/ubsan/pr70035.C b/gcc/testsuite/g++.dg/ubsan/pr70035.C
index a1d3dc2..521dcb8 100644
--- a/gcc/testsuite/g++.dg/ubsan/pr70035.C
+++ b/gcc/testsuite/g++.dg/ubsan/pr70035.C
@@ -1,7 +1,7 @@
 // PR c++/70035
 // { dg-do run }
 // { dg-shouldfail "ubsan" }
-// { dg-options "-fsanitize=vptr -fno-sanitize-recover=undefined" }
+// { dg-options "-fsanitize=vptr -fno-sanitize-recover=undefined -fno-implicit-constexpr" }
 
 struct A {
   A (int) {}
diff --git a/gcc/testsuite/g++.dg/vect/pr99149.cc b/gcc/testsuite/g++.dg/vect/pr99149.cc
index 9d58426..e6e0594 100755
--- a/gcc/testsuite/g++.dg/vect/pr99149.cc
+++ b/gcc/testsuite/g++.dg/vect/pr99149.cc
@@ -25,4 +25,3 @@ public:
 main() { n.j(); }
 
 /* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_MUL" 1 "slp2" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_FMA" 1 "slp2" } } */
diff --git a/gcc/testsuite/g++.dg/warn/Warray-bounds-16.C b/gcc/testsuite/g++.dg/warn/Warray-bounds-16.C
index 17b4d0d..89cbadb 100644
--- a/gcc/testsuite/g++.dg/warn/Warray-bounds-16.C
+++ b/gcc/testsuite/g++.dg/warn/Warray-bounds-16.C
@@ -19,11 +19,11 @@ struct S
     p = (int*) new unsigned char [sizeof (int) * m];
 
     for (int i = 0; i < m; i++)
-      new (p + i) int ();
+      new (p + i) int (); /* { dg-bogus "bounds" "pr102690" { xfail *-*-* } } */
   }
 };
 
 S a (0);
 
-/* Verify the loop has been eliminated.
-   { dg-final { scan-tree-dump-not "goto" "optimized" } } */
+/* The loop cannot be eliminated since the global 'new' can change 'm'.  */
+/* { dg-final { scan-tree-dump-not "goto" "optimized" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C b/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C
index 92c75df..bac2b68 100644
--- a/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C
+++ b/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C
@@ -1,7 +1,7 @@
 /* PR c++/100876 - -Wmismatched-new-delete should either look through
    or ignore placement new
    { dg-do compile }
-   { dg-options "-O2 -Wall" } */
+   { dg-options "-O2 -Wall -fdelete-null-pointer-checks" } */
 
 extern "C" {
   void* malloc (__SIZE_TYPE__);
diff --git a/gcc/testsuite/g++.old-deja/g++.eh/spec1.C b/gcc/testsuite/g++.old-deja/g++.eh/spec1.C
index ea32045..3c832fd 100644
--- a/gcc/testsuite/g++.old-deja/g++.eh/spec1.C
+++ b/gcc/testsuite/g++.old-deja/g++.eh/spec1.C
@@ -17,7 +17,7 @@ f () throw (char, int, std::bad_exception)	// { dg-warning "deprecated" "" { tar
 int main ()
 {
   std::set_terminate (my_term);
-  std::set_unexpected (my_unexp);
+  std::set_unexpected (my_unexp); // { dg-warning "deprecated" "" { target c++11 } }
 
   try
     {
diff --git a/gcc/testsuite/g++.old-deja/g++.eh/spec2.C b/gcc/testsuite/g++.old-deja/g++.eh/spec2.C
index d1aa698..2f7556b 100644
--- a/gcc/testsuite/g++.old-deja/g++.eh/spec2.C
+++ b/gcc/testsuite/g++.old-deja/g++.eh/spec2.C
@@ -17,7 +17,7 @@ f () throw (int, std::bad_exception)	// { dg-warning "deprecated" "" { target c+
 int main ()
 {
   std::set_terminate (my_term);
-  std::set_unexpected (my_unexp);
+  std::set_unexpected (my_unexp); // { dg-warning "deprecated" "" { target c++11 } }
 
   try
     {
diff --git a/gcc/testsuite/g++.old-deja/g++.eh/spec3.C b/gcc/testsuite/g++.old-deja/g++.eh/spec3.C
index 1d57df3..278de2e 100644
--- a/gcc/testsuite/g++.old-deja/g++.eh/spec3.C
+++ b/gcc/testsuite/g++.old-deja/g++.eh/spec3.C
@@ -17,7 +17,7 @@ f () throw (std::bad_exception)		// { dg-warning "deprecated" "" { target c++11
 int main ()
 {
   std::set_terminate (my_term);
-  std::set_unexpected (my_unexp);
+  std::set_unexpected (my_unexp); // { dg-warning "deprecated" "" { target c++11 } }
 
   try
     {
diff --git a/gcc/testsuite/g++.old-deja/g++.eh/spec4.C b/gcc/testsuite/g++.old-deja/g++.eh/spec4.C
index e102239..d5d4896 100644
--- a/gcc/testsuite/g++.old-deja/g++.eh/spec4.C
+++ b/gcc/testsuite/g++.old-deja/g++.eh/spec4.C
@@ -17,7 +17,7 @@ f () throw (short)		// { dg-warning "deprecated" "" { target c++11 } }
 int main ()
 {
   std::set_terminate (my_term);
-  std::set_unexpected (my_unexp);
+  std::set_unexpected (my_unexp); // { dg-warning "deprecated" "" { target c++11 } }
 
   try
     {
diff --git a/gcc/testsuite/g++.old-deja/g++.mike/eh33.C b/gcc/testsuite/g++.old-deja/g++.mike/eh33.C
index cffb0c4..8d50f73 100644
--- a/gcc/testsuite/g++.old-deja/g++.mike/eh33.C
+++ b/gcc/testsuite/g++.old-deja/g++.mike/eh33.C
@@ -11,7 +11,7 @@ void my_unexpected() {
 void foo() throw (int) { throw "Hi"; }	// { dg-warning "deprecated" "" { target c++11 } }
 
 int main() {
-  std::set_unexpected (my_unexpected);
+  std::set_unexpected (my_unexpected); // { dg-warning "deprecated" "" { target c++11 } }
   try {
     foo();
   } catch (int i) {
diff --git a/gcc/testsuite/g++.old-deja/g++.mike/eh34.C b/gcc/testsuite/g++.old-deja/g++.mike/eh34.C
index 3218dd9..36512ae 100644
--- a/gcc/testsuite/g++.old-deja/g++.mike/eh34.C
+++ b/gcc/testsuite/g++.old-deja/g++.mike/eh34.C
@@ -11,7 +11,7 @@ void my_unexpected() {
 void foo() throw () { throw "Hi"; }
 
 int main() {
-  std::set_unexpected (my_unexpected);
+  std::set_unexpected (my_unexpected); // { dg-warning "deprecated" "" { target c++11 } }
   foo();
   return 1;
 }
diff --git a/gcc/testsuite/g++.old-deja/g++.mike/eh50.C b/gcc/testsuite/g++.old-deja/g++.mike/eh50.C
index 028a2de..2507cf2 100644
--- a/gcc/testsuite/g++.old-deja/g++.mike/eh50.C
+++ b/gcc/testsuite/g++.old-deja/g++.mike/eh50.C
@@ -12,7 +12,7 @@ template <class T> void foo(T) throw (int) { throw "Hi"; }	// { dg-warning "depr
 
 int
 main() {
-  std::set_unexpected (my_unexpected);
+  std::set_unexpected (my_unexpected); // { dg-warning "deprecated" "" { target c++11 } }
   try {
     foo(1);
   } catch (int i) {
diff --git a/gcc/testsuite/g++.old-deja/g++.mike/eh51.C b/gcc/testsuite/g++.old-deja/g++.mike/eh51.C
index 428635b..d1902f0 100644
--- a/gcc/testsuite/g++.old-deja/g++.mike/eh51.C
+++ b/gcc/testsuite/g++.old-deja/g++.mike/eh51.C
@@ -12,7 +12,7 @@ template <class T> void foo(T) throw (T) { throw "Hi"; }	// { dg-warning "deprec
 
 int
 main() {
-  std::set_unexpected (my_unexpected);
+  std::set_unexpected (my_unexpected); // { dg-warning "deprecated" "" { target c++11 } }
   try {
     foo(1);
   } catch (int i) {
diff --git a/gcc/testsuite/g++.old-deja/g++.other/delete6.C b/gcc/testsuite/g++.old-deja/g++.other/delete6.C
index 733a6e2..56cfa00 100644
--- a/gcc/testsuite/g++.old-deja/g++.other/delete6.C
+++ b/gcc/testsuite/g++.old-deja/g++.other/delete6.C
@@ -25,4 +25,4 @@ inline void A::operator delete(void*p)
 
 int main()
 {A *ap=new A;
-delete ap;}
+delete ap;}			// { dg-prune-output "unallocated object 'i'" }
diff --git a/gcc/testsuite/g++.target/arm/pr102842.C b/gcc/testsuite/g++.target/arm/pr102842.C
new file mode 100644
index 0000000..a2bac66
--- /dev/null
+++ b/gcc/testsuite/g++.target/arm/pr102842.C
@@ -0,0 +1,30 @@
+/* PR rtl-optimization/102842 */
+/* { dg-do compile } */
+/* { dg-options "-fPIC  -O2 -fno-omit-frame-pointer -mthumb -march=armv7-a+fp" } */
+
+struct Plane {
+  using T = float;
+  T *Row();
+};
+using ImageF = Plane;
+long long Mirror_x;
+struct EnsurePaddingInPlaceRowByRow {
+  void Process() {
+    switch (strategy_) {
+    case kSlow:
+      float *row = img_.Row();
+      long long xsize = x1_;
+      while (Mirror_x >= xsize)
+        if (Mirror_x)
+          Mirror_x = 2 * xsize - 1;
+      *row = Mirror_x;
+    }
+  }
+  ImageF img_;
+  unsigned x1_;
+  enum { kSlow } strategy_;
+};
+void FinalizeImageRect() {
+  EnsurePaddingInPlaceRowByRow ensure_padding;
+  ensure_padding.Process();
+}
diff --git a/gcc/testsuite/g++.target/i386/pr102566-1.C b/gcc/testsuite/g++.target/i386/pr102566-1.C
new file mode 100644
index 0000000..94a66d7
--- /dev/null
+++ b/gcc/testsuite/g++.target/i386/pr102566-1.C
@@ -0,0 +1,31 @@
+/* { dg-do compile { target c++11 } } */
+/* { dg-options "-O2" } */
+
+#include <atomic>
+
+bool
+tbit0 (std::atomic<int> &i)
+{
+#define BIT (1 << 0)
+  return i.fetch_or(BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+bool
+tbit30 (std::atomic<int> &i)
+{
+#define BIT (1 << 30)
+  return i.fetch_or(BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+bool
+tbit31 (std::atomic<int> &i)
+{
+#define BIT (1 << 31)
+  return i.fetch_or(BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btsl" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/g++.target/i386/pr102566-2.C b/gcc/testsuite/g++.target/i386/pr102566-2.C
new file mode 100644
index 0000000..4f2aea9
--- /dev/null
+++ b/gcc/testsuite/g++.target/i386/pr102566-2.C
@@ -0,0 +1,31 @@
+/* { dg-do compile { target c++11 } } */
+/* { dg-options "-O2" } */
+
+#include <atomic>
+
+bool
+tbit0 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 0)
+  return i.fetch_or(BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+bool
+tbit30 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 30)
+  return i.fetch_or(BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+bool
+tbit31 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 31)
+  return i.fetch_or(BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btsl" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/g++.target/i386/pr102566-3.C b/gcc/testsuite/g++.target/i386/pr102566-3.C
new file mode 100644
index 0000000..e88921d
--- /dev/null
+++ b/gcc/testsuite/g++.target/i386/pr102566-3.C
@@ -0,0 +1,31 @@
+/* { dg-do compile { target c++11 } } */
+/* { dg-options "-O2" } */
+
+#include <atomic>
+
+bool
+tbit0 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 0)
+  return !(i.fetch_or(BIT, std::memory_order_relaxed) & BIT);
+#undef BIT 
+}
+
+bool
+tbit30 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 30)
+  return !(i.fetch_or(BIT, std::memory_order_relaxed) & BIT);
+#undef BIT 
+}
+
+bool
+tbit31 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 31)
+  return !(i.fetch_or(BIT, std::memory_order_relaxed) & BIT);
+#undef BIT 
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btsl" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/g++.target/i386/pr102566-4.C b/gcc/testsuite/g++.target/i386/pr102566-4.C
new file mode 100644
index 0000000..44d1362
--- /dev/null
+++ b/gcc/testsuite/g++.target/i386/pr102566-4.C
@@ -0,0 +1,29 @@
+/* { dg-do compile { target c++11 } } */
+/* { dg-options "-O2" } */
+
+#include <atomic>
+
+typedef int __attribute__ ((mode (__word__))) int_type;
+
+#define BIT (1 << 0)
+
+bool
+tbit0 (std::atomic<int_type> &i)
+{
+  return i.fetch_or(BIT, std::memory_order_relaxed) & ~1;
+}
+
+bool
+tbit30 (std::atomic<int_type> &i)
+{
+  return i.fetch_or(BIT, std::memory_order_relaxed) & ~2;
+}
+
+bool
+tbit31 (std::atomic<int_type> &i)
+{
+  return i.fetch_or(BIT, std::memory_order_relaxed) & ~4;
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*cmpxchg" 3 } } */
+/* { dg-final { scan-assembler-not "bts" } } */
diff --git a/gcc/testsuite/g++.target/i386/pr102566-5a.C b/gcc/testsuite/g++.target/i386/pr102566-5a.C
new file mode 100644
index 0000000..f9595be
--- /dev/null
+++ b/gcc/testsuite/g++.target/i386/pr102566-5a.C
@@ -0,0 +1,31 @@
+/* { dg-do compile { target c++11 } } */
+/* { dg-options "-O2" } */
+
+#include <atomic>
+
+bool
+tbit0 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 0)
+  return i.fetch_and(~BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+bool
+tbit30 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 30)
+  return i.fetch_and(~BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+bool
+tbit31 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 31)
+  return i.fetch_and(~BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btrl" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/g++.target/i386/pr102566-5b.C b/gcc/testsuite/g++.target/i386/pr102566-5b.C
new file mode 100644
index 0000000..d917b27
--- /dev/null
+++ b/gcc/testsuite/g++.target/i386/pr102566-5b.C
@@ -0,0 +1,31 @@
+/* { dg-do compile { target { c++11 && { ! ia32 } } } } */
+/* { dg-options "-O2" } */
+
+#include <atomic>
+
+bool
+tbit0 (std::atomic<unsigned long long> &i)
+{
+#define BIT (1ll << 0)
+  return i.fetch_and(~BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+bool
+tbit30 (std::atomic<unsigned long long> &i)
+{
+#define BIT (1ll << 30)
+  return i.fetch_and(~BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+bool
+tbit31 (std::atomic<unsigned long long> &i)
+{
+#define BIT (1ll << 63)
+  return i.fetch_and(~BIT, std::memory_order_relaxed) & BIT;
+#undef BIT 
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btrq" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/g++.target/i386/pr102566-6a.C b/gcc/testsuite/g++.target/i386/pr102566-6a.C
new file mode 100644
index 0000000..01d495e
--- /dev/null
+++ b/gcc/testsuite/g++.target/i386/pr102566-6a.C
@@ -0,0 +1,31 @@
+/* { dg-do compile { target c++11 } } */
+/* { dg-options "-O2" } */
+
+#include <atomic>
+
+bool
+tbit0 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 0)
+  return !(i.fetch_and(~BIT, std::memory_order_relaxed) & BIT);
+#undef BIT 
+}
+
+bool
+tbit30 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 30)
+  return !(i.fetch_and(~BIT, std::memory_order_relaxed) & BIT);
+#undef BIT 
+}
+
+bool
+tbit31 (std::atomic<unsigned int> &i)
+{
+#define BIT (1 << 31)
+  return !(i.fetch_and(~BIT, std::memory_order_relaxed) & BIT);
+#undef BIT 
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btrl" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/g++.target/i386/pr102566-6b.C b/gcc/testsuite/g++.target/i386/pr102566-6b.C
new file mode 100644
index 0000000..adc11fc
--- /dev/null
+++ b/gcc/testsuite/g++.target/i386/pr102566-6b.C
@@ -0,0 +1,31 @@
+/* { dg-do compile { target { c++11 && { ! ia32 } } } } */
+/* { dg-options "-O2" } */
+
+#include <atomic>
+
+bool
+tbit0 (std::atomic<unsigned long long> &i)
+{
+#define BIT (1ll << 0)
+  return !(i.fetch_and(~BIT, std::memory_order_relaxed) & BIT);
+#undef BIT 
+}
+
+bool
+tbit30 (std::atomic<unsigned long long> &i)
+{
+#define BIT (1ll << 30)
+  return !(i.fetch_and(~BIT, std::memory_order_relaxed) & BIT);
+#undef BIT 
+}
+
+bool
+tbit31 (std::atomic<unsigned long long> &i)
+{
+#define BIT (1ll << 63)
+  return !(i.fetch_and(~BIT, std::memory_order_relaxed) & BIT);
+#undef BIT 
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btrq" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr100278.c b/gcc/testsuite/gcc.c-torture/compile/pr100278.c
index 8d70228..4631080 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr100278.c
+++ b/gcc/testsuite/gcc.c-torture/compile/pr100278.c
@@ -1,5 +1,3 @@
-/* { dg-do compile } */
-/* { dg-options "-O2" } */
 
 void a()
 {
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr101189.c b/gcc/testsuite/gcc.c-torture/compile/pr101189.c
index 5730708..ad83d32 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr101189.c
+++ b/gcc/testsuite/gcc.c-torture/compile/pr101189.c
@@ -1,6 +1,4 @@
 /* PR tree-optimization/101189  */
-/* { dg-do compile } */
-/* { dg-options "-O2" } */
 
 static int a, b;
 int main() {
diff --git a/gcc/testsuite/gcc.c-torture/compile/vector-dup-1.c b/gcc/testsuite/gcc.c-torture/compile/vector-dup-1.c
new file mode 100644
index 0000000..3475360
--- /dev/null
+++ b/gcc/testsuite/gcc.c-torture/compile/vector-dup-1.c
@@ -0,0 +1,15 @@
+/* { dg-additional-options "-fno-strict-aliasing" } */
+
+
+/* PR target/103170 */
+/* AARCH64 used to ICE on this for a typo in the string template. */
+#define vector __attribute__((vector_size(4*sizeof(float))))
+
+typedef vector float v4sf;
+
+v4sf f(int t)
+{
+  float tt = *(float*)&t;
+  asm("":"+r"(tt));
+  return (v4sf){tt,tt,tt,tt};
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr100453.c b/gcc/testsuite/gcc.c-torture/execute/pr100453.c
index 0cf0ad2..853a892 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr100453.c
+++ b/gcc/testsuite/gcc.c-torture/execute/pr100453.c
@@ -1,5 +1,3 @@
-/* { dg-do run } */
-/* { dg-options "-O1" } */
 
 struct a {
   int b : 4;
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr101335.c b/gcc/testsuite/gcc.c-torture/execute/pr101335.c
index 921362c..45a399f 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr101335.c
+++ b/gcc/testsuite/gcc.c-torture/execute/pr101335.c
@@ -1,5 +1,3 @@
-/* { dg-do run } */
-/* { dg-options "-O2" } */
 
 unsigned a = 0xFFFFFFFF;
 int b;
diff --git a/gcc/testsuite/gcc.c-torture/execute/pr103209.c b/gcc/testsuite/gcc.c-torture/execute/pr103209.c
new file mode 100644
index 0000000..4816893
--- /dev/null
+++ b/gcc/testsuite/gcc.c-torture/execute/pr103209.c
@@ -0,0 +1,36 @@
+#include <stdio.h>
+#include <stdint.h>
+
+int32_t a[6];
+int64_t b;
+int32_t *c;
+int32_t **d = &c;
+int64_t *e = &b;
+int32_t **const *f = &d;
+int32_t **const **g = &f;
+int32_t *h();
+static int16_t j();
+static uint32_t k(int8_t, const int32_t *, int64_t);
+static uint32_t l() {
+  int32_t *m = &a[3];
+  int32_t n = 0;
+  int8_t o = 0;
+  int32_t *p[] = {&n, &n, &n, &n};
+  uint32_t q[6][1][2] = {};
+  for (o = 0; o <= 1; o = 6)
+    if (h(j(k(3, 0, q[2][0][0]), &n), n) == p[3])
+      *m = *e;
+  return 0;
+}
+int32_t *h(uint32_t, int32_t) { return ***g; }
+int16_t j(uint32_t, int32_t *r) { **f = r; return 0;}
+uint32_t k(int8_t, const int32_t *, int64_t) { *e = 3; return 0;}
+int main() {
+  int i = 0;
+  l();
+  for (i = 0; i < 6; i++){
+    if (i == 3 && a[i] != 3)
+	__builtin_abort ();
+  }
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.c-torture/execute/pr10352-1.c b/gcc/testsuite/gcc.c-torture/execute/pr10352-1.c
new file mode 100644
index 0000000..babb9d4
--- /dev/null
+++ b/gcc/testsuite/gcc.c-torture/execute/pr10352-1.c
@@ -0,0 +1,12 @@
+/* this is another case where phiopt
+   would create -signed1bit which is undefined. */
+struct {
+  int a:1;
+} b;
+int *c = (int *)&b, d;
+int main() {
+  d = c && (b.a = (d < 0) ^ 3);
+  if (d != 1)
+    __builtin_abort();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/Warray-bounds-48.c b/gcc/testsuite/gcc.dg/Warray-bounds-48.c
index 19b7634c..29b2086 100644
--- a/gcc/testsuite/gcc.dg/Warray-bounds-48.c
+++ b/gcc/testsuite/gcc.dg/Warray-bounds-48.c
@@ -30,7 +30,7 @@ static void nowarn_ax_extern (struct AX *p)
 
 static void warn_ax_local_buf (struct AX *p)
 {
-  p->ax[0] = 4; p->ax[1] = 5;  // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v2hi_store &&  { ! vect_slp_v4hi_store } } } }
+  p->ax[0] = 4; p->ax[1] = 5;  // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v2hi_store_align &&  { ! vect_slp_v4hi_store_unalign } } } }
 
   p->ax[2] = 6;     // { dg-warning "\\\[-Warray-bounds" }
   p->ax[3] = 7;     // { dg-warning "\\\[-Warray-bounds" }
@@ -130,7 +130,7 @@ static void warn_a0_extern (struct A0 *p)
 
 static void warn_a0_local_buf (struct A0 *p)
 {
-  p->a0[0] = 4; p->a0[1] = 5;  // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v2hi_store && { ! vect_slp_v4hi_store } } } }
+  p->a0[0] = 4; p->a0[1] = 5;  // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v2hi_store_align && { ! vect_slp_v4hi_store_unalign } } } }
 
   p->a0[2] = 6;     // { dg-warning "\\\[-Warray-bounds" }
   p->a0[3] = 7;     // { dg-warning "\\\[-Warray-bounds" }
diff --git a/gcc/testsuite/gcc.dg/Warray-bounds-51.c b/gcc/testsuite/gcc.dg/Warray-bounds-51.c
index 8b589f3..7519b0f 100644
--- a/gcc/testsuite/gcc.dg/Warray-bounds-51.c
+++ b/gcc/testsuite/gcc.dg/Warray-bounds-51.c
@@ -39,7 +39,7 @@ void test_struct_char_vla_location (void)
   } s;
 
   s.cvla[0] = __LINE__;
-  s.cvla[nelts - 1] = 0; // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v2qi_store } } }
+  s.cvla[nelts - 1] = 0; // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v2qi_store_align } } }
   s.cvla[nelts] = 0;  // { dg-warning "\\\[-Warray-bounds" }
 
   sink (&s);
diff --git a/gcc/testsuite/gcc.dg/Warray-bounds-90.c b/gcc/testsuite/gcc.dg/Warray-bounds-90.c
new file mode 100644
index 0000000..1ff6077
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/Warray-bounds-90.c
@@ -0,0 +1,147 @@
+/* PR middle-end/102453 - buffer overflow by atomic built-ins not diagnosed
+   Verify that out-of-bounds accesses by atomic functions are diagnosed.
+   { dg-do compile }
+   { dg-options "-O2 -Wall -ftrack-macro-expansion=0" }  */
+
+#ifndef __cplusplus
+#  define bool _Bool
+#endif
+
+#define load        __atomic_load
+#define store       __atomic_store
+#define add_fetch   __atomic_add_fetch
+#define sub_fetch   __atomic_sub_fetch
+#define and_fetch   __atomic_and_fetch
+#define or_fetch    __atomic_or_fetch
+#define xor_fetch   __atomic_xor_fetch
+#define nand_fetch  __atomic_nand_fetch
+
+typedef __SIZE_TYPE__ size_t;
+
+void sink (void*, ...);
+#define sink(...) sink (0, __VA_ARGS__)
+
+extern _Bool eb;
+extern char ec;
+extern short int esi;
+extern int ei;
+extern long int eli;
+extern long long int elli;
+
+extern const _Bool ecb;
+extern const char ecc;
+extern const short int ecsi;
+extern const int eci;
+extern const long int ecli;
+extern const long long int eclli;
+
+extern _Atomic _Bool eab;
+extern _Atomic char eac;
+extern _Atomic short int easi;
+extern _Atomic int eai;
+extern _Atomic long int eali;
+extern _Atomic long long int ealli;
+
+extern _Atomic const _Bool eacb;
+extern _Atomic const char eacc;
+extern _Atomic const short int eacsi;
+extern _Atomic const int eaci;
+extern _Atomic const long int eacli;
+extern _Atomic const long long int eaclli;
+
+
+void nowarn_atomic_load (void)
+{
+  load (&eacb, &eb, 0);
+  load (&eacc, &ec, 0);
+  load (&eacsi, &esi, 0);
+  load (&eaci, &ei, 0);
+  load (&eacli, &eli, 0);
+  load (&eaclli, &elli, 0);
+}
+
+
+void warn_atomic_load_note (void)
+{
+  int i;                            // { dg-message "'i'" }
+
+  int *pi = (int*)((char*)&i + 1);
+  load (&eaci, pi, 0);              // { dg-warning "-Warray-bounds" }
+  sink (&i);
+
+  pi = (int*)((char*)&i + 2);
+  load (&eaci, pi, 0);              // { dg-warning "-Warray-bounds" }
+  sink (&i);
+
+  pi = &i + 1;
+  load (&eaci, pi, 0);              // { dg-warning "-Warray-bounds" }
+  sink (&i);
+}
+
+
+void warn_atomic_load (void)
+{
+  bool *pb = &eb + 1;
+  load (&eacb, pb, 0);              // { dg-warning "-Warray-bounds" }
+
+  char *pc = &ec + 1;
+  load (&eacc, pc, 0);              // { dg-warning "-Warray-bounds" }
+
+  short *psi = (short*)((char*)&esi + 1);
+  load (&eacsi, psi, 0);            // { dg-warning "-Warray-bounds" }
+  psi = (short*)((char*)&esi + 2);
+  load (&eacsi, psi, 0);            // { dg-warning "-Warray-bounds" }
+
+  int *pi = (int*)((char*)&ei + 1);
+  load (&eaci, pi, 0);              // { dg-warning "-Warray-bounds" }
+  pi = (int*)((char*)&ei + 2);
+  load (&eaci, pi, 0);              // { dg-warning "-Warray-bounds" }
+  pi = (int*)((char*)&ei + sizeof ei);
+  load (&eaci, pi, 0);              // { dg-warning "-Warray-bounds" }
+
+  long *pli = (long*)((char*)&eli + 1);
+  load (&eacli, pli, 0);            // { dg-warning "-Warray-bounds" }
+  pli = (long*)((char*)&eli + 1);
+  load (&eacli, pli, 0);            // { dg-warning "-Warray-bounds" }
+  pli = &eli + 1;
+  load (&eacli, pli, 0);            // { dg-warning "-Warray-bounds" }
+
+  long long *plli = (long long*)((char*)&elli + 1);
+  load (&eaclli, plli, 0);          // { dg-warning "-Warray-bounds" }
+  plli = (long long*)((char*)&elli + 1);
+  load (&eaclli, plli, 0);          // { dg-warning "-Warray-bounds" }
+  plli = &elli + 1;
+  load (&eaclli, plli, 0);          // { dg-warning "-Warray-bounds" }
+}
+
+
+void warn_atomic_store (void)
+{
+  const bool *pb = &eb + 1;
+  store (&eab, pb, 0);              // { dg-warning "-Warray-bounds" }
+
+  const char *pc = &ec + 1;
+  store (&eac, pc, 0);              // { dg-warning "-Warray-bounds" }
+
+  const short *psi = (const short*)((const char*)&ecsi + 1);
+  store (&easi, psi, 0);            // { dg-warning "-Warray-bounds" }
+  psi = (const short*)((const char*)&esi + 2);
+  store (&easi, psi, 0);            // { dg-warning "-Warray-bounds" }
+
+  const int *pi = (const int*)((const char*)&eci + 1);
+  store (&eai, pi, 0);              // { dg-warning "-Warray-bounds" }
+  pi = (const int*)((const char*)&ei + 2);
+  store (&eai, pi, 0);              // { dg-warning "-Warray-bounds" }
+  pi = (const int*)((const char*)&ei + sizeof ei);
+  store (&eai, pi, 0);              // { dg-warning "-Warray-bounds" }
+
+  const long *pli = (const long*)((const char*)&eli + 1);
+  store (&eali, pli, 0);            // { dg-warning "-Warray-bounds" }
+  pli = (const long*)((const char*)&eli + sizeof (eli));
+  store (&eali, pli, 0);            // { dg-warning "-Warray-bounds" }
+
+  const long long *plli = (const long long*)((const char*)&elli + 1);
+  store (&ealli, plli, 0);          // { dg-warning "-Warray-bounds" }
+  plli = (const long long*)((const char*)&elli + sizeof elli);
+  store (&ealli, plli, 0);          // { dg-warning "-Warray-bounds" }
+}
diff --git a/gcc/testsuite/gcc.dg/Warray-parameter-3.c b/gcc/testsuite/gcc.dg/Warray-parameter-3.c
index b6ed8da..b888511 100644
--- a/gcc/testsuite/gcc.dg/Warray-parameter-3.c
+++ b/gcc/testsuite/gcc.dg/Warray-parameter-3.c
@@ -77,7 +77,7 @@ gia3 (int a[3])
 __attribute__ ((noipa)) void
 gcas3 (char a[static 3])
 {
-  a[0] = 0; a[1] = 1; a[2] = 2; // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v4qi_store } } }
+  a[0] = 0; a[1] = 1; a[2] = 2; // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v4qi_store_unalign } } }
   a[3] = 3;                   // { dg-warning "\\\[-Warray-bounds" }
 }
 
diff --git a/gcc/testsuite/gcc.dg/Wobjsize-1.c b/gcc/testsuite/gcc.dg/Wobjsize-1.c
index 2bd2f93..988b8bc 100644
--- a/gcc/testsuite/gcc.dg/Wobjsize-1.c
+++ b/gcc/testsuite/gcc.dg/Wobjsize-1.c
@@ -7,11 +7,12 @@ char buf[6];
 
 int main(int argc, char **argv)
 {
-  strcpy (buf,"hello ");    /* { dg-warning "\\\[-Wstringop-overflow" } */
+  strcpy (buf,"hello ");
   return 0;
 }
 
-/* { dg-message "file included" "included" { target *-*-* } 0 }
+/* { dg-warning "\\\[-Wstringop-overflow" "warning" { target *-*-* } 0 }
+   { dg-message "file included" "included" { target *-*-* } 0 }
    { dg-message "inlined from" "inlined" { target *-*-* } 0 }
 
    The test might emit two warnings, one for the strcpy call and
diff --git a/gcc/testsuite/gcc.dg/Wrestrict-23.c b/gcc/testsuite/gcc.dg/Wrestrict-23.c
new file mode 100644
index 0000000..c7a828b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/Wrestrict-23.c
@@ -0,0 +1,146 @@
+/* PR tree-optimization/102238 - missing -Wrestrict on sprintf formatting
+   a struct member into enclosing object
+   { dg-do compile }
+   { dg-options "-O2 -Wall -Wno-format-overflow" } */
+
+extern int sprintf (char*, const char*, ...);
+
+extern void sink (void*, ...);
+
+struct A
+{
+  char a[4];
+};
+
+struct B
+{
+  struct A a1, a2;
+};
+
+extern struct B eb;
+
+enum { B_a2_a_off = __builtin_offsetof (struct B, a2.a) };
+
+
+void test_warn_src_decl_plus (void)
+{
+  {
+    char *s = (char*)&eb + B_a2_a_off;
+    char *d = eb.a2.a;
+    sprintf (d, "%s", s);     // { dg-warning "overlaps" }
+  }
+
+  {
+    // If strlen (s) > 0 there is overlap with a[1].
+    char *s = (char*)&eb + B_a2_a_off + 1;
+    char *d = eb.a2.a;
+    sprintf (d, "%s", s);     // { dg-warning "may overlap" }
+  }
+
+  {
+    // strlen (s) must be at most 1 so there can be no overlap with a.
+    char *s = (char*)&eb + B_a2_a_off + 2;
+    char *d = eb.a2.a;
+    sprintf (d, "%s", s);     // { dg-bogus "-Wrestrict" }
+  }
+
+  {
+    // strlen (s) must be at most 0 so there can be no overlap with a.
+    char *s = (char*)&eb + B_a2_a_off + 3;
+    char *d = eb.a2.a;
+    sprintf (d, "%s", s);     // { dg-bogus "-Wrestrict" }
+  }
+}
+
+
+void test_warn_src_ptr_plus (struct B *p)
+{
+  {
+    char *s = (char*)p + B_a2_a_off;
+    char *d = p->a2.a;
+    sprintf (d, "%s", s);     // { dg-warning "overlaps" }
+  }
+
+  {
+    // If strlen (s) > 0 there is overlap with a[1].
+    char *s = (char*)p + B_a2_a_off + 1;
+    char *d = p->a2.a;
+    sprintf (d, "%s", s);     // { dg-warning "may overlap" }
+  }
+
+  {
+    // strlen (s) must be at most 1 so there can be no overlap with a.
+    char *s = (char*)p + B_a2_a_off + 2;
+    char *d = p->a2.a;
+    sprintf (d, "%s", s);     // { dg-bogus "-Wrestrict" }
+  }
+
+  {
+    // strlen (s) must be at most 0 so there can be no overlap with a.
+    char *s = (char*)p + B_a2_a_off + 3;
+    char *d = p->a2.a;
+    sprintf (d, "%s", s);     // { dg-bogus "-Wrestrict" }
+  }
+}
+
+
+void test_warn_dst_decl_plus (void)
+{
+  {
+    char *s = eb.a2.a;
+    char *d = (char*)&eb + B_a2_a_off;
+    sprintf (d, "%s", s);     // { dg-warning "overlaps" }
+  }
+
+  {
+    // If strlen (a) > 0 there is overlap with a[1].
+    char *s = eb.a2.a;
+    char *d = (char*)&eb + B_a2_a_off + 1;
+    sprintf (d, "%s", s);     // { dg-warning "may overlap" }
+  }
+
+  {
+    // If strlen (a) > 1 there is overlap with a[2].
+    char *s = eb.a2.a;
+    char *d = (char*)&eb + B_a2_a_off + 2;
+    sprintf (d, "%s", s);     // { dg-warning "may overlap" }
+  }
+
+  {
+    // If strlen (a) > 2 there is overlap with a[3].
+    char *s = eb.a2.a;
+    char *d = (char*)&eb + B_a2_a_off + 3;
+    sprintf (d, "%s", s);     // { dg-warning "may overlap" }
+  }
+}
+
+
+void test_warn_dst_ptr_plus (struct B *p)
+{
+  {
+    char *s = p->a2.a;
+    char *d = (char*)p + B_a2_a_off;
+    sprintf (d, "%s", s);     // { dg-warning "overlaps" }
+  }
+
+  {
+    // If strlen (a) > 0 there is overlap with a[1].
+    char *s = p->a2.a;
+    char *d = (char*)p + B_a2_a_off + 1;
+    sprintf (d, "%s", s);     // { dg-warning "may overlap" }
+  }
+
+  {
+    // If strlen (a) > 1 there is overlap with a[2].
+    char *s = p->a2.a;
+    char *d = (char*)p + B_a2_a_off + 2;
+    sprintf (d, "%s", s);     // { dg-warning "may overlap" }
+  }
+
+  {
+    // If strlen (a) > 2 there is overlap with a[3].
+    char *s = p->a2.a;
+    char *d = (char*)p + B_a2_a_off + 3;
+    sprintf (d, "%s", s);     // { dg-warning "may overlap" }
+  }
+}
diff --git a/gcc/testsuite/gcc.dg/Wstringop-overflow-14.c b/gcc/testsuite/gcc.dg/Wstringop-overflow-14.c
index c4a3f05..3d1d9e5 100644
--- a/gcc/testsuite/gcc.dg/Wstringop-overflow-14.c
+++ b/gcc/testsuite/gcc.dg/Wstringop-overflow-14.c
@@ -36,8 +36,8 @@ void test_memcpy_cond (int i)
 void test_int16 (void)
 {
   char *p = a4 + 1;
-  *(int16_t*)p = 0;    // { dg-warning "writing 4 bytes into a region of size 3" "pr102706" { target { vect_slp_v2hi_store } } }
-  *(int16_t*)(p + 2) = 0;   // { dg-warning "writing 2 bytes into a region of size 1" "pr102706" { xfail { vect_slp_v2hi_store } } }
+  *(int16_t*)p = 0;    // { dg-warning "writing 4 bytes into a region of size 3" "pr102706" { target { vect_slp_v2hi_store_unalign } } }
+  *(int16_t*)(p + 2) = 0;   // { dg-warning "writing 2 bytes into a region of size 1" "pr102706" { xfail { vect_slp_v2hi_store_unalign } } }
 }
 
 
diff --git a/gcc/testsuite/gcc.dg/Wstringop-overflow-21.c b/gcc/testsuite/gcc.dg/Wstringop-overflow-21.c
index 3fccfc9..4cd3c1b 100644
--- a/gcc/testsuite/gcc.dg/Wstringop-overflow-21.c
+++ b/gcc/testsuite/gcc.dg/Wstringop-overflow-21.c
@@ -23,10 +23,10 @@ void test_store_zero_length (int i)
 {
   char a[3];
   struct S0 *p = (struct S0*)a;
-  p->a = 0;                         // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v4qi_store } } }
+  p->a = 0;                         // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v4qi_store_align } } }
   p->b[0] = 0;
   p->b[1] = 1;                      // { dg-bogus "\\\[-Wstringop-overflow" }
-  p->b[2] = 2;                      // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { xfail { vect_slp_v4qi_store } } }
+  p->b[2] = 2;                      // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { xfail { vect_slp_v4qi_store_align } } }
   p->b[i] = 2;
   sink (p);
 }
@@ -50,10 +50,10 @@ void test_store_flexarray (int i)
 {
   char a[3];
   struct Sx *p = (struct Sx*)a;
-  p->a = 0;                         // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v4qi_store } } }
+  p->a = 0;                         // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v4qi_store_align } } }
   p->b[0] = 0;
   p->b[1] = 1;                      // { dg-bogus "\\\[-Wstringop-overflow" }
-  p->b[2] = 1;                      // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { xfail { vect_slp_v4qi_store } } }
+  p->b[2] = 1;                      // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { xfail { vect_slp_v4qi_store_align } } }
   p->b[i] = 2;
   sink (p);
 }
diff --git a/gcc/testsuite/gcc.dg/Wstringop-overflow-22.c b/gcc/testsuite/gcc.dg/Wstringop-overflow-22.c
index 8eaaa71..764b199 100644
--- a/gcc/testsuite/gcc.dg/Wstringop-overflow-22.c
+++ b/gcc/testsuite/gcc.dg/Wstringop-overflow-22.c
@@ -260,13 +260,12 @@ T (puts_unlocked, a);       // { dg-warning "missing terminating nul" "puts_unlo
 
 // Exerise exec functions.
 T (execl, a, s, NULL);      // { dg-warning "missing terminating nul" "execl" }
-T (execl, a, s, NULL);      // { dg-warning "missing terminating nul" "execl" }
-T (execle, a, s, NULL, NULL);   // { dg-warning "missing terminating nul" "execl" }
-T (execlp, a, s, NULL);     // { dg-warning "missing terminating nul" "execl" }
+T (execle, a, s, NULL, NULL);   // { dg-warning "missing terminating nul" "execle" }
+T (execlp, a, s, NULL);     // { dg-warning "missing terminating nul" "execlp" }
 
-T (execv, a, &d);           // { dg-warning "missing terminating nul" "execl" }
-T (execve, a, &d, &d);      // { dg-warning "missing terminating nul" "execl" }
-T (execvp, a, &d);          // { dg-warning "missing terminating nul" "execl" }
+T (execv, a, &d);           // { dg-warning "missing terminating nul" "execv" }
+T (execve, a, &d, &d);      // { dg-warning "missing terminating nul" "execve" }
+T (execvp, a, &d);          // { dg-warning "missing terminating nul" "execvp" }
 
 T (gettext, a);             // { dg-warning "missing terminating nul" "gettext" }
 
diff --git a/gcc/testsuite/gcc.dg/Wstringop-overflow-62.c b/gcc/testsuite/gcc.dg/Wstringop-overflow-62.c
index 4b6d1ab..ea19aa8 100644
--- a/gcc/testsuite/gcc.dg/Wstringop-overflow-62.c
+++ b/gcc/testsuite/gcc.dg/Wstringop-overflow-62.c
@@ -217,14 +217,14 @@ void test_max (void)
   {
     /* Exercise both pointers pointing to the same object plus constant
        offset.  */
-    char a2[2];               // { dg-message "at offset 1 into destination object 'a2' of size 2" "note" }
+    char a2[2];
     char *pi = a2 + 1;
     char *pj = a2 + 2;
 
     char *q = MAX (pi, pj);
 
-    memset (q, 0, 1);
-    memset (q, 0, 2);         // { dg-warning "writing 2 bytes into a region of size 1 " }
+    memset (q, 0, 1);         // { dg-warning "writing 1 byte into a region of size 0 " }
+    memset (q, 0, 2);         // { dg-warning "writing 2 bytes into a region of size 0 " }
   }
 
   {
diff --git a/gcc/testsuite/gcc.dg/Wstringop-overflow-68.c b/gcc/testsuite/gcc.dg/Wstringop-overflow-68.c
index 04e91af..05ea56f 100644
--- a/gcc/testsuite/gcc.dg/Wstringop-overflow-68.c
+++ b/gcc/testsuite/gcc.dg/Wstringop-overflow-68.c
@@ -65,11 +65,11 @@ void warn_comp_lit (void)
   // MEM <vector(8) char> [(char *)&a7] = { 0, 1, 2, 3, 4, 5, 6, 7 };
   // MEM <vector(16) char> [(char *)&a15] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
   // and warning should be expected, refer to PR102722.
-  *(AC4*)a2 = Ac4;      // { dg-warning "writing 4 bytes into a region of size 2" "pr101475" { xfail { ! { vect_slp_v4qi_store } } } }
-  *(AC4*)a3 = Ac4;      // { dg-warning "writing 4 bytes into a region of size 3" "pr101475" { xfail { ! { vect_slp_v4qi_store } } } }
-  *(AC8*)a4 = Ac8;      // { dg-warning "writing 8 bytes into a region of size 4" "pr101475" { xfail { ! { vect_slp_v8qi_store } } } }
-  *(AC8*)a7 = Ac8;      // { dg-warning "writing 8 bytes into a region of size 7" "pr101475" { xfail { ! { vect_slp_v8qi_store } } } }
-  *(AC16*)a15 = Ac16;   // { dg-warning "writing 16 bytes into a region of size 15" "pr101475" { xfail { ! { vect_slp_v16qi_store } } } }
+  *(AC4*)a2 = Ac4;      // { dg-warning "writing 4 bytes into a region of size 2" "pr101475" { xfail { ! { vect_slp_v4qi_store_unalign_1 } } } }
+  *(AC4*)a3 = Ac4;      // { dg-warning "writing 4 bytes into a region of size 3" "pr101475" { xfail { ! { vect_slp_v4qi_store_unalign_1 } } } }
+  *(AC8*)a4 = Ac8;      // { dg-warning "writing 8 bytes into a region of size 4" "pr101475" { xfail { ! { vect_slp_v8qi_store_unalign_1 } } } }
+  *(AC8*)a7 = Ac8;      // { dg-warning "writing 8 bytes into a region of size 7" "pr101475" { xfail { ! { vect_slp_v8qi_store_unalign_1 } } } }
+  *(AC16*)a15 = Ac16;   // { dg-warning "writing 16 bytes into a region of size 15" "pr101475" { xfail { ! { vect_slp_v16qi_store_unalign_1 } } } }
 }
 
 void warn_aggr_decl (void)
diff --git a/gcc/testsuite/gcc.dg/Wstringop-overflow-76.c b/gcc/testsuite/gcc.dg/Wstringop-overflow-76.c
index 5246726..ee315a0 100644
--- a/gcc/testsuite/gcc.dg/Wstringop-overflow-76.c
+++ b/gcc/testsuite/gcc.dg/Wstringop-overflow-76.c
@@ -27,10 +27,10 @@ void max_a3_a5 (int i)
      by its own warning independently of -Wstringop-overflow.  */
   char *d = MAX (p, q);
 
-  d[2] = 0;         // { dg-warning "writing 4 bytes into a region of size 3" "pr102706" { target { vect_slp_v4qi_store } } }
+  d[2] = 0;         // { dg-warning "writing 4 bytes into a region of size 3" "pr102706" { target { vect_slp_v4qi_store_unalign } } }
   d[3] = 0;
   d[4] = 0;
-  d[5] = 0;         // { dg-warning "writing 1 byte into a region of size 0" "pr102706" { xfail { vect_slp_v4qi_store } } }
+  d[5] = 0;         // { dg-warning "writing 1 byte into a region of size 0" "pr102706" { xfail { vect_slp_v4qi_store_unalign } } }
 }
 
 
@@ -44,10 +44,10 @@ void max_b6_b4 (int i)
   char *q = b4 + i;
   char *d = MAX (p, q);
 
-  d[3] = 0;         // { dg-warning "writing 4 bytes into a region of size 3" "pr102706" { target { vect_slp_v4qi_store } } }
+  d[3] = 0;         // { dg-warning "writing 4 bytes into a region of size 3" "pr102706" { target { vect_slp_v4qi_store_unalign } } }
   d[4] = 0;
   d[5] = 0;
-  d[6] = 0;         // { dg-warning "writing 1 byte into a region of size 0" "pr102706" { xfail { vect_slp_v4qi_store } } }
+  d[6] = 0;         // { dg-warning "writing 1 byte into a region of size 0" "pr102706" { xfail { vect_slp_v4qi_store_unalign } } }
 }
 
 
@@ -83,7 +83,7 @@ struct A3_5
 {
   char a3[3];  // { dg-message "at offset 3 into destination object 'a3' of size 3" "pr??????" { xfail *-*-* } }
   // refer to pr102697 for xfail
-  char a5[5];  // { dg-message "at offset 5 into destination object 'a5' of size 5" "note" { xfail { vect_slp_v4qi_store } } }
+  char a5[5];  // { dg-message "at offset 5 into destination object 'a5' of size 5" "note" { xfail { vect_slp_v4qi_store_unalign } } }
 };
 
 void max_A3_A5 (int i, struct A3_5 *pa3_5)
@@ -96,7 +96,7 @@ void max_A3_A5 (int i, struct A3_5 *pa3_5)
   d[2] = 0;
   d[3] = 0;         // { dg-warning "writing 1 byte into a region of size 0" "pr??????" { xfail *-*-* } }
   d[4] = 0;
-  d[5] = 0;         // { dg-warning "writing 1 byte into a region of size 0" "pr102697" { xfail { vect_slp_v4qi_store } } }
+  d[5] = 0;         // { dg-warning "writing 1 byte into a region of size 0" "pr102697" { xfail { vect_slp_v4qi_store_unalign } } }
 }
 
 
@@ -104,7 +104,7 @@ struct B4_B6
 {
   char b4[4];
   // refer to pr102697 for xfail
-  char b6[6];       // { dg-message "at offset \[^a-zA-Z\n\r\]*6\[^a-zA-Z0-9\]* into destination object 'b6' of size 6" "note" { xfail { vect_slp_v4qi_store } } }
+  char b6[6];       // { dg-message "at offset \[^a-zA-Z\n\r\]*6\[^a-zA-Z0-9\]* into destination object 'b6' of size 6" "note" { xfail { vect_slp_v4qi_store_unalign } } }
 };
 
 void max_B6_B4 (int i, struct B4_B6 *pb4_b6)
@@ -116,7 +116,7 @@ void max_B6_B4 (int i, struct B4_B6 *pb4_b6)
   d[3] = 0;
   d[4] = 0;
   d[5] = 0;
-  d[6] = 0;         // { dg-warning "writing 1 byte into a region of size 0" "pr102697" { xfail { vect_slp_v4qi_store } } }
+  d[6] = 0;         // { dg-warning "writing 1 byte into a region of size 0" "pr102697" { xfail { vect_slp_v4qi_store_unalign } } }
 }
 
 
diff --git a/gcc/testsuite/gcc.dg/Wstringop-overflow-77.c b/gcc/testsuite/gcc.dg/Wstringop-overflow-77.c
new file mode 100644
index 0000000..732f568
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/Wstringop-overflow-77.c
@@ -0,0 +1,516 @@
+/* PR middle-end/102453 - buffer overflow by atomic built-ins not diagnosed
+   Verify that out-of-bounds accesses by atomic functions are diagnosed with
+   optimization disabled.
+   { dg-do compile }
+   { dg-options "-O0 -Wall -ftrack-macro-expansion=0" }  */
+
+#ifndef __cplusplus
+#  define bool _Bool
+#endif
+
+#define add_fetch(p, q)    __atomic_add_fetch (p, q, 0)
+#define sub_fetch(p, q)    __atomic_sub_fetch (p, q, 0)
+#define and_fetch(p, q)    __atomic_and_fetch (p, q, 0)
+#define or_fetch(p, q)     __atomic_or_fetch (p, q, 0)
+#define xor_fetch(p, q)    __atomic_xor_fetch (p, q, 0)
+#define nand_fetch(p, q)   __atomic_nand_fetch (p, q, 0)
+#define exchange(p, q, r)  __atomic_exchange (p, q, r, 0)
+#define exchange_n(p, n)   __atomic_exchange_n (p, n, 0)
+#define cmpxchg(p, q, r)   __atomic_compare_exchange (p, q, r, 0, 0, 0)
+
+typedef __SIZE_TYPE__ size_t;
+
+void sink (void*, ...);
+#define sink(...) sink (0, __VA_ARGS__)
+
+extern _Bool eb;
+extern char ec;
+extern short int esi;
+extern int ei;
+extern long int eli;
+extern long long int elli;
+
+extern const _Bool ecb;
+extern const char ecc;
+extern const short int ecsi;
+extern const int eci;
+extern const long int ecli;
+extern const long long int eclli;
+
+extern _Atomic _Bool eab;
+extern _Atomic char eac;
+extern _Atomic short int easi;
+extern _Atomic int eai;
+extern _Atomic long int eali;
+extern _Atomic long long int ealli;
+
+extern _Atomic const _Bool eacb;
+extern _Atomic const char eacc;
+extern _Atomic const short int eacsi;
+extern _Atomic const int eaci;
+extern _Atomic const long int eacli;
+extern _Atomic const long long int eaclli;
+
+
+void nowarn_atomic_add_fetch (void)
+{
+  add_fetch (&eac, ecc);
+  add_fetch (&easi, esi);
+  add_fetch (&eai, ei);
+  add_fetch (&eali, eli);
+  add_fetch (&ealli, elli);
+}
+
+
+void warn_atomic_add_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  add_fetch (pc, ecc);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  add_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  add_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  add_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  add_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  add_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  add_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  add_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  add_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  add_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  add_fetch (plli, eali);               // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  add_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+}
+
+
+void nowarn_atomic_sub_fetch (void)
+{
+  _Atomic char *pc = &eac;
+  sub_fetch (pc, ecc);
+
+  _Atomic short *psi = &easi;
+  sub_fetch (psi, esi);
+
+  _Atomic int *pi = &eai;
+  sub_fetch (pi, ei);
+
+  _Atomic long *pli = &eali;
+  sub_fetch (pli, eli);
+
+  _Atomic long long *plli = &ealli;
+  sub_fetch (plli, elli);
+}
+
+
+void warn_atomic_sub_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  sub_fetch (pc, ecc);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  sub_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  sub_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  sub_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  sub_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  sub_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  sub_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  sub_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  sub_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  sub_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  sub_fetch (plli, eali);               // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  sub_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+}
+
+
+void nowarn_atomic_and_fetch (void)
+{
+  _Atomic char *pc = &eac;
+  and_fetch (pc, ecc);
+
+  _Atomic short *psi = &easi;
+  and_fetch (psi, esi);
+
+  _Atomic int *pi = &eai;
+  and_fetch (pi, ei);
+
+  _Atomic long *pli = &eali;
+  and_fetch (pli, eli);
+
+  _Atomic long long *plli = &ealli;
+  and_fetch (plli, elli);
+}
+
+
+void warn_atomic_and_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  and_fetch (pc, ecc);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  and_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  and_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  and_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  and_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  and_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  and_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  and_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  and_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  and_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  and_fetch (plli, eali);               // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  and_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+}
+
+
+void nowarn_atomic_or_fetch (void)
+{
+  _Atomic char *pc = &eac;
+  or_fetch (pc, ecc);
+
+  _Atomic short *psi = &easi;
+  or_fetch (psi, esi);
+
+  _Atomic int *pi = &eai;
+  or_fetch (pi, ei);
+
+  _Atomic long *pli = &eali;
+  or_fetch (pli, eli);
+
+  _Atomic long long *plli = &ealli;
+  or_fetch (plli, elli);
+}
+
+
+void warn_atomic_or_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  or_fetch (pc, ecc);                   // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  or_fetch (psi, esi);                  // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  or_fetch (psi, esi);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  or_fetch (pi, ei);                    // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  or_fetch (pi, ei);                    // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  or_fetch (pi, ei);                    // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  or_fetch (pli, eli);                  // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  or_fetch (pli, eli);                  // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  or_fetch (pli, eli);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  or_fetch (plli, elli);                // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  or_fetch (plli, eali);                // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  or_fetch (plli, elli);                // { dg-warning "-Wstringop-overflow" }
+}
+
+
+void nowarn_atomic_xor_fetch (void)
+{
+  _Atomic char *pc = &eac;
+  xor_fetch (pc, ecc);
+
+  _Atomic short *psi = &easi;
+  xor_fetch (psi, esi);
+
+  _Atomic int *pi = &eai;
+  xor_fetch (pi, ei);
+
+  _Atomic long *pli = &eali;
+  xor_fetch (pli, eli);
+
+  _Atomic long long *plli = &ealli;
+  xor_fetch (plli, elli);
+}
+
+
+void warn_atomic_xor_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  xor_fetch (pc, ecc);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  xor_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 1);
+  xor_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  xor_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  xor_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  xor_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  xor_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  xor_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  xor_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  xor_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&eali + 1);
+  xor_fetch (plli, eali);               // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  xor_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+}
+
+
+void nowarn_atomic_nand_fetch (void)
+{
+  _Atomic char *pc = &eac;
+  nand_fetch (pc, ecc);
+
+  _Atomic short *psi = &easi;
+  nand_fetch (psi, esi);
+
+  _Atomic int *pi = &eai;
+  nand_fetch (pi, ei);
+
+  _Atomic long *pli = &eali;
+  nand_fetch (pli, eli);
+
+  _Atomic long long *plli = &ealli;
+  nand_fetch (plli, elli);
+}
+
+
+void warn_atomic_nand_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  nand_fetch (pc, ecc);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  nand_fetch (psi, esi);                // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 1);
+  nand_fetch (psi, esi);                // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  nand_fetch (pi, ei);                  // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  nand_fetch (pi, ei);                  // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  nand_fetch (pi, ei);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  nand_fetch (pli, eli);                // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  nand_fetch (pli, eli);                // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  nand_fetch (pli, eli);                // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  nand_fetch (plli, elli);              // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&eai + 1);
+  nand_fetch (plli, eali);              // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  nand_fetch (plli, elli);              // { dg-warning "-Wstringop-overflow" }
+}
+
+
+void nowarn_atomic_exchange (void)
+{
+  char rc;
+  _Atomic char *pc = &eac;
+  exchange (pc, &ecc, &rc);
+
+  short rsi;
+  _Atomic short *psi = &easi;
+  exchange (psi, &esi, &rsi);
+
+  int ri;
+  _Atomic int *pi = &eai;
+  exchange (pi, &ei, &ri);
+
+  long rli;
+  _Atomic long *pli = &eali;
+  exchange (pli, &eli, &rli);
+
+  long long rlli;
+  _Atomic long long *plli = &ealli;
+  exchange (plli, &elli, &rlli);
+
+  sink (&rc, &rsi, &ri, &rli, &rlli);
+}
+
+void warn_atomic_exchange (void)
+{
+  char rc;
+  _Atomic char *pc = &eac + 1;
+  exchange (pc, &ecc, &rc);             // { dg-warning "-Wstringop-overflow" }
+
+  short rsi[2];
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  exchange (psi, &ecsi, rsi);           // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  exchange (psi, &ecsi, rsi + 1);       // { dg-warning "-Wstringop-overflow" }
+
+  int ri[3];
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  exchange (pi, &eci, ri);              // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  exchange (pi, &eci, ri + 1);          // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  exchange (pi, &eci, ri + 2);          // { dg-warning "-Wstringop-overflow" }
+
+  long rli[3];
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  exchange (pli, &ecli, rli);           // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  exchange (pli, &ecli, rli + 1);       // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  exchange (pli, &ecli, rli + 2);       // { dg-warning "-Wstringop-overflow" }
+
+  long long rlli[3];
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  exchange (plli, &eclli, rlli);        // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  exchange (plli, &eclli, rlli + 1);    // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  exchange (plli, &eclli, rlli + 2);    // { dg-warning "-Wstringop-overflow" }
+
+  sink (&rc, rsi, ri, rli, rlli);
+}
+
+
+void nowarn_atomic_exchange_n (_Atomic unsigned char *pauc,
+			       _Atomic unsigned short *pausi,
+			       _Atomic unsigned int *paui,
+			       _Atomic unsigned long *pauli,
+			       _Atomic unsigned long long *paulli)
+{
+  char rc = exchange_n (&eac, ecc);
+  short rsi = exchange_n (&easi, esi);
+  int ri = exchange_n (&eai, ei);
+  long rli = exchange_n (&eali, eli);
+  long long rlli = exchange_n (&ealli, elli);
+
+  sink (rc, rsi, ri, rli, rlli);
+
+  char ruc = exchange_n (pauc, ecc);
+  short rusi = exchange_n (pausi, esi);
+  int rui = exchange_n (paui, ei);
+  long ruli = exchange_n (pauli, eli);
+  long long rulli = exchange_n (paulli, elli);
+
+  sink (ruc, rusi, rui, ruli, rulli);
+}
+
+
+void warn_atomic_exchange_n (void)
+{
+  _Atomic char *pc = &eac + 1;
+  char rc = exchange_n (pc, ecc);       // { dg-warning "-Wstringop-overflow" }
+
+  short rsi[2];
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  rsi[0] = exchange_n (psi, ecsi);      // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  rsi[1] = exchange_n (psi, ecsi);      // { dg-warning "-Wstringop-overflow" }
+
+  int ri[3];
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  ri[0] = exchange_n (pi, eci);         // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  ri[1] = exchange_n (pi, eci);         // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  ri[2] = exchange_n (pi, eci);         // { dg-warning "-Wstringop-overflow" }
+
+  long rli[3];
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  rli[0] = exchange_n (pli, ecli);      // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  rli[1] = exchange_n (pli, ecli);      // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  rli[2] = exchange_n (pli, ecli);      // { dg-warning "-Wstringop-overflow" }
+
+  long long rlli[3];
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  rlli[0] = exchange_n (plli, eclli);   // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  rlli[1] = exchange_n (plli, eclli);   // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  rlli[2] = exchange_n (plli, eclli);   // { dg-warning "-Wstringop-overflow" }
+
+  sink (&rc, rsi, ri, rli, rlli);
+}
+
+
+void warn_atomic_compare_exchange (void)
+{
+  _Atomic char *pc = &eac + 1;
+  cmpxchg (pc, &ec, &ecc);              // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  cmpxchg (psi, &esi, &ecsi);           // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  cmpxchg (psi, &esi, &ecsi);           // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  cmpxchg (pi, &ei, &eci);              // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  cmpxchg (pi, &ei, &eci);              // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  cmpxchg (pi, &ei, &eci);              // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  cmpxchg (pli, &eli, &ecli);           // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  cmpxchg (pli, &eli, &ecli);           // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  cmpxchg (pli, &eli, &ecli);           // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  cmpxchg (plli, &elli, &eclli);        // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  cmpxchg (plli, &elli, &eclli);        // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  cmpxchg (plli, &elli, &eclli);        // { dg-warning "-Wstringop-overflow" }
+}
diff --git a/gcc/testsuite/gcc.dg/Wstringop-overflow-78.c b/gcc/testsuite/gcc.dg/Wstringop-overflow-78.c
new file mode 100644
index 0000000..a25a418
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/Wstringop-overflow-78.c
@@ -0,0 +1,518 @@
+/* PR middle-end/102453 - buffer overflow by atomic built-ins not diagnosed
+   Verify that out-of-bounds accesses by atomic functions are diagnosed with
+   optimization enabled.
+   { dg-do compile }
+   { dg-options "-O3 -Wall -ftrack-macro-expansion=0" }  */
+
+#ifndef __cplusplus
+#  define bool _Bool
+#endif
+
+#define NOIPA __attribute__ ((noipa))
+
+#define add_fetch(p, q)   __atomic_add_fetch (p, q, 0)
+#define sub_fetch(p, q)   __atomic_sub_fetch (p, q, 0)
+#define and_fetch(p, q)   __atomic_and_fetch (p, q, 0)
+#define or_fetch(p, q)    __atomic_or_fetch (p, q, 0)
+#define xor_fetch(p, q)   __atomic_xor_fetch (p, q, 0)
+#define nand_fetch(p, q)  __atomic_nand_fetch (p, q, 0)
+#define exchange(p, q, r) __atomic_exchange (p, q, r, 0)
+#define exchange_n(p, n)  __atomic_exchange_n (p, n, 0)
+#define cmpxchg(p, q, r)  __atomic_compare_exchange (p, q, r, __COUNTER__, 0, 0)
+
+typedef __SIZE_TYPE__ size_t;
+
+void sink (void*, ...);
+#define sink(...) sink (0, __VA_ARGS__)
+
+extern _Bool eb;
+extern char ec;
+extern short int esi;
+extern int ei;
+extern long int eli;
+extern long long int elli;
+
+extern const _Bool ecb;
+extern const char ecc;
+extern const short int ecsi;
+extern const int eci;
+extern const long int ecli;
+extern const long long int eclli;
+
+extern _Atomic _Bool eab;
+extern _Atomic char eac;
+extern _Atomic short int easi;
+extern _Atomic int eai;
+extern _Atomic long int eali;
+extern _Atomic long long int ealli;
+
+extern _Atomic const _Bool eacb;
+extern _Atomic const char eacc;
+extern _Atomic const short int eacsi;
+extern _Atomic const int eaci;
+extern _Atomic const long int eacli;
+extern _Atomic const long long int eaclli;
+
+
+NOIPA void nowarn_atomic_add_fetch (void)
+{
+  add_fetch (&eac, ecc);
+  add_fetch (&easi, esi);
+  add_fetch (&eai, ei);
+  add_fetch (&eali, eli);
+  add_fetch (&ealli, elli);
+}
+
+
+NOIPA void warn_atomic_add_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  add_fetch (pc, ecc);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  add_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  add_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  add_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  add_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  add_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  add_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  add_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  add_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  add_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  add_fetch (plli, eali);               // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  add_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+}
+
+
+NOIPA void nowarn_atomic_sub_fetch (void)
+{
+  _Atomic char *pc = &eac;
+  sub_fetch (pc, ecc);
+
+  _Atomic short *psi = &easi;
+  sub_fetch (psi, esi);
+
+  _Atomic int *pi = &eai;
+  sub_fetch (pi, ei);
+
+  _Atomic long *pli = &eali;
+  sub_fetch (pli, eli);
+
+  _Atomic long long *plli = &ealli;
+  sub_fetch (plli, elli);
+}
+
+
+NOIPA void warn_atomic_sub_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  sub_fetch (pc, ecc);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  sub_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  sub_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  sub_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  sub_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  sub_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  sub_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  sub_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  sub_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  sub_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  sub_fetch (plli, eali);               // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  sub_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+}
+
+
+NOIPA void nowarn_atomic_and_fetch (void)
+{
+  _Atomic char *pc = &eac;
+  and_fetch (pc, ecc);
+
+  _Atomic short *psi = &easi;
+  and_fetch (psi, esi);
+
+  _Atomic int *pi = &eai;
+  and_fetch (pi, ei);
+
+  _Atomic long *pli = &eali;
+  and_fetch (pli, eli);
+
+  _Atomic long long *plli = &ealli;
+  and_fetch (plli, elli);
+}
+
+
+NOIPA void warn_atomic_and_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  and_fetch (pc, ecc);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  and_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  and_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  and_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  and_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  and_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  and_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  and_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  and_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  and_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  and_fetch (plli, eali);               // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  and_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+}
+
+
+NOIPA void nowarn_atomic_or_fetch (void)
+{
+  _Atomic char *pc = &eac;
+  or_fetch (pc, ecc);
+
+  _Atomic short *psi = &easi;
+  or_fetch (psi, esi);
+
+  _Atomic int *pi = &eai;
+  or_fetch (pi, ei);
+
+  _Atomic long *pli = &eali;
+  or_fetch (pli, eli);
+
+  _Atomic long long *plli = &ealli;
+  or_fetch (plli, elli);
+}
+
+
+NOIPA void warn_atomic_or_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  or_fetch (pc, ecc);                   // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  or_fetch (psi, esi);                  // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  or_fetch (psi, esi);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  or_fetch (pi, ei);                    // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  or_fetch (pi, ei);                    // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  or_fetch (pi, ei);                    // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  or_fetch (pli, eli);                  // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  or_fetch (pli, eli);                  // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  or_fetch (pli, eli);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  or_fetch (plli, elli);                // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  or_fetch (plli, eali);                // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  or_fetch (plli, elli);                // { dg-warning "-Wstringop-overflow" }
+}
+
+
+NOIPA void nowarn_atomic_xor_fetch (void)
+{
+  _Atomic char *pc = &eac;
+  xor_fetch (pc, ecc);
+
+  _Atomic short *psi = &easi;
+  xor_fetch (psi, esi);
+
+  _Atomic int *pi = &eai;
+  xor_fetch (pi, ei);
+
+  _Atomic long *pli = &eali;
+  xor_fetch (pli, eli);
+
+  _Atomic long long *plli = &ealli;
+  xor_fetch (plli, elli);
+}
+
+
+NOIPA void warn_atomic_xor_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  xor_fetch (pc, ecc);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  xor_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 1);
+  xor_fetch (psi, esi);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  xor_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  xor_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  xor_fetch (pi, ei);                   // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  xor_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  xor_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  xor_fetch (pli, eli);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  xor_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&eali + 1);
+  xor_fetch (plli, eali);               // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  xor_fetch (plli, elli);               // { dg-warning "-Wstringop-overflow" }
+}
+
+
+NOIPA void nowarn_atomic_nand_fetch (void)
+{
+  _Atomic char *pc = &eac;
+  nand_fetch (pc, ecc);
+
+  _Atomic short *psi = &easi;
+  nand_fetch (psi, esi);
+
+  _Atomic int *pi = &eai;
+  nand_fetch (pi, ei);
+
+  _Atomic long *pli = &eali;
+  nand_fetch (pli, eli);
+
+  _Atomic long long *plli = &ealli;
+  nand_fetch (plli, elli);
+}
+
+
+NOIPA void warn_atomic_nand_fetch (void)
+{
+  _Atomic char *pc = &eac + 1;
+  nand_fetch (pc, ecc);                 // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  nand_fetch (psi, esi);                // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 1);
+  nand_fetch (psi, esi);                // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  nand_fetch (pi, ei);                  // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  nand_fetch (pi, ei);                  // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  nand_fetch (pi, ei);                  // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  nand_fetch (pli, eli);                // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  nand_fetch (pli, eli);                // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  nand_fetch (pli, eli);                // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  nand_fetch (plli, elli);              // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&eai + 1);
+  nand_fetch (plli, eali);              // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  nand_fetch (plli, elli);              // { dg-warning "-Wstringop-overflow" }
+}
+
+
+NOIPA void nowarn_atomic_exchange (void)
+{
+  char rc;
+  _Atomic char *pc = &eac;
+  exchange (pc, &ecc, &rc);
+
+  short rsi;
+  _Atomic short *psi = &easi;
+  exchange (psi, &esi, &rsi);
+
+  int ri;
+  _Atomic int *pi = &eai;
+  exchange (pi, &ei, &ri);
+
+  long rli;
+  _Atomic long *pli = &eali;
+  exchange (pli, &eli, &rli);
+
+  long long rlli;
+  _Atomic long long *plli = &ealli;
+  exchange (plli, &elli, &rlli);
+
+  sink (&rc, &rsi, &ri, &rli, &rlli);
+}
+
+NOIPA void warn_atomic_exchange (void)
+{
+  char rc;
+  _Atomic char *pc = &eac + 1;
+  exchange (pc, &ecc, &rc);             // { dg-warning "-Wstringop-overflow" }
+
+  short rsi[2];
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  exchange (psi, &ecsi, rsi);           // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  exchange (psi, &ecsi, rsi + 1);       // { dg-warning "-Wstringop-overflow" }
+
+  int ri[3];
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  exchange (pi, &eci, ri);              // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  exchange (pi, &eci, ri + 1);          // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  exchange (pi, &eci, ri + 2);          // { dg-warning "-Wstringop-overflow" }
+
+  long rli[3];
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  exchange (pli, &ecli, rli);           // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  exchange (pli, &ecli, rli + 1);       // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  exchange (pli, &ecli, rli + 2);       // { dg-warning "-Wstringop-overflow" }
+
+  long long rlli[3];
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  exchange (plli, &eclli, rlli);        // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  exchange (plli, &eclli, rlli + 1);    // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  exchange (plli, &eclli, rlli + 2);    // { dg-warning "-Wstringop-overflow" }
+
+  sink (&rc, rsi, ri, rli, rlli);
+}
+
+
+NOIPA void nowarn_atomic_exchange_n (_Atomic unsigned char *pauc,
+			       _Atomic unsigned short *pausi,
+			       _Atomic unsigned int *paui,
+			       _Atomic unsigned long *pauli,
+			       _Atomic unsigned long long *paulli)
+{
+  char rc = exchange_n (&eac, ecc);
+  short rsi = exchange_n (&easi, esi);
+  int ri = exchange_n (&eai, ei);
+  long rli = exchange_n (&eali, eli);
+  long long rlli = exchange_n (&ealli, elli);
+
+  sink (rc, rsi, ri, rli, rlli);
+
+  char ruc = exchange_n (pauc, ecc);
+  short rusi = exchange_n (pausi, esi);
+  int rui = exchange_n (paui, ei);
+  long ruli = exchange_n (pauli, eli);
+  long long rulli = exchange_n (paulli, elli);
+
+  sink (ruc, rusi, rui, ruli, rulli);
+}
+
+
+NOIPA void warn_atomic_exchange_n (void)
+{
+  _Atomic char *pc = &eac + 1;
+  char rc = exchange_n (pc, ecc);       // { dg-warning "-Wstringop-overflow" }
+
+  short rsi[2];
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  rsi[0] = exchange_n (psi, ecsi);      // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  rsi[1] = exchange_n (psi, ecsi);      // { dg-warning "-Wstringop-overflow" }
+
+  int ri[3];
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  ri[0] = exchange_n (pi, eci);         // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  ri[1] = exchange_n (pi, eci);         // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  ri[2] = exchange_n (pi, eci);         // { dg-warning "-Wstringop-overflow" }
+
+  long rli[3];
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  rli[0] = exchange_n (pli, ecli);      // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  rli[1] = exchange_n (pli, ecli);      // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  rli[2] = exchange_n (pli, ecli);      // { dg-warning "-Wstringop-overflow" }
+
+  long long rlli[3];
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  rlli[0] = exchange_n (plli, eclli);   // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  rlli[1] = exchange_n (plli, eclli);   // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  rlli[2] = exchange_n (plli, eclli);   // { dg-warning "-Wstringop-overflow" }
+
+  sink (&rc, rsi, ri, rli, rlli);
+}
+
+
+NOIPA void warn_atomic_compare_exchange (void)
+{
+  _Atomic char *pc = &eac + 1;
+  cmpxchg (pc, &ec, &ecc);              // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic short *psi = (_Atomic short*)((char*)&easi + 1);
+  cmpxchg (psi, &esi, &ecsi);           // { dg-warning "-Wstringop-overflow" }
+  psi = (_Atomic short*)((char*)&easi + 2);
+  cmpxchg (psi, &esi, &ecsi);           // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic int *pi = (_Atomic int*)((char*)&eai + 1);
+  cmpxchg (pi, &ei, &eci);              // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + 2);
+  cmpxchg (pi, &ei, &eci);              // { dg-warning "-Wstringop-overflow" }
+  pi = (_Atomic int*)((char*)&eai + sizeof eai);
+  cmpxchg (pi, &ei, &eci);              // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long *pli = (_Atomic long*)((char*)&eali + 1);
+  cmpxchg (pli, &eli, &ecli);           // { dg-warning "-Wstringop-overflow" }
+  pli = (_Atomic long*)((char*)&eali + 1);
+  cmpxchg (pli, &eli, &ecli);           // { dg-warning "-Wstringop-overflow" }
+  pli = &eali + 1;
+  cmpxchg (pli, &eli, &ecli);           // { dg-warning "-Wstringop-overflow" }
+
+  _Atomic long long *plli = (_Atomic long long*)((char*)&ealli + 1);
+  cmpxchg (plli, &elli, &eclli);        // { dg-warning "-Wstringop-overflow" }
+  plli = (_Atomic long long*)((char*)&ealli + 1);
+  cmpxchg (plli, &elli, &eclli);        // { dg-warning "-Wstringop-overflow" }
+  plli = &ealli + 1;
+  cmpxchg (plli, &elli, &eclli);        // { dg-warning "-Wstringop-overflow" }
+}
diff --git a/gcc/testsuite/gcc.dg/Wstringop-overflow-79.c b/gcc/testsuite/gcc.dg/Wstringop-overflow-79.c
new file mode 100644
index 0000000..15eb26f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/Wstringop-overflow-79.c
@@ -0,0 +1,70 @@
+/* Verify that a separate note is issued for each offset into the same
+   object after a -Wstringop-overflow.  Since all arguments are known
+   the test doesn't need optimization.  Wstringop-overflow-79.c verifies
+   they're also issued at -O2.
+   { dg-do compile }
+   { dg-options "-O0 -Wno-array-bounds" } */
+
+extern char a[8];                 // dg-message at offset \\\[3, 6] into destination object 'a'" "note 1" }
+                                  // dg-message at offset \\\[5, 8] into destination object 'a'" "note 2" { target *-*-* } .-1 }
+
+void test_2_notes (int i)
+{
+  char *p = i ? a + 3 : a + 5;
+  __builtin_memset (p, 0, 7);     // { dg-warning "-Wstringop-overflow" }
+}
+
+
+extern char b[8];                 // dg-message at offset \\\[3, 6] into destination object 'b'" "note 1" }
+                                  // dg-message at offset \\\[4, 7] into destination object 'b'" "note 2" { target *-*-* } .-1 }
+                                  // dg-message at offset \\\[5, 8] into destination object 'b'" "note 3" { target *-*-* } .-2 }
+
+void test_3_notes (int i)
+{
+  char *p = i < 0 ? b + 3 : 0 < i ? b + 5 : b + 4;
+  __builtin_memset (p, 0, 7);     // { dg-warning "-Wstringop-overflow" }
+}
+
+
+extern char c[8];                 // dg-message at offset \\\[3, 6] into destination object 'c'" "note 1" }
+                                  // dg-message at offset \\\[4, 7] into destination object 'c'" "note 2" { target *-*-* } .-1 }
+                                  // dg-message at offset \\\[5, 8] into destination object 'c'" "note 3" { target *-*-* } .-2 }
+                                  // dg-message at offset \\\[6, 8] into destination object 'c'" "note 3" { target *-*-* } .-2 }
+
+void test_4_notes (int i)
+{
+  char *p;
+  if (i < -1)
+    p = c + 3;
+  else if (i < 0)
+    p = c + 4;
+  else if (0 < i)
+    p = c + 6;
+  else
+    p = c + 5;
+
+  __builtin_memset (p, 0, 7);     // { dg-warning "-Wstringop-overflow" }
+}
+
+
+extern char d[8];                 // dg-message at offset \\\[3, 6] into destination object 'd'" "note 1" }
+                                  // dg-message at offset \\\[4, 7] into destination object 'd'" "note 2" { target *-*-* } .-1 }
+                                  // dg-message at offset \\\[5, 8] into destination object 'd'" "note 3" { target *-*-* } .-2 }
+                                  // dg-message at offset \\\[6, 8] into destination object 'd'" "note 3" { target *-*-* } .-3 }
+                                  // dg-message at offset \\\[7, 8] into destination object 'd'" "note 3" { target *-*-* } .-4 }
+
+void test_5_notes (int i)
+{
+  char *p;
+  switch (i)
+    {
+    case -9: p = d + 3; break;
+    case -5: p = d + 4; break;
+    case  0: p = d + 5; break;
+    case  3: p = d + 6; break;
+    case  4: p = d + 7; break;
+    default: return;
+    }
+
+  __builtin_memset (p, 0, 7);     // { dg-warning "-Wstringop-overflow" }
+}
diff --git a/gcc/testsuite/gcc.dg/Wstringop-overflow-80.c b/gcc/testsuite/gcc.dg/Wstringop-overflow-80.c
new file mode 100644
index 0000000..1628c2f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/Wstringop-overflow-80.c
@@ -0,0 +1,70 @@
+/* Verify that a separate note is issued for each offset into the same
+   object after a -Wstringop-overflow.  Even though the warnings don't
+   need optimization the test enables it to verify they're still issued
+   with it.  Wstringop-overflow-78.c verifies they're issued at -O0.
+   { dg-do compile }
+   { dg-options "-O2 -Wno-array-bounds" } */
+
+extern char a[8];                 // dg-message at offset \\\[3, 6] into destination object 'a'" "note 1" }
+                                  // dg-message at offset \\\[5, 8] into destination object 'a'" "note 2" { target *-*-* } .-1 }
+
+void test_2_notes (int i)
+{
+  char *p = i ? a + 3 : a + 5;
+  __builtin_memset (p, 0, 7);     // { dg-warning "-Wstringop-overflow" }
+}
+
+
+extern char b[8];                 // dg-message at offset \\\[3, 6] into destination object 'b'" "note 1" }
+                                  // dg-message at offset \\\[4, 7] into destination object 'b'" "note 2" { target *-*-* } .-1 }
+                                  // dg-message at offset \\\[5, 8] into destination object 'b'" "note 3" { target *-*-* } .-2 }
+
+void test_3_notes (int i)
+{
+  char *p = i < 0 ? b + 3 : 0 < i ? b + 5 : b + 4;
+  __builtin_memset (p, 0, 7);     // { dg-warning "-Wstringop-overflow" }
+}
+
+
+extern char c[8];                 // dg-message at offset \\\[3, 6] into destination object 'c'" "note 1" }
+                                  // dg-message at offset \\\[4, 7] into destination object 'c'" "note 2" { target *-*-* } .-1 }
+                                  // dg-message at offset \\\[5, 8] into destination object 'c'" "note 3" { target *-*-* } .-2 }
+                                  // dg-message at offset \\\[6, 8] into destination object 'c'" "note 3" { target *-*-* } .-2 }
+
+void test_4_notes (int i)
+{
+  char *p;
+  if (i < -1)
+    p = c + 3;
+  else if (i < 0)
+    p = c + 4;
+  else if (0 < i)
+    p = c + 6;
+  else
+    p = c + 5;
+
+  __builtin_memset (p, 0, 7);     // { dg-warning "-Wstringop-overflow" }
+}
+
+
+extern char d[8];                 // dg-message at offset \\\[3, 6] into destination object 'd'" "note 1" }
+                                  // dg-message at offset \\\[4, 7] into destination object 'd'" "note 2" { target *-*-* } .-1 }
+                                  // dg-message at offset \\\[5, 8] into destination object 'd'" "note 3" { target *-*-* } .-2 }
+                                  // dg-message at offset \\\[6, 8] into destination object 'd'" "note 3" { target *-*-* } .-3 }
+                                  // dg-message at offset \\\[7, 8] into destination object 'd'" "note 3" { target *-*-* } .-4 }
+
+void test_5_notes (int i)
+{
+  char *p;
+  switch (i)
+    {
+    case -9: p = d + 3; break;
+    case -5: p = d + 4; break;
+    case  0: p = d + 5; break;
+    case  3: p = d + 6; break;
+    case  4: p = d + 7; break;
+    default: return;
+    }
+
+  __builtin_memset (p, 0, 7);     // { dg-warning "-Wstringop-overflow" }
+}
diff --git a/gcc/testsuite/gcc.dg/Wstringop-overflow-81.c b/gcc/testsuite/gcc.dg/Wstringop-overflow-81.c
new file mode 100644
index 0000000..e8bc327
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/Wstringop-overflow-81.c
@@ -0,0 +1,38 @@
+/* Verify that -Wstringop-overflow uses context-sensitive range info
+   even at -O0.
+   { dg-do compile }
+   { dg-options "-O0 -Wall" } */
+
+extern void* memset (void*, int, __SIZE_TYPE__);
+
+char a[8];
+
+void warn_offset_range (int i)
+{
+  if (i < 4)
+    i = 4;
+  memset (a + i, 0, 5);       // { dg-warning "writing 5 bytes into a region of size 4 " }
+}
+
+void warn_size_range (int i, int n)
+{
+  if (n < 5)
+    n = 5;
+
+  memset (a + 4, 1, n);      // { dg-warning "writing between 5 and \\d+ bytes into a region of size 4 " }
+}
+
+void warn_offset_and_size_range (int i, int n)
+{
+  if (n < 5)
+    n = 5;
+
+  if (i < 4)
+    {
+      if (n < 9)
+	n = 9;
+      memset (a + i, 1, n);   // { dg-warning "writing between 9 and \\d+ bytes into a region of size 8 " }
+    }
+  else
+    memset (a + i, 0, n);     // { dg-warning "writing between 5 and \\d+ bytes into a region of size 4 " }
+}
diff --git a/gcc/testsuite/gcc.dg/Wzero-length-array-bounds-2.c b/gcc/testsuite/gcc.dg/Wzero-length-array-bounds-2.c
index b232149..2ef5ccd 100644
--- a/gcc/testsuite/gcc.dg/Wzero-length-array-bounds-2.c
+++ b/gcc/testsuite/gcc.dg/Wzero-length-array-bounds-2.c
@@ -87,7 +87,7 @@ void test_C_global_buf (void)
   p->b1.a[ 1].i = 0;     // { dg-warning "\\\[-Wzero-length-bounds" }
   sink (p);
 
-  p->b2.a[ 0].i = 0;    // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v2si_store &&  { ! vect_slp_v4si_store } } } }
+  p->b2.a[ 0].i = 0;    // { dg-warning "\\\[-Wstringop-overflow" "pr102706" { target { vect_slp_v2si_store_align &&  { ! vect_slp_v4si_store_unalign } } } }
   p->b2.a[ 1].i = 0;
   p->b2.a[ 2].i = 0;     // { dg-warning "\\\[-Warray-bounds" }
   p->b2.a[ 3].i = 0;     // { dg-warning "\\\[-Warray-bounds" }
diff --git a/gcc/testsuite/gcc.dg/analyzer/pr93382.c b/gcc/testsuite/gcc.dg/analyzer/pr93382.c
index 210b97d..1e6612d 100644
--- a/gcc/testsuite/gcc.dg/analyzer/pr93382.c
+++ b/gcc/testsuite/gcc.dg/analyzer/pr93382.c
@@ -23,5 +23,5 @@ int
 pl (void)
 {
   ql ();
-  return arr[idx]; /* { dg-warning "use of tainted value 'idx' in array lookup without bounds checking" "" { xfail *-*-* } } */
+  return arr[idx]; /* { dg-warning "use of attacker-controlled value 'idx' in array lookup without bounds checking" "" { xfail *-*-* } } */
 }
diff --git a/gcc/testsuite/gcc.dg/analyzer/taint-alloc-1.c b/gcc/testsuite/gcc.dg/analyzer/taint-alloc-1.c
new file mode 100644
index 0000000..102aa4b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/analyzer/taint-alloc-1.c
@@ -0,0 +1,64 @@
+// TODO: remove need for this option
+/* { dg-additional-options "-fanalyzer-checker=taint" } */
+
+#include "analyzer-decls.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct foo
+{
+  size_t sz;
+};
+
+/* malloc with tainted size.  */
+
+void *test_1 (FILE *f)
+{
+  struct foo tmp;
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) { /* { dg-message "\\(\[0-9\]+\\) 'tmp' gets an unchecked value here" "event: tmp gets unchecked value" { xfail *-*-* } } */
+                                             /* { dg-message "\\(\[0-9\]+\\) following 'true' branch\\.\\.\\." "event: following true branch" { target *-*-* } .-1 } */
+    __analyzer_dump_state ("taint", tmp.sz); /* { dg-warning "state: 'tainted'" } */
+    /* { dg-message "\\(\[0-9\]+\\) \\.\\.\\.to here" "event: to here" { target *-*-* } .-1 } */
+    
+    return malloc (tmp.sz); /* { dg-warning "use of attacker-controlled value 'tmp\\.sz' as allocation size without upper-bounds checking" "warning" } */
+    /* { dg-message "23: \\(\[0-9\]+\\) 'tmp.i' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.i has an unchecked value" { xfail *-*-* } .-1 } */
+    /* { dg-message "\\(\[0-9\]+\\) use of attacker-controlled value 'tmp\\.sz' as allocation size without upper-bounds checking" "final event" { target *-*-* } .-2 } */
+    
+    // TOOD: better messages for state changes
+  }
+  return 0;
+}
+
+/* VLA with tainted size.  */
+
+void *test_2 (FILE *f)
+{
+  struct foo tmp;
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) { /* { dg-message "\\(\[0-9\]+\\) 'tmp' gets an unchecked value here" "event: tmp gets unchecked value" { xfail *-*-* } } */
+                                             /* { dg-message "\\(\[0-9\]+\\) following 'true' branch\\.\\.\\." "event: following true branch" { target *-*-* } .-1 } */
+    __analyzer_dump_state ("taint", tmp.sz); /* { dg-warning "state: 'tainted'" } */
+    /* { dg-message "\\(\[0-9\]+\\) \\.\\.\\.to here" "event: to here" { target *-*-* } .-1 } */
+
+    /* VLA with tainted size.  */
+    {
+      char buf[tmp.sz]; /* { dg-warning "use of attacker-controlled value 'tmp\\.sz' as allocation size without upper-bounds checking" "warning" } */
+      /* { dg-message "\\(\[0-9\]+\\) 'tmp.i' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.i has an unchecked value" { xfail *-*-* } .-1 } */
+      /* { dg-message "\\(\[0-9\]+\\) use of attacker-controlled value 'tmp\\.sz' as allocation size without upper-bounds checking" "final event" { target *-*-* } .-2 } */
+      fread (buf, tmp.sz, 1, f);
+    }
+    
+    // TOOD: better messages for state changes
+  }
+  return 0;
+}
+
+void *test_3 (FILE *f)
+{
+  int num;
+  fread (&num, sizeof (int), 1, f);
+  __analyzer_dump_state ("taint", num); /* { dg-warning "state: 'tainted'" } */
+  __analyzer_dump_state ("taint", num * 16); /* { dg-warning "state: 'tainted'" } */
+  __analyzer_dump_state ("taint", (size_t)(num * 16)); /* { dg-warning "state: 'tainted'" } */
+  return malloc (num * 16); /* { dg-warning "use of attacker-controlled value 'num \\* 16' as allocation size without upper-bounds checking" } */
+}
diff --git a/gcc/testsuite/gcc.dg/analyzer/taint-alloc-2.c b/gcc/testsuite/gcc.dg/analyzer/taint-alloc-2.c
new file mode 100644
index 0000000..72dbca5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/analyzer/taint-alloc-2.c
@@ -0,0 +1,27 @@
+// TODO: remove need for this option:
+/* { dg-additional-options "-fanalyzer-checker=taint" } */
+
+#include "analyzer-decls.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct foo
+{
+  int num;
+};
+
+/* malloc with tainted size from a field.  */
+
+void *test_1 (FILE *f)
+{
+  struct foo tmp;
+  fread(&tmp, sizeof(tmp), 1, f); /* { dg-message "\\(\[0-9\]+\\) 'tmp' gets an unchecked value here" "event: tmp gets unchecked value" { xfail *-*-* } } */
+
+  __analyzer_dump_state ("taint", tmp.num); /* { dg-warning "state: 'tainted'" } */
+  __analyzer_dump_state ("taint", tmp.num * 16); /* { dg-warning "state: 'tainted'" } */
+
+  return malloc (tmp.num * 16); /* { dg-warning "use of attacker-controlled value 'tmp\\.num \\* 16' as allocation size without upper-bounds checking" "warning" } */
+  /* { dg-message "\\(\[0-9\]+\\) use of attacker-controlled value 'tmp\\.num \\* 16' as allocation size without upper-bounds checking" "final event with expr" { target *-*-* } .-1 } */
+  // TODO: show where tmp.num * 16 gets the bogus value
+}
diff --git a/gcc/testsuite/gcc.dg/analyzer/taint-divisor-1.c b/gcc/testsuite/gcc.dg/analyzer/taint-divisor-1.c
new file mode 100644
index 0000000..5a5a0b9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/analyzer/taint-divisor-1.c
@@ -0,0 +1,26 @@
+// TODO: remove need for this option:
+/* { dg-additional-options "-fanalyzer-checker=taint" } */
+
+#include "analyzer-decls.h"
+#include <stdio.h>
+
+struct st1
+{
+  int a;
+  int b;
+};
+
+
+int test_1 (FILE *f)
+{
+  struct st1 s;
+  fread (&s, sizeof (s), 1, f);
+  return s.a / s.b;  /* { dg-warning "use of attacker-controlled value 's\\.b' as divisor without checking for zero" } */
+}
+
+int test_2 (FILE *f)
+{
+  struct st1 s;
+  fread (&s, sizeof (s), 1, f);
+  return s.a % s.b;  /* { dg-warning "use of attacker-controlled value 's\\.b' as divisor without checking for zero" } */
+}
diff --git a/gcc/testsuite/gcc.dg/analyzer/taint-1.c b/gcc/testsuite/gcc.dg/analyzer/taint-read-index-1.c
index cd46dd5..71c0816 100644
--- a/gcc/testsuite/gcc.dg/analyzer/taint-1.c
+++ b/gcc/testsuite/gcc.dg/analyzer/taint-read-index-1.c
@@ -1,3 +1,4 @@
+// TODO: remove need for this option:
 /* { dg-additional-options "-fanalyzer-checker=taint" } */
 
 #include <stdio.h>
@@ -18,10 +19,10 @@ char test_1(FILE *f)
                                              /* { dg-message "\\(\[0-9\]+\\) following 'true' branch\\.\\.\\." "event: following true branch" { target *-*-* } .-1 } */
     /* BUG: the following array lookup trusts that the input data's index is
        in the range 0 <= i < 256; otherwise it's accessing the stack */
-    return tmp.buf[tmp.i]; // { dg-warning "use of tainted value 'tmp.i' in array lookup without bounds checking" "warning" } */
+    return tmp.buf[tmp.i]; // { dg-warning "use of attacker-controlled value 'tmp.i' in array lookup without bounds checking" "warning" } */
     /* { dg-message "23: \\(\[0-9\]+\\) \\.\\.\\.to here" "event: to here" { target *-*-* } .-1 } */
     /* { dg-message "23: \\(\[0-9\]+\\) 'tmp.i' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.i has an unchecked value" { xfail *-*-* } .-2 } */
-    /* { dg-message "\\(\[0-9\]+\\) use of tainted value 'tmp.i' in array lookup without bounds checking" "final event" { target *-*-* } .-3 } */
+    /* { dg-message "\\(\[0-9\]+\\) use of attacker-controlled value 'tmp.i' in array lookup without bounds checking" "final event" { target *-*-* } .-3 } */
     
     // TOOD: better messages for state changes
   }
@@ -53,8 +54,8 @@ char test_4(FILE *f)
 
   if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
     if (tmp.i >= 0) { /* { dg-message "'tmp.i' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.i has an unchecked value" { xfail *-*-* } } */
-      /* { dg-message "'tmp.i' has its lower bound checked here" "event: lower bound checked" { target *-*-* } .-1 } */
-      return tmp.buf[tmp.i]; /* { dg-warning "use of tainted value 'tmp.i' in array lookup without upper-bounds checking" "warning" } */
+      /* { dg-message "'tmp.i' has its lower bound checked here" "event: lower bound checked" { xfail *-*-* } .-1 } */
+      return tmp.buf[tmp.i]; /* { dg-warning "use of attacker-controlled value 'tmp.i' in array lookup without upper-bounds checking" "warning" } */
     }
   }
   return 0;
@@ -66,8 +67,8 @@ char test_5(FILE *f)
 
   if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
     if (tmp.i < 256) { /* { dg-message "'tmp.i' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.i has an unchecked value" { xfail *-*-* } } */
-      /* { dg-message "'tmp.i' has its upper bound checked here" "event: upper bound checked" { target *-*-* } .-1 } */
-      return tmp.buf[tmp.i]; /* { dg-warning "use of tainted value 'tmp.i' in array lookup without lower-bounds checking" "warning" } */
+      /* { dg-message "'tmp.i' has its upper bound checked here" "event: upper bound checked" { xfail *-*-* } .-1 } */
+      return tmp.buf[tmp.i]; /* { dg-warning "use of attacker-controlled value 'tmp.i' in array lookup without checking for negative" "warning" } */
     }
   }
   return 0;
@@ -85,7 +86,7 @@ char test_6(FILE *f)
   struct bar tmp;
 
   if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
-    return tmp.buf[tmp.i]; /* { dg-warning "use of tainted value 'tmp.i' in array lookup without upper-bounds checking" } */
+    return tmp.buf[tmp.i]; /* { dg-warning "use of attacker-controlled value 'tmp.i' in array lookup without upper-bounds checking" } */
   }
   return 0;
 }
@@ -96,7 +97,7 @@ char test_7(FILE *f)
 
   if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
     if (tmp.i >= 0) {
-      return tmp.buf[tmp.i]; /* { dg-warning "use of tainted value 'tmp.i' in array lookup without upper-bounds checking" } */
+      return tmp.buf[tmp.i]; /* { dg-warning "use of attacker-controlled value 'tmp.i' in array lookup without upper-bounds checking" } */
     }
   }
   return 0;
@@ -122,7 +123,7 @@ char test_9(FILE *f)
   if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
     if (tmp.i == 42) {
       /* not a bug: tmp.i compared against a specific value: */
-      return tmp.buf[tmp.i]; /* { dg-bogus "tainted" "" { xfail *-*-* } } */
+      return tmp.buf[tmp.i]; /* { dg-bogus "attacker-controlled" "" { xfail *-*-* } } */
       // TODO: xfail
     }
   }
diff --git a/gcc/testsuite/gcc.dg/analyzer/taint-read-offset-1.c b/gcc/testsuite/gcc.dg/analyzer/taint-read-offset-1.c
new file mode 100644
index 0000000..6db59bc
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/analyzer/taint-read-offset-1.c
@@ -0,0 +1,128 @@
+// TODO: remove need for this option:
+/* { dg-additional-options "-fanalyzer-checker=taint" } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct foo
+{
+  ssize_t offset;
+};
+
+char *p;
+
+char test_1(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) { /* { dg-message "\\(\[0-9\]+\\) 'tmp' gets an unchecked value here" "event: tmp gets unchecked value" { xfail *-*-* } } */
+                                             /* { dg-message "\\(\[0-9\]+\\) following 'true' branch\\.\\.\\." "event: following true branch" { target *-*-* } .-1 } */
+    return *(p + tmp.offset); // { dg-warning "use of attacker-controlled value 'tmp.offset' as offset without bounds checking" "warning" } */
+    /* { dg-message "\\(\[0-9\]+\\) \\.\\.\\.to here" "event: to here" { target *-*-* } .-1 } */
+    /* { dg-message "\\(\[0-9\]+\\) 'tmp.offset' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.offset has an unchecked value" { xfail *-*-* } .-2 } */
+    /* { dg-message "\\(\[0-9\]+\\) use of attacker-controlled value 'tmp.offset' as offset without bounds checking" "final event" { target *-*-* } .-3 } */
+    
+    // TOOD: better messages for state changes
+  }
+  return 0;
+}
+
+char test_2(struct foo *f)
+{
+  /* not a bug: the data is not known to be tainted: */
+  return *(p + f->offset);
+}
+
+char test_3(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset >= 0 && tmp.offset < 256) {
+      /* not a bug: the access is guarded by upper and lower bounds: */
+      return *(p + tmp.offset);
+    }
+  }
+  return 0;
+}
+
+char test_4(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset >= 0) { /* { dg-message "'tmp.offset' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.offset has an unchecked value" { xfail *-*-* } } */
+      /* { dg-message "'tmp.offset' has its lower bound checked here" "event: lower bound checked" { xfail *-*-* } .-1 } */
+      return *(p + tmp.offset); /* { dg-warning "use of attacker-controlled value 'tmp.offset' as offset without upper-bounds checking" "warning" } */
+    }
+  }
+  return 0;
+}
+
+char test_5(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset < 256) { /* { dg-message "'tmp.offset' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.offset has an unchecked value" { xfail *-*-* } } */
+      /* { dg-message "'tmp.offset' has its upper bound checked here" "event: upper bound checked" { xfail *-*-* } .-1 } */
+      return *(p + tmp.offset); /* { dg-warning "use of attacker-controlled value 'tmp.offset' as offset without lower-bounds checking" "warning" } */
+    }
+  }
+  return 0;
+}
+
+/* unsigned types have a natural lower bound of 0 */
+struct bar
+{
+  size_t offset;
+};
+
+char test_6(FILE *f)
+{
+  struct bar tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    return *(p + tmp.offset); /* { dg-warning "use of attacker-controlled value 'tmp.offset' as offset without upper-bounds checking" } */
+  }
+  return 0;
+}
+
+char test_7(FILE *f)
+{
+  struct bar tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset >= 0) {
+      return *(p + tmp.offset); /* { dg-warning "use of attacker-controlled value 'tmp.offset' as offset without upper-bounds checking" } */
+    }
+  }
+  return 0;
+}
+
+char test_8(FILE *f)
+{
+  struct bar tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset < 256) {
+      /* not a bug: has an upper bound, and an implicit lower bound: */
+      return *(p + tmp.offset);
+    }
+  }
+  return 0;
+}
+
+char test_9(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset == 42) {
+      /* not a bug: tmp.offset compared against a specific value: */
+      return *(p + tmp.offset); /* { dg-bogus "attacker-controlled" "" { xfail *-*-* } } */
+    }
+  }
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/analyzer/taint-size-1.c b/gcc/testsuite/gcc.dg/analyzer/taint-size-1.c
new file mode 100644
index 0000000..64c9c26
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/analyzer/taint-size-1.c
@@ -0,0 +1,32 @@
+// TODO: remove need for this option:
+/* { dg-additional-options "-fanalyzer-checker=taint" } */
+
+#include "analyzer-decls.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct foo
+{
+  size_t sz;
+};
+
+char buf[100];
+
+/* memset with tainted size.  */
+
+void test_1 (FILE *f)
+{
+  struct foo tmp;
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) { /* { dg-message "\\(\[0-9\]+\\) 'tmp' gets an unchecked value here" "event: tmp gets unchecked value" { xfail *-*-* } } */
+                                             /* { dg-message "\\(\[0-9\]+\\) following 'true' branch\\.\\.\\." "event: following true branch" { target *-*-* } .-1 } */
+    __analyzer_dump_state ("taint", tmp.sz); /* { dg-warning "state: 'tainted'" } */
+    /* { dg-message "\\(\[0-9\]+\\) \\.\\.\\.to here" "event: to here" { target *-*-* } .-1 } */
+
+    memset (buf, 0, tmp.sz); /* { dg-warning "use of attacker-controlled value 'tmp\\.sz' as size without upper-bounds checking" "warning" } */
+    /* { dg-message "23: \\(\[0-9\]+\\) 'tmp.i' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.i has an unchecked value" { xfail *-*-* } .-1 } */
+    /* { dg-message "\\(\[0-9\]+\\) use of attacker-controlled value 'tmp\\.sz' as size without upper-bounds checking" "final event" { target *-*-* } .-2 } */
+    
+    // TOOD: better messages for state changes
+  }
+}
diff --git a/gcc/testsuite/gcc.dg/analyzer/taint-write-index-1.c b/gcc/testsuite/gcc.dg/analyzer/taint-write-index-1.c
new file mode 100644
index 0000000..cc7ab1c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/analyzer/taint-write-index-1.c
@@ -0,0 +1,132 @@
+// TODO: remove need for this option:
+/* { dg-additional-options "-fanalyzer-checker=taint" } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct foo
+{
+  signed int i;
+  char buf[256];
+};
+
+struct foo g;
+char test_1(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) { /* { dg-message "\\(\[0-9\]+\\) 'tmp' gets an unchecked value here" "event: tmp gets unchecked value" { xfail *-*-* } } */
+                                             /* { dg-message "\\(\[0-9\]+\\) following 'true' branch\\.\\.\\." "event: following true branch" { target *-*-* } .-1 } */
+    /* BUG: the following array lookup trusts that the input data's index is
+       in the range 0 <= i < 256; otherwise it's accessing the stack */
+    g.buf[tmp.i] = 42; // { dg-warning "use of attacker-controlled value 'tmp.i' in array lookup without bounds checking" "warning" } */
+    /* { dg-message "\\(\[0-9\]+\\) \\.\\.\\.to here" "event: to here" { target *-*-* } .-1 } */
+    /* { dg-message "\\(\[0-9\]+\\) 'tmp.i' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.i has an unchecked value" { xfail *-*-* } .-2 } */
+    /* { dg-message "\\(\[0-9\]+\\) use of attacker-controlled value 'tmp.i' in array lookup without bounds checking" "final event" { target *-*-* } .-3 } */
+    
+    // TOOD: better messages for state changes
+  }
+  return 0;
+}
+
+char test_2(struct foo *f, int i)
+{
+  /* not a bug: the data is not known to be tainted: */
+  return f->buf[f->i];
+}
+
+char test_3(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.i >= 0 && tmp.i < 256) {
+      /* not a bug: the access is guarded by upper and lower bounds: */
+      g.buf[tmp.i] = 42;
+    }
+  }
+  return 0;
+}
+
+char test_4(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.i >= 0) { /* { dg-message "'tmp.i' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.i has an unchecked value" { xfail *-*-* } } */
+      /* { dg-message "'tmp.i' has its lower bound checked here" "event: lower bound checked" { xfail *-*-* } .-1 } */
+      g.buf[tmp.i] = 42; /* { dg-warning "use of attacker-controlled value 'tmp.i' in array lookup without upper-bounds checking" "warning" } */
+    }
+  }
+  return 0;
+}
+
+char test_5(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.i < 256) { /* { dg-message "'tmp.i' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.i has an unchecked value" { xfail *-*-* } } */
+      /* { dg-message "'tmp.i' has its upper bound checked here" "event: upper bound checked" { xfail *-*-* } .-1 } */
+      g.buf[tmp.i] = 42; /* { dg-warning "use of attacker-controlled value 'tmp.i' in array lookup without checking for negative" "warning" } */
+    }
+  }
+  return 0;
+}
+
+/* unsigned types have a natural lower bound of 0 */
+struct bar
+{
+  unsigned int i;
+  char buf[256];
+};
+
+char test_6(FILE *f)
+{
+  struct bar tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    g.buf[tmp.i] = 42; /* { dg-warning "use of attacker-controlled value 'tmp.i' in array lookup without upper-bounds checking" } */
+  }
+  return 0;
+}
+
+char test_7(FILE *f)
+{
+  struct bar tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.i >= 0) {
+      g.buf[tmp.i] = 42; /* { dg-warning "use of attacker-controlled value 'tmp.i' in array lookup without upper-bounds checking" } */
+    }
+  }
+  return 0;
+}
+
+char test_8(FILE *f)
+{
+  struct bar tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.i < 256) {
+      /* not a bug: has an upper bound, and an implicit lower bound: */
+      g.buf[tmp.i] = 42;
+    }
+  }
+  return 0;
+}
+
+char test_9(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.i == 42) {
+      /* not a bug: tmp.i compared against a specific value: */
+      g.buf[tmp.i] = 42; /* { dg-bogus "attacker-controlled" "" { xfail *-*-* } } */
+      // TODO: xfail
+    }
+  }
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/analyzer/taint-write-offset-1.c b/gcc/testsuite/gcc.dg/analyzer/taint-write-offset-1.c
new file mode 100644
index 0000000..d0df622
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/analyzer/taint-write-offset-1.c
@@ -0,0 +1,132 @@
+// TODO: remove need for this option:
+/* { dg-additional-options "-fanalyzer-checker=taint" } */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct foo
+{
+  ssize_t offset;
+};
+
+char *p;
+
+char test_1(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) { /* { dg-message "\\(\[0-9\]+\\) 'tmp' gets an unchecked value here" "event: tmp gets unchecked value" { xfail *-*-* } } */
+                                             /* { dg-message "\\(\[0-9\]+\\) following 'true' branch\\.\\.\\." "event: following true branch" { target *-*-* } .-1 } */
+    /* BUG: the following array lookup trusts that the input data's index is
+       in the range 0 <= i < 256; otherwise it's accessing the stack */
+    *(p + tmp.offset) = 42; // { dg-warning "use of attacker-controlled value 'tmp.offset' as offset without bounds checking" "warning" } */
+    /* { dg-message "\\(\[0-9\]+\\) \\.\\.\\.to here" "event: to here" { target *-*-* } .-1 } */
+    /* { dg-message "\\(\[0-9\]+\\) 'tmp.offset' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.offset has an unchecked value" { xfail *-*-* } .-2 } */
+    /* { dg-message "\\(\[0-9\]+\\) use of attacker-controlled value 'tmp.offset' as offset without bounds checking" "final event" { target *-*-* } .-3 } */
+    
+    // TOOD: better messages for state changes
+  }
+  return 0;
+}
+
+char test_2(struct foo *f)
+{
+  /* not a bug: the data is not known to be tainted: */
+  *(p + f->offset) = 42;
+}
+
+char test_3(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset >= 0 && tmp.offset < 256) {
+      /* not a bug: the access is guarded by upper and lower bounds: */
+      *(p + tmp.offset) = 42;
+    }
+  }
+  return 0;
+}
+
+char test_4(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset >= 0) { /* { dg-message "'tmp.offset' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.offset has an unchecked value" { xfail *-*-* } } */
+      /* { dg-message "'tmp.offset' has its lower bound checked here" "event: lower bound checked" { xfail *-*-* } .-1 } */
+      *(p + tmp.offset) = 42; /* { dg-warning "use of attacker-controlled value 'tmp.offset' as offset without upper-bounds checking" "warning" } */
+    }
+  }
+  return 0;
+}
+
+char test_5(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset < 256) { /* { dg-message "'tmp.offset' has an unchecked value here \\(from 'tmp'\\)" "event: tmp.offset has an unchecked value" { xfail *-*-* } } */
+      /* { dg-message "'tmp.offset' has its upper bound checked here" "event: upper bound checked" { xfail *-*-* } .-1 } */
+      *(p + tmp.offset) = 42; /* { dg-warning "use of attacker-controlled value 'tmp.offset' as offset without lower-bounds checking" "warning" } */
+    }
+  }
+  return 0;
+}
+
+/* unsigned types have a natural lower bound of 0 */
+struct bar
+{
+  unsigned int offset;
+  char buf[256];
+};
+
+char test_6(FILE *f)
+{
+  struct bar tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    *(p + tmp.offset) = 42; /* { dg-warning "use of attacker-controlled value 'tmp.offset' as offset without upper-bounds checking" } */
+  }
+  return 0;
+}
+
+char test_7(FILE *f)
+{
+  struct bar tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset >= 0) {
+      *(p + tmp.offset) = 42; /* { dg-warning "use of attacker-controlled value 'tmp.offset' as offset without upper-bounds checking" } */
+    }
+  }
+  return 0;
+}
+
+char test_8(FILE *f)
+{
+  struct bar tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset < 256) {
+      /* not a bug: has an upper bound, and an implicit lower bound: */
+      *(p + tmp.offset) = 42;
+    }
+  }
+  return 0;
+}
+
+char test_9(FILE *f)
+{
+  struct foo tmp;
+
+  if (1 == fread(&tmp, sizeof(tmp), 1, f)) {
+    if (tmp.offset == 42) {
+      /* not a bug: tmp.offset compared against a specific value: */
+      *(p + tmp.offset) = 42; /* { dg-bogus "tainted" "" { xfail *-*-* } } */
+      // TODO: xfail
+    }
+  }
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/attr-returns-nonnull.c b/gcc/testsuite/gcc.dg/attr-returns-nonnull.c
index 22ee30a..e4e20b8 100644
--- a/gcc/testsuite/gcc.dg/attr-returns-nonnull.c
+++ b/gcc/testsuite/gcc.dg/attr-returns-nonnull.c
@@ -1,7 +1,7 @@
 /* Verify that attribute returns_nonnull on global and local function
    declarations is merged.
    { dg-do compile }
-   { dg-options "-Wall -fdump-tree-optimized" } */
+   { dg-options "-Wall -fdump-tree-optimized -fdelete-null-pointer-checks" } */
 
 void foo (void);
 
diff --git a/gcc/testsuite/gcc.dg/bad-pragma-locations.c b/gcc/testsuite/gcc.dg/bad-pragma-locations.c
new file mode 100644
index 0000000..8068839
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/bad-pragma-locations.c
@@ -0,0 +1,86 @@
+/* Verify that we use precise locations when emitting diagnostics
+   about pragmas.  */
+
+/* { dg-do assemble } */
+/* { dg-options "-fdiagnostics-show-caret" } */
+
+/* pack ****************************************************************************/
+
+#pragma pack
+/* { dg-warning "missing '\\(' after '#pragma pack' - ignored" "" { target *-*-* } .-1 }
+   { dg-begin-multiline-output "" }
+ #pragma pack
+         ^~~~
+   { dg-end-multiline-output "" }  */
+
+#pragma pack (
+/* { dg-warning "malformed '#pragma pack' - ignored" "" { target *-*-* } .-1 }
+   { dg-begin-multiline-output "" }
+ #pragma pack (
+         ^~~~
+   { dg-end-multiline-output "" }  */
+
+#pragma pack (32
+/* { dg-warning "malformed '#pragma pack' - ignored" "" { target *-*-* } .-1 }
+   { dg-begin-multiline-output "" }
+ #pragma pack (32
+         ^~~~
+   { dg-end-multiline-output "" }  */
+
+#pragma pack (3.14159
+/* { dg-warning "invalid constant in '#pragma pack' - ignored" "" { target *-*-* } .-1 }
+   { dg-begin-multiline-output "" }
+ #pragma pack (3.14159
+               ^~~~~~~
+   { dg-end-multiline-output "" }  */
+
+#pragma pack (push, 3.14159
+/* { dg-warning "invalid constant in '#pragma pack' - ignored" "" { target *-*-* } .-1 }
+   { dg-begin-multiline-output "" }
+ #pragma pack (push, 3.14159
+                     ^~~~~~~
+   { dg-end-multiline-output "" }  */
+
+#pragma pack (toothbrush
+/* { dg-warning "unknown action 'toothbrush' for '#pragma pack' - ignored" "" { target *-*-* } .-1 }
+   { dg-begin-multiline-output "" }
+ #pragma pack (toothbrush
+               ^~~~~~~~~~
+   { dg-end-multiline-output "" }  */
+
+#pragma pack() pyjamas
+/* { dg-warning "junk at end of '#pragma pack'" "" { target *-*-* } .-1 }
+   { dg-begin-multiline-output "" }
+ #pragma pack() pyjamas
+                ^~~~~~~
+   { dg-end-multiline-output "" }  */
+
+/* target ****************************************************************************/
+
+#pragma GCC target 42
+/* { dg-warning "#pragma GCC option' is not a string" "" { target *-*-* } .-1 }
+   { dg-begin-multiline-output "" }
+ #pragma GCC target 42
+                    ^~
+   { dg-end-multiline-output "" }  */
+
+#pragma GCC target ( 1776
+/* { dg-warning "#pragma GCC option' is not a string" "" { target *-*-* } .-1 }
+   { dg-begin-multiline-output "" }
+ #pragma GCC target ( 1776
+                      ^~~~
+   { dg-end-multiline-output "" }  */
+
+/* message ****************************************************************************/
+
+#pragma message "foo" int
+/* { dg-warning "junk at end of '#pragma message'" "" { target *-*-* } .-1 }
+   { dg-message "'#pragma message: foo'" "" { target *-*-* } .-2 }
+   { dg-begin-multiline-output "" }
+ #pragma message "foo" int
+                       ^~~
+   { dg-end-multiline-output "" }
+   { dg-begin-multiline-output "" }
+ #pragma message "foo" int
+         ^~~~~~~
+   { dg-end-multiline-output "" }  */
diff --git a/gcc/testsuite/gcc.dg/cpp/warn-normalized-4-bytes.c b/gcc/testsuite/gcc.dg/cpp/warn-normalized-4-bytes.c
new file mode 100644
index 0000000..e6e1f3e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/cpp/warn-normalized-4-bytes.c
@@ -0,0 +1,21 @@
+// { dg-do preprocess }
+// { dg-options "-std=gnu99 -Werror=normalized=nfc -fdiagnostics-show-caret -fdiagnostics-escape-format=bytes" }
+/* { dg-message "some warnings being treated as errors" "" {target "*-*-*"} 0 } */
+
+/* གྷ = U+0F43 TIBETAN LETTER GHA, which has decomposition "0F42 0FB7" i.e.
+   U+0F42 TIBETAN LETTER GA: ག
+   U+0FB7 TIBETAN SUBJOINED LETTER HA: ྷ
+
+   The UTF-8 encoding of U+0F43 TIBETAN LETTER GHA is: E0 BD 83.  */
+
+foo before_\u0F43_after bar // { dg-error "`before_.U00000f43_after' is not in NFC .-Werror=normalized=." }
+/* { dg-begin-multiline-output "" }
+ foo before_\u0F43_after bar
+     ^~~~~~~~~~~~~~~~~~~
+   { dg-end-multiline-output "" } */
+
+foo before_གྷ_after bar // { dg-error "`before_.U00000f43_after' is not in NFC .-Werror=normalized=." }
+/* { dg-begin-multiline-output "" }
+ foo before_<e0><bd><83>_after bar
+     ^~~~~~~~~~~~~~~~~~~~~~~~~
+   { dg-end-multiline-output "" } */
diff --git a/gcc/testsuite/gcc.dg/cpp/warn-normalized-4-unicode.c b/gcc/testsuite/gcc.dg/cpp/warn-normalized-4-unicode.c
new file mode 100644
index 0000000..f9f2f7a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/cpp/warn-normalized-4-unicode.c
@@ -0,0 +1,19 @@
+// { dg-do preprocess }
+// { dg-options "-std=gnu99 -Werror=normalized=nfc -fdiagnostics-show-caret -fdiagnostics-escape-format=unicode" }
+/* { dg-message "some warnings being treated as errors" "" {target "*-*-*"} 0 } */
+
+/* གྷ = U+0F43 TIBETAN LETTER GHA, which has decomposition "0F42 0FB7" i.e.
+   U+0F42 TIBETAN LETTER GA: ག
+   U+0FB7 TIBETAN SUBJOINED LETTER HA: ྷ  */
+
+foo before_\u0F43_after bar  // { dg-error "`before_.U00000f43_after' is not in NFC .-Werror=normalized=." }
+/* { dg-begin-multiline-output "" }
+ foo before_\u0F43_after bar
+     ^~~~~~~~~~~~~~~~~~~
+   { dg-end-multiline-output "" } */
+
+foo before_གྷ_after bar // { dg-error "`before_.U00000f43_after' is not in NFC .-Werror=normalized=." }
+/* { dg-begin-multiline-output "" }
+ foo before_<U+0F43>_after bar
+     ^~~~~~~~~~~~~~~~~~~~~
+   { dg-end-multiline-output "" } */
diff --git a/gcc/testsuite/gcc.dg/debug/btf/btf-datasec-1.c b/gcc/testsuite/gcc.dg/debug/btf/btf-datasec-1.c
index f809d93..dbb236b 100644
--- a/gcc/testsuite/gcc.dg/debug/btf/btf-datasec-1.c
+++ b/gcc/testsuite/gcc.dg/debug/btf/btf-datasec-1.c
@@ -12,6 +12,7 @@
 /* { dg-do compile )  */
 /* { dg-options "-O0 -gbtf -dA" } */
 /* { dg-options "-O0 -gbtf -dA -msdata=none" { target { { powerpc*-*-* } && ilp32 } } } */
+/* { dg-options "-O0 -gbtf -dA -G0" { target { nios2-*-* } } } */
 
 /* Check for two DATASEC entries with vlen 3, and one with vlen 1.  */
 /* { dg-final { scan-assembler-times "0xf000003\[\t \]+\[^\n\]*btt_info" 2 } } */
diff --git a/gcc/testsuite/gcc.dg/encoding-issues-bytes.c b/gcc/testsuite/gcc.dg/encoding-issues-bytes.c
new file mode 100644
index 0000000..d3dc59f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/encoding-issues-bytes.c
diff --git a/gcc/testsuite/gcc.dg/encoding-issues-unicode.c b/gcc/testsuite/gcc.dg/encoding-issues-unicode.c
new file mode 100644
index 0000000..39f3713
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/encoding-issues-unicode.c
diff --git a/gcc/testsuite/gcc.dg/fold-stringops-1.c b/gcc/testsuite/gcc.dg/fold-stringops-1.c
new file mode 100644
index 0000000..e26632a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/fold-stringops-1.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-lower-details" } */
+
+char dst[2048];
+
+char *
+copy1 (const char *src, int cond)
+{
+  __builtin___stpncpy_chk (dst, src, 42, __builtin_object_size (dst, 0));
+
+  return dst;
+}
+
+char *
+copy2 (void)
+{
+  __builtin___stpcpy_chk (dst, "Hello world", __builtin_object_size (dst, 0));
+
+  return dst;
+}
+/* { dg-final { scan-tree-dump "simplified __builtin___stpncpy_chk to __builtin_strncpy" "lower" } } */
+/* { dg-final { scan-tree-dump "simplified __builtin___stpcpy_chk to __builtin_strcpy" "lower" } } */
+
diff --git a/gcc/testsuite/gcc.dg/fold-stringops-2.c b/gcc/testsuite/gcc.dg/fold-stringops-2.c
new file mode 100644
index 0000000..ac7d29e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/fold-stringops-2.c
@@ -0,0 +1,63 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+typedef __SIZE_TYPE__ size_t;
+
+#define bos(__d) __builtin_object_size ((__d), 0)
+
+char *
+safe1 (const char *src, int cond, size_t len)
+{
+  char *dst;
+
+  if (cond)
+    dst = __builtin_malloc (1024);
+  else
+    dst = __builtin_malloc (2048);
+
+  len = len > 2048 ? 2048 : len;
+
+  return __builtin___memcpy_chk (dst, src, len, bos (dst));
+}
+
+char *
+safe2 (const char *src, int cond, unsigned char len)
+{
+  char *dst;
+
+  if (cond)
+    dst = __builtin_malloc (1024);
+  else
+    dst = __builtin_malloc (2048);
+
+  return __builtin___strncpy_chk (dst, src, len, bos (dst));
+}
+
+int
+safe3 (const char *src, int cond, unsigned char len)
+{
+  char *dst;
+
+  if (cond)
+    dst = __builtin_malloc (1024);
+  else
+    dst = __builtin_malloc (2048);
+
+  return __builtin___snprintf_chk (dst, len, 0, bos (dst), "%s", src);
+}
+
+char dst[1024];
+
+void
+safe4 (size_t len)
+{
+  len = len > sizeof (dst) - 1 ? sizeof (dst) - 1 : len;
+  len = len < sizeof (dst) / 2 ? sizeof (dst) / 2 : len;
+
+  __builtin_strncat (dst, "hello", len);
+}
+
+/* { dg-final { scan-assembler-not "__memcpy_chk" } } */
+/* { dg-final { scan-assembler-not "__strncpy_chk" } } */
+/* { dg-final { scan-assembler-not "__snprintf_chk" } } */
+/* { dg-final { scan-assembler-not "strncat" } } */
diff --git a/gcc/testsuite/gcc.dg/fold-stringops-3.c b/gcc/testsuite/gcc.dg/fold-stringops-3.c
new file mode 100644
index 0000000..ae2efbf
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/fold-stringops-3.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+typedef __SIZE_TYPE__ size_t;
+
+char dst[1024];
+
+void
+safe1 (size_t len)
+{
+  len = len > sizeof (dst) ? sizeof (dst) : len;
+  len = len < sizeof (dst) / 2 ? sizeof (dst) / 2 : len;
+
+  __builtin_snprintf (dst, len, "hello");
+  __builtin_snprintf (dst + 5, len, "%s", " world");
+}
+
+/* { dg-final { scan-assembler-not "snprintf" } } */
diff --git a/gcc/testsuite/gcc.dg/gcov-info-to-gcda.c b/gcc/testsuite/gcc.dg/gcov-info-to-gcda.c
index a42a768..4583360 100644
--- a/gcc/testsuite/gcc.dg/gcov-info-to-gcda.c
+++ b/gcc/testsuite/gcc.dg/gcov-info-to-gcda.c
@@ -1,5 +1,4 @@
-/* { dg-do run } */
-/* { dg-skip-if "profile-info-section" { powerpc-ibm-aix* } } */
+/* { dg-do run { target *-*-linux* *-*-gnu* } } */
 /* { dg-options "-fprofile-arcs -fprofile-info-section" } */
 
 #define assert(expr)                                            \
diff --git a/gcc/testsuite/gcc.dg/graphite/scop-dsyr2k-2.c b/gcc/testsuite/gcc.dg/graphite/scop-dsyr2k-2.c
index 06aa19a..42e23fc 100644
--- a/gcc/testsuite/gcc.dg/graphite/scop-dsyr2k-2.c
+++ b/gcc/testsuite/gcc.dg/graphite/scop-dsyr2k-2.c
@@ -1,4 +1,5 @@
 /* { dg-require-effective-target size32plus } */
+/* { dg-additional-options "-fno-thread-jumps" } */
 #define NMAX 3000
 
 static double a[NMAX][NMAX], b[NMAX][NMAX], c[NMAX][NMAX];
diff --git a/gcc/testsuite/gcc.dg/graphite/scop-dsyr2k.c b/gcc/testsuite/gcc.dg/graphite/scop-dsyr2k.c
index 41c91b9..feb9935 100644
--- a/gcc/testsuite/gcc.dg/graphite/scop-dsyr2k.c
+++ b/gcc/testsuite/gcc.dg/graphite/scop-dsyr2k.c
@@ -1,4 +1,5 @@
 /* { dg-require-effective-target size32plus } */
+/* { dg-additional-options "-fno-thread-jumps" } */
 #define NMAX 3000
 
 static double a[NMAX][NMAX], b[NMAX][NMAX], c[NMAX][NMAX];
diff --git a/gcc/testsuite/gcc.dg/graphite/scop-dsyrk-2.c b/gcc/testsuite/gcc.dg/graphite/scop-dsyrk-2.c
index 5622dce..935ade3 100644
--- a/gcc/testsuite/gcc.dg/graphite/scop-dsyrk-2.c
+++ b/gcc/testsuite/gcc.dg/graphite/scop-dsyrk-2.c
@@ -1,4 +1,5 @@
 /* { dg-require-effective-target size32plus } */
+/* { dg-additional-options "-fno-thread-jumps" } */
 #define NMAX 3000
 #define MEASURE_TIME 1
 
diff --git a/gcc/testsuite/gcc.dg/graphite/scop-dsyrk.c b/gcc/testsuite/gcc.dg/graphite/scop-dsyrk.c
index e01a517..5c65e40 100644
--- a/gcc/testsuite/gcc.dg/graphite/scop-dsyrk.c
+++ b/gcc/testsuite/gcc.dg/graphite/scop-dsyrk.c
@@ -1,4 +1,5 @@
 /* { dg-require-effective-target size32plus } */
+/* { dg-additional-options "-fno-thread-jumps" } */
 #define NMAX 3000
 #define MEASURE_TIME 1
 
diff --git a/gcc/testsuite/gcc.dg/guality/ipa-sra-1.c b/gcc/testsuite/gcc.dg/guality/ipa-sra-1.c
new file mode 100644
index 0000000..5434b3d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/guality/ipa-sra-1.c
@@ -0,0 +1,45 @@
+/* { dg-do run } */
+/* { dg-options "-g -fno-ipa-icf" } */
+
+
+void __attribute__((noipa))
+use (int x)
+{
+  asm volatile ("" : : "r" (x) : "memory");
+}
+
+static int __attribute__((noinline))
+bar (int i, int k)
+{
+  asm ("" : "+r" (i));
+  use (i);		/* { dg-final { gdb-test . "k" "3" { xfail *-*-* } } } */
+  return 6;
+}
+
+volatile int v;
+
+static int __attribute__((noinline))
+foo (int i, int k)
+{
+  int r;
+  v = 9;
+  k = (k + 14)/k;
+  r = bar (i, k);		/* { dg-final { gdb-test . "k" "3" } } */
+  return r;
+}
+
+volatile int v;
+
+int __attribute__((noipa))
+get_val1 (void)  {return 20;}
+int __attribute__((noipa))
+get_val2 (void)  {return 7;}
+
+int
+main (void)
+{
+  int k = get_val2 ();
+  int r = foo (get_val1 (), k);
+  v = r + k;   /* k has to live accross the call or all is probably lost  */
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/ifcvt-4.c b/gcc/testsuite/gcc.dg/ifcvt-4.c
index ec142cf..0525102 100644
--- a/gcc/testsuite/gcc.dg/ifcvt-4.c
+++ b/gcc/testsuite/gcc.dg/ifcvt-4.c
@@ -2,7 +2,7 @@
 /* { dg-additional-options "-misel" { target { powerpc*-*-* } } } */
 /* { dg-additional-options "-march=z196" { target { s390x-*-* } } } */
 /* { dg-additional-options "-mtune-ctrl=^one_if_conv_insn" { target { i?86-*-* x86_64-*-* } } } */
-/* { dg-skip-if "Multiple set if-conversion not guaranteed on all subtargets" { "arm*-*-* avr-*-* hppa*64*-*-* s390-*-* visium-*-*" riscv*-*-* msp430-*-* } }  */
+/* { dg-skip-if "Multiple set if-conversion not guaranteed on all subtargets" { "arm*-*-* avr-*-* hppa*64*-*-* s390-*-* visium-*-*" riscv*-*-* msp430-*-* nios2-*-*} }  */
 /* { dg-skip-if "" { "s390x-*-*" } { "-m31" } }  */
 
 typedef int word __attribute__((mode(word)));
@@ -13,8 +13,7 @@ foo (word x, word y, word a)
   word i = x;
   word j = y;
   /* Try to make taking the branch likely.  */
-  __builtin_expect (x > y, 1);
-  if (x > y)
+  if (__builtin_expect (x > y, 0))
     {
       i = a;
       j = i;
diff --git a/gcc/testsuite/gcc.dg/init-rounding-math-1.c b/gcc/testsuite/gcc.dg/init-rounding-math-1.c
new file mode 100644
index 0000000..2bece1a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/init-rounding-math-1.c
@@ -0,0 +1,11 @@
+/* Test static initializer folding of implicit conversions to floating point
+   types, even with -frounding-math and related options.  Bug 103031.  */
+/* { dg-do compile } */
+/* { dg-options "-frounding-math -ftrapping-math -fsignaling-nans" } */
+
+float f1 = -1ULL;
+float f2 = __DBL_MAX__;
+float f3 = __DBL_MIN__;
+float f4 = 0.1;
+float f5 = __builtin_nans ("");
+double d1 = -1ULL;
diff --git a/gcc/testsuite/gcc.dg/ipa/ipa-sra-4.c b/gcc/testsuite/gcc.dg/ipa/ipa-sra-4.c
index fdbd5e5..c86ae83 100644
--- a/gcc/testsuite/gcc.dg/ipa/ipa-sra-4.c
+++ b/gcc/testsuite/gcc.dg/ipa/ipa-sra-4.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fipa-sra -fno-ipa-pure-const -fdump-ipa-sra" } */
+/* { dg-options "-O2 -fipa-sra -fno-ipa-pure-const -fdump-ipa-sra -fno-ipa-modref" } */
 
 static int
 __attribute__((noinline))
diff --git a/gcc/testsuite/gcc.dg/ipa/ipa-sra-ret-nonull.c b/gcc/testsuite/gcc.dg/ipa/ipa-sra-ret-nonull.c
new file mode 100644
index 0000000..18c13ef
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/ipa/ipa-sra-ret-nonull.c
@@ -0,0 +1,40 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-ipa-sra-details"  } */
+
+volatile void *gp;
+volatile void *gq;
+char buf[16];
+
+__attribute__((returns_nonnull, noinline))
+static char *
+foo (char *p, char *q)
+{
+  gq = q;
+  gp = p;
+  return q;
+}
+
+__attribute__((returns_nonnull, noinline))
+static char *
+bar (char *p, char *q)
+{
+  return foo (p, q) + 8;
+}
+
+__attribute__((noipa))
+static char *
+get_charp (void)
+{
+  return &buf[0];
+}
+
+int
+main ()
+{
+  char *r;
+  asm volatile ("" : : : "memory");
+  r = bar (get_charp (), get_charp ());
+  return 0;
+}
+
+/* { dg-final { scan-ipa-dump-times "Will SKIP return." 2 "sra" } } */
diff --git a/gcc/testsuite/gcc.dg/ipa/modref-3.c b/gcc/testsuite/gcc.dg/ipa/modref-3.c
new file mode 100644
index 0000000..9a20e01
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/ipa/modref-3.c
@@ -0,0 +1,20 @@
+/* { dg-options "-O2 -fdump-ipa-modref"  } */
+/* { dg-do link } */
+int *ptr;
+void linker_error ();
+int
+main ()
+{
+  int a;
+  __attribute__((noinline)) int test2 ()
+  {
+    ptr = 0;
+    return a;
+  }
+  a = 1;
+  test2 ();
+  if (a != 1)
+    linker_error ();
+  return 0;
+}
+/* { dg-final { scan-ipa-dump "Static chain flags: no_direct_clobber no_indirect_clobber no_direct_escape no_indirect_escape not_returned_directly no_indirect_read" "modref" } } */
diff --git a/gcc/testsuite/gcc.dg/ipa/pr103107.c b/gcc/testsuite/gcc.dg/ipa/pr103107.c
new file mode 100644
index 0000000..3ea3fc5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/ipa/pr103107.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-Og -g -fipa-sra -fno-tree-dce" } */
+
+typedef int __attribute__((__vector_size__ (8))) V;
+V v;
+
+static void
+bar (int i)
+{
+  V l = v + i;
+}
+
+void
+foo (void)
+{
+  bar (0);
+}
diff --git a/gcc/testsuite/gcc.dg/ipa/pr103132.c b/gcc/testsuite/gcc.dg/ipa/pr103132.c
new file mode 100644
index 0000000..bef5649
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/ipa/pr103132.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -g" } */
+
+int globus_i_GLOBUS_GRIDFTP_SERVER_debug_handle_1;
+int globus_l_gfs_ipc_unpack_data__sz;
+void globus_i_GLOBUS_GRIDFTP_SERVER_debug_printf(const char *, ...);
+static void globus_l_gfs_ipc_unpack_cred(int len) {
+  if (globus_i_GLOBUS_GRIDFTP_SERVER_debug_handle_1)
+    globus_i_GLOBUS_GRIDFTP_SERVER_debug_printf("", __func__);
+}
+static void globus_l_gfs_ipc_unpack_data(int len) {
+  for (; globus_l_gfs_ipc_unpack_data__sz;)
+    len--;
+  len -= 4;
+  len -= 4;
+  globus_l_gfs_ipc_unpack_cred(len);
+}
+void globus_l_gfs_ipc_reply_read_body_cb(int len)
+{ globus_l_gfs_ipc_unpack_data(len); }
diff --git a/gcc/testsuite/gcc.dg/loop-unswitch-2.c b/gcc/testsuite/gcc.dg/loop-unswitch-2.c
index 0931f6e..f8e99bd 100644
--- a/gcc/testsuite/gcc.dg/loop-unswitch-2.c
+++ b/gcc/testsuite/gcc.dg/loop-unswitch-2.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -funswitch-loops -fdump-tree-unswitch-details -fdisable-tree-thread2 -fdisable-tree-thread3" } */
+/* { dg-options "-O2 -funswitch-loops -fdump-tree-unswitch-details -fno-thread-jumps" } */
 
 void foo (float **a, float **b, float *c, int n, int m, int l)
 {
diff --git a/gcc/testsuite/gcc.dg/lto/modref-3_0.c b/gcc/testsuite/gcc.dg/lto/modref-3_0.c
index bd8f96f..0210d11 100644
--- a/gcc/testsuite/gcc.dg/lto/modref-3_0.c
+++ b/gcc/testsuite/gcc.dg/lto/modref-3_0.c
@@ -14,4 +14,4 @@ main()
     __builtin_abort ();
   return 0;
 }
-/* { dg-final { scan-wpa-ipa-dump "parm 1 flags: nodirectescape"  "modref"  } } */
+/* { dg-final { scan-wpa-ipa-dump "parm 1 flags: no_direct_clobber no_direct_escape"  "modref"  } } */
diff --git a/gcc/testsuite/gcc.dg/lto/modref-4_0.c b/gcc/testsuite/gcc.dg/lto/modref-4_0.c
index db90b4f..9437585 100644
--- a/gcc/testsuite/gcc.dg/lto/modref-4_0.c
+++ b/gcc/testsuite/gcc.dg/lto/modref-4_0.c
@@ -14,4 +14,4 @@ main()
     __builtin_abort ();
   return 0;
 }
-/* { dg-final { scan-wpa-ipa-dump "parm 1 flags: nodirectescape"  "modref"  } } */
+/* { dg-final { scan-wpa-ipa-dump "parm 1 flags: no_direct_clobber no_direct_escape"  "modref"  } } */
diff --git a/gcc/testsuite/gcc.dg/old-style-asm-1.c b/gcc/testsuite/gcc.dg/old-style-asm-1.c
index f9406ff0..c8fb668 100644
--- a/gcc/testsuite/gcc.dg/old-style-asm-1.c
+++ b/gcc/testsuite/gcc.dg/old-style-asm-1.c
@@ -1,9 +1,6 @@
 /* PR inline-asm/8832 */
 /* { dg-do compile } */
-/* { dg-options "-O2 -dP -fdisable-tree-ethread -fdisable-tree-thread1 -fdisable-tree-thread2 -fdisable-tree-thread3 -fdisable-tree-thread4" } */
-
-/* Note: Threader will duplicate BBs and replace one conditional branch by an
-   unconditional one.  */
+/* { dg-options "-O2 -dP -fno-thread-jumps" } */
 
 /* Verify that GCC doesn't optimize
    old style asm instructions.  */
diff --git a/gcc/testsuite/gcc.dg/pr100520.c b/gcc/testsuite/gcc.dg/pr100520.c
new file mode 100644
index 0000000..60f79c2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr100520.c
@@ -0,0 +1,5 @@
+/* PR gcov-profile/100520 */
+/* { dg-do compile } */
+/* { dg-options "-fcompare-debug -fprofile-generate" } */
+
+static int f() {}
diff --git a/gcc/testsuite/gcc.dg/pr102009.c b/gcc/testsuite/gcc.dg/pr102009.c
new file mode 100644
index 0000000..5b3a39b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr102009.c
@@ -0,0 +1,10 @@
+/* PR tree-optimization/102009 */
+/* { dg-do compile } */
+
+void *realloc ();	/* { dg-message "declared here" } */
+
+void *
+foo (void *p)
+{
+  return realloc (p);	/* { dg-warning "too few arguments to built-in function 'realloc' expecting " } */
+}
diff --git a/gcc/testsuite/gcc.dg/pr102585.c b/gcc/testsuite/gcc.dg/pr102585.c
new file mode 100644
index 0000000..efd066b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr102585.c
@@ -0,0 +1,6 @@
+/* PR debug/102585 */
+/* { dg-do compile } */
+/* { dg-options "-fvar-tracking-assignments -fno-var-tracking" } */
+
+#pragma GCC optimize 0
+void d_demangle_callback_Og() { int c = 0; }
diff --git a/gcc/testsuite/gcc.dg/pr102892-1.c b/gcc/testsuite/gcc.dg/pr102892-1.c
new file mode 100644
index 0000000..a9302b5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr102892-1.c
@@ -0,0 +1,21 @@
+/* { dg-do link } */
+/* { dg-options "-O3" } */
+/* { dg-additional-sources "pr102892-2.c" } */
+
+static long b[2][1] = {0};
+
+extern void bar(void);
+extern void foo(void);
+
+int
+main ()
+{
+  long c = 0;
+  for (long a; a < 1; ++a)
+    for (; c <= 1; c++) {
+      bar();
+      if (1 == b[c][0])
+	foo();
+    }
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/pr102892-2.c b/gcc/testsuite/gcc.dg/pr102892-2.c
new file mode 100644
index 0000000..b39bd6c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr102892-2.c
@@ -0,0 +1,7 @@
+/* { dg-do compile } */
+/* { dg-options "-O3" } */
+
+void
+bar (void)
+{
+}
diff --git a/gcc/testsuite/gcc.dg/pr102897.c b/gcc/testsuite/gcc.dg/pr102897.c
new file mode 100644
index 0000000..8e0d25e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr102897.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* Specify C99 to avoid the warning/error on compound literals.  */
+/* { dg-options "-O1 -std=c99 -Wno-psabi" } */
+
+/* Verify that there is no ICE.  */
+
+typedef __attribute__((vector_size(8))) signed char int8x8_t;
+typedef __attribute__((vector_size(8))) unsigned char uint8x8_t;
+
+int8x8_t fn1 (int8x8_t val20, char tmp)
+{
+  uint8x8_t __trans_tmp_3;
+  __trans_tmp_3 = (uint8x8_t){tmp};
+  int8x8_t __a = (int8x8_t) __trans_tmp_3;
+  return __builtin_shuffle (__a, val20, (uint8x8_t){0});
+}
diff --git a/gcc/testsuite/gcc.dg/pr102983.c b/gcc/testsuite/gcc.dg/pr102983.c
new file mode 100644
index 0000000..ef58af6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr102983.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-evrp" } */
+void foo(void);
+
+static int a = 1;
+
+int main() {
+  int c = 0;
+  for (int b = 0; b <= 0; b++) {
+    if (!a)
+      foo();
+    if (b > c){
+      if (c)
+        continue;
+      a = 0;
+    }
+    c = 1;
+  }
+}
+
+/* { dg-final { scan-tree-dump-times "Folding predicate c_.* to 1" 1 "evrp" } } */
diff --git a/gcc/testsuite/gcc.dg/pr103003.c b/gcc/testsuite/gcc.dg/pr103003.c
new file mode 100644
index 0000000..d3d65f8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103003.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+typedef char int8_t;
+int8_t c_4, uli_5;
+unsigned short us_6;
+void func_1() {
+  int uli_9;
+  short ptr_16ptr_11 = &uli_9; /* { dg-warning "initialization of*" } */
+  for (; us_6 <= 6;)
+    if ((us_6 *= uli_9) < (uli_5 || 0) ?: ((c_4 = us_6) >= us_6) - uli_5)
+      uli_9 = 9;
+}
diff --git a/gcc/testsuite/gcc.dg/pr103062.c b/gcc/testsuite/gcc.dg/pr103062.c
new file mode 100644
index 0000000..cbc371b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103062.c
@@ -0,0 +1,7 @@
+// { dg-do compile }
+// { dg-options "-O2 -fno-tree-forwprop" }
+
+void *a, *b, *c;
+void foo(void) {
+  c = (void *)((__INTPTR_TYPE__)a & (__INTPTR_TYPE__)b);
+}
diff --git a/gcc/testsuite/gcc.dg/pr103075.c b/gcc/testsuite/gcc.dg/pr103075.c
new file mode 100644
index 0000000..b332fb0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103075.c
@@ -0,0 +1,8 @@
+/* { dg-do compile } */
+/* { dg-options "-O -frounding-math" } */
+
+float
+foo (void)
+{
+  return (float) 0x1699925 * 1.1;
+}
diff --git a/gcc/testsuite/gcc.dg/pr103079.c b/gcc/testsuite/gcc.dg/pr103079.c
new file mode 100644
index 0000000..7f6632f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103079.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-Os -fdump-tree-vrp2" } */
+
+int a, b = -2;
+int main() {
+  int d = 0;
+  int t;
+  if (b)
+    goto t1;
+  if (t) {
+t1:
+    if (!a)
+      d = b;
+    while (d > -1)
+      ;
+  }
+  return 0;
+}
+/* { dg-final { scan-tree-dump "PHI" "vrp2" } } */
+
diff --git a/gcc/testsuite/gcc.dg/pr103093.c b/gcc/testsuite/gcc.dg/pr103093.c
new file mode 100644
index 0000000..f425721
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103093.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+int i_0, c_4, uc_7, func_2_c_11;
+
+short *func_2_ptr_10;
+
+void func_2() {
+  uc_7 = 7;
+  for (; uc_7 <= 60; uc_7 += 1) {
+    c_4 = 5;
+    for (; c_4 <= 76; c_4 += 1) {
+      func_2_ptr_10 = &i_0;	/* { dg-warning "assignment to .*" } */
+      if ((i_0 |= 5) > 0 ?: (60 && uc_7) | *func_2_ptr_10)
+        if (func_2_c_11)
+          for (;;)
+            ;
+    }
+  }
+}
diff --git a/gcc/testsuite/gcc.dg/pr103120.c b/gcc/testsuite/gcc.dg/pr103120.c
new file mode 100644
index 0000000..b680a6c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103120.c
@@ -0,0 +1,33 @@
+// { dg-do run }
+// { dg-options "-O2" }
+
+#define radix 10
+__INT32_TYPE__ numDigits(__UINT64_TYPE__ value)
+{
+     __INT32_TYPE__ n = 1;
+     while (value > __UINT32_MAX__)
+     {
+        n += 4;
+        value /= radix * radix * radix * radix;
+     }
+     __UINT32_TYPE__ v = (__UINT32_TYPE__)value;
+     while (1)
+     {
+         if (v < radix)
+             return n;
+         if (v < radix * radix)
+             return n + 1;
+         if (v < radix * radix * radix)
+             return n + 2;
+         if (v < radix * radix * radix * radix)
+             return n + 3;
+         n += 4;
+         v /= radix * radix * radix * radix;
+     }
+}
+
+int main()
+{
+    if (numDigits(__UINT64_MAX__) != 20)
+        __builtin_abort();
+}
diff --git a/gcc/testsuite/gcc.dg/pr103184-1.c b/gcc/testsuite/gcc.dg/pr103184-1.c
new file mode 100644
index 0000000..e567f95
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103184-1.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+extern char foo;
+extern unsigned char bar;
+
+int
+foo1 (void)
+{
+  return __sync_fetch_and_and (&foo, ~1) & 1;
+}
+
+int
+foo2 (void)
+{
+  return __sync_fetch_and_or (&foo, 1) & 1;
+}
+
+int
+foo3 (void)
+{
+  return __sync_fetch_and_xor (&foo, 1) & 1;
+}
+
+unsigned short
+bar1 (void)
+{
+  return __sync_fetch_and_and (&bar, ~1) & 1;
+}
+
+unsigned short
+bar2 (void)
+{
+  return __sync_fetch_and_or (&bar, 1) & 1;
+}
+
+unsigned short
+bar3 (void)
+{
+  return __sync_fetch_and_xor (&bar, 1) & 1;
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*cmpxchgb" 6 { target { x86_64-*-* i?86-*-* } } } } */
diff --git a/gcc/testsuite/gcc.dg/pr103184-2.c b/gcc/testsuite/gcc.dg/pr103184-2.c
new file mode 100644
index 0000000..499761f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103184-2.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+
+int
+tbit0 (_Atomic int* a, int n)
+{
+#define BIT (0x1 << n)
+  return atomic_fetch_or (a, BIT) & BIT;
+#undef BIT
+}
diff --git a/gcc/testsuite/gcc.dg/pr103207.c b/gcc/testsuite/gcc.dg/pr103207.c
new file mode 100644
index 0000000..69c0f55
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103207.c
@@ -0,0 +1,15 @@
+// { dg-do compile }
+// { dg-options "-O2 --param case-values-threshold=1 -w" }
+
+int f (int i)
+{
+  switch (i) {
+  case 2147483647:
+    return 1;
+  case 9223372036854775807L:
+    return 2;
+  case (2147483647*4)%4:
+    return 4;
+  }
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/pr103222.c b/gcc/testsuite/gcc.dg/pr103222.c
new file mode 100644
index 0000000..2a84437
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103222.c
@@ -0,0 +1,33 @@
+// { dg-do run }
+// { dg-options "-O2" }
+
+#include <stdint.h>
+#include <stdio.h>
+int16_t a;
+static uint32_t *b ;
+static uint8_t func_2();
+static int32_t func_1() {
+  int16_t a = 1;
+  func_2(0, a, a);
+  return 0;
+}
+uint8_t func_2(uint32_t p1, uint32_t p2, uint32_t p3) {
+  int p = 0;
+  for (15;; a++) {
+    for (0;;) {
+      if (p2)
+        break;
+      b = &p2;
+      return p2;
+    }
+     p3 = (p2 = p3, p);
+  }
+  return 0;
+}
+
+int main() {
+  func_1();
+  if (a != 2)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/pr103268-1.c b/gcc/testsuite/gcc.dg/pr103268-1.c
new file mode 100644
index 0000000..6d583d5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103268-1.c
@@ -0,0 +1,10 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+extern int si;
+long
+test_types (void)
+{
+  unsigned int u2 = __atomic_fetch_xor (&si, 0, 5);
+  return u2;
+}
diff --git a/gcc/testsuite/gcc.dg/pr103268-2.c b/gcc/testsuite/gcc.dg/pr103268-2.c
new file mode 100644
index 0000000..12283bb
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr103268-2.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+ 
+extern long pscc_a_2_3;
+extern int pscc_a_1_4;
+
+void
+pscc (void)
+{
+  pscc_a_1_4 = __sync_fetch_and_and (&pscc_a_2_3, 1);
+}
+
diff --git a/gcc/testsuite/gcc.dg/rtl/aarch64/big-endian-cse-1.c b/gcc/testsuite/gcc.dg/rtl/aarch64/big-endian-cse-1.c
new file mode 100644
index 0000000..1559a48
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/rtl/aarch64/big-endian-cse-1.c
@@ -0,0 +1,42 @@
+/* { dg-do compile { target aarch64*-*-* } } */
+/* { dg-require-effective-target lp64 } */
+/* { dg-options "-O3 -mbig-endian" } */
+
+void __RTL (startwith ("vregs")) foo (void *ptr1, void *ptr2)
+{
+  (function "foo"
+    (param "ptr1"
+      (DECL_RTL (reg/v:DI <0> [ ptr1 ]))
+      (DECL_RTL_INCOMING (reg:DI x0 [ ptr1 ]))
+    ) ;; param "ptr1"
+    (param "ptr2"
+      (DECL_RTL (reg/v:DI <1> [ ptr2 ]))
+      (DECL_RTL_INCOMING (reg:DI x1 [ ptr2 ]))
+    ) ;; param "ptr2"
+    (insn-chain
+      (block 2
+	(edge-from entry (flags "FALLTHRU"))
+	(cnote 3 [bb 2] NOTE_INSN_BASIC_BLOCK)
+	(insn 4 (set (reg:DI <0>) (reg:DI x0)))
+	(insn 5 (set (reg:DI <1>) (reg:DI x1)))
+	(insn 6 (set (reg:V2SI <2>)
+		     (const_vector:V2SI [(const_int 1)
+					 (const_int 0)])) (nil))
+	(insn 7 (set (mem:V2SI (reg:DI <0>) [1 ptr1+0 S8 A8])
+		     (reg:V2SI <2>)))
+	(insn 8 (set (reg:V4SI <3>)
+		     (const_vector:V4SI [(const_int 1)
+					 (const_int 1)
+					 (const_int 1)
+					 (const_int 1)])) (nil))
+	(insn 9 (set (reg:SI <4>) (subreg:SI (reg:V4SI <3>) 12))
+		(expr_list:REG_EQUAL (const_int 1) (nil)))
+	(insn 10 (set (mem:SI (reg:DI <1>) [1 ptr2+0 S4 A4])
+		      (reg:SI <4>)))
+	(edge-to exit (flags "FALLTHRU"))
+      ) ;; block 2
+    ) ;; insn-chain
+  ) ;; function
+}
+
+/* { dg-final { scan-assembler-not {\tstr\twzr,} } } */
diff --git a/gcc/testsuite/gcc.dg/signbit-2.c b/gcc/testsuite/gcc.dg/signbit-2.c
new file mode 100644
index 0000000..d8501e9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/signbit-2.c
@@ -0,0 +1,23 @@
+/* { dg-do assemble } */
+/* { dg-options "-O3 --save-temps -fdump-tree-optimized" } */
+
+/* This test does not work when the truth type does not match vector type.  */
+/* { dg-additional-options "-mno-avx512f" { target { i?86-*-* x86_64-*-* } } } */
+/* { dg-additional-options "-march=armv8-a" { target aarch64_sve } } */
+
+#include <stdint.h>
+
+void fun1(int32_t *x, int n)
+{
+    for (int i = 0; i < (n & -16); i++)
+      x[i] = (-x[i]) >> 31;
+}
+
+void fun2(int32_t *x, int n)
+{
+    for (int i = 0; i < (n & -16); i++)
+      x[i] = (-x[i]) >> 30;
+}
+
+/* { dg-final { scan-tree-dump {\s+>\s+\{ 0, 0, 0(, 0)+ \}} optimized } } */
+/* { dg-final { scan-tree-dump-not {\s+>>\s+31} optimized } } */
diff --git a/gcc/testsuite/gcc.dg/signbit-3.c b/gcc/testsuite/gcc.dg/signbit-3.c
new file mode 100644
index 0000000..19e9c06
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/signbit-3.c
@@ -0,0 +1,13 @@
+/* { dg-do assemble } */
+/* { dg-options "-O1 --save-temps -fdump-tree-optimized" } */
+
+#include <stdint.h>
+
+void fun1(int32_t *x, int n)
+{
+    for (int i = 0; i < (n & -16); i++)
+      x[i] = (-x[i]) >> 31;
+}
+
+/* { dg-final { scan-tree-dump-times {\s+>\s+0;} 1 optimized } } */
+/* { dg-final { scan-tree-dump-not {\s+>>\s+31} optimized } } */
diff --git a/gcc/testsuite/gcc.dg/signbit-4.c b/gcc/testsuite/gcc.dg/signbit-4.c
new file mode 100644
index 0000000..bc459ba
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/signbit-4.c
@@ -0,0 +1,65 @@
+/* { dg-do run } */
+/* { dg-options "-O1 -fwrapv" } */
+
+#include <stdint.h>
+#include <limits.h>
+#include <stdio.h>
+
+#ifndef N
+#define N 65
+#endif
+
+#ifndef TYPE
+#define TYPE int32_t
+#endif
+
+#ifndef DEBUG
+#define DEBUG 1
+#endif
+
+#define BASE ((TYPE) -1 < 0 ? -126 : 4)
+
+__attribute__ ((noinline, noipa))
+void fun1(TYPE *x, int n)
+{
+    for (int i = 0; i < n; i++)
+      x[i] = (-x[i]) >> 31;
+}
+
+__attribute__ ((noinline, noipa, optimize("O0")))
+void fun2(TYPE *x, int n)
+{
+    for (int i = 0; i < n; i++)
+      x[i] = (-x[i]) >> 31;
+}
+
+int main ()
+{
+  TYPE a[N];
+  TYPE b[N];
+
+  a[0] = INT_MIN;
+  b[0] = INT_MIN;
+
+  for (int i = 1; i < N; ++i)
+    {
+      a[i] = BASE + i * 13;
+      b[i] = BASE + i * 13;
+      if (DEBUG)
+        printf ("%d: 0x%x\n", i, a[i]);
+    }
+
+  fun1 (a, N);
+  fun2 (b, N);
+
+  for (int i = 0; i < N; ++i)
+    {
+      if (DEBUG)
+        printf ("%d = 0x%x == 0x%x\n", i, a[i], b[i]);
+
+      if (a[i] != b[i])
+        __builtin_abort ();
+    }
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.dg/signbit-5.c b/gcc/testsuite/gcc.dg/signbit-5.c
new file mode 100644
index 0000000..2b119cd
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/signbit-5.c
@@ -0,0 +1,70 @@
+/* { dg-do run } */
+/* { dg-options "-O3" } */
+
+/* This test does not work when the truth type does not match vector type.  */
+/* { dg-additional-options "-mno-avx512f" { target { i?86-*-* x86_64-*-* } } } */
+/* { dg-additional-options "-march=armv8-a" { target aarch64_sve } } */
+
+
+#include <stdint.h>
+#include <limits.h>
+#include <stdio.h>
+
+#ifndef N
+#define N 65
+#endif
+
+#ifndef TYPE
+#define TYPE int32_t
+#endif
+
+#ifndef DEBUG
+#define DEBUG 1
+#endif
+
+#define BASE ((TYPE) -1 < 0 ? -126 : 4)
+
+__attribute__ ((noinline, noipa))
+void fun1(TYPE *x, int n)
+{
+    for (int i = 0; i < n; i++)
+      x[i] = (-x[i]) >> 31;
+}
+
+__attribute__ ((noinline, noipa, optimize("O1")))
+void fun2(TYPE *x, int n)
+{
+    for (int i = 0; i < n; i++)
+      x[i] = (-x[i]) >> 31;
+}
+
+int main ()
+{
+  TYPE a[N];
+  TYPE b[N];
+
+  a[0] = INT_MIN;
+  b[0] = INT_MIN;
+
+  for (int i = 1; i < N; ++i)
+    {
+      a[i] = BASE + i * 13;
+      b[i] = BASE + i * 13;
+      if (DEBUG)
+        printf ("%d: 0x%x\n", i, a[i]);
+    }
+
+  fun1 (a, N);
+  fun2 (b, N);
+
+  for (int i = 0; i < N; ++i)
+    {
+      if (DEBUG)
+        printf ("%d = 0x%x == 0x%x\n", i, a[i], b[i]);
+
+      if (a[i] != b[i])
+        __builtin_abort ();
+    }
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.dg/signbit-6.c b/gcc/testsuite/gcc.dg/signbit-6.c
new file mode 100644
index 0000000..da18662
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/signbit-6.c
@@ -0,0 +1,72 @@
+/* { dg-do run } */
+/* { dg-options "-O1" } */
+
+#include <stdint.h>
+#include <limits.h>
+#include <stdio.h>
+
+#ifndef N
+#define N 65
+#endif
+
+#ifndef TYPE
+#define TYPE int32_t
+#endif
+
+#ifndef DEBUG
+#define DEBUG 1
+#endif
+
+#define BASE ((TYPE) -1 < 0 ? -126 : 4)
+
+__attribute__ ((noinline, noipa))
+void fun1(TYPE *x, int n)
+{
+    for (int i = 0; i < n; i++)
+      x[i] = (-x[i]) >> 31;
+}
+
+__attribute__ ((noinline, noipa, optimize("O0")))
+void fun2(TYPE *x, int n)
+{
+    for (int i = 0; i < n; i++)
+      x[i] = (-x[i]) >> 31;
+}
+
+int main ()
+{
+  TYPE a[N];
+  TYPE b[N];
+
+  a[0] = INT_MIN;
+  b[0] = INT_MIN;
+
+  for (int i = 1; i < N; ++i)
+    {
+      a[i] = BASE + i * 13;
+      b[i] = BASE + i * 13;
+      if (DEBUG)
+        printf ("%d: 0x%x\n", i, a[i]);
+    }
+
+  fun1 (a, N);
+  fun2 (b, N);
+
+  if (DEBUG)
+    printf ("%d = 0x%x == 0x%x\n", 0, a[0], b[0]);
+
+  if (a[0] != 0x0 || b[0] != -1)
+        __builtin_abort ();
+
+
+  for (int i = 1; i < N; ++i)
+    {
+      if (DEBUG)
+        printf ("%d = 0x%x == 0x%x\n", i, a[i], b[i]);
+
+      if (a[i] != b[i])
+        __builtin_abort ();
+    }
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.dg/struct-by-value-1.c b/gcc/testsuite/gcc.dg/struct-by-value-1.c
index addf253..ae7adb5 100644
--- a/gcc/testsuite/gcc.dg/struct-by-value-1.c
+++ b/gcc/testsuite/gcc.dg/struct-by-value-1.c
@@ -1,6 +1,7 @@
 /* Test structure passing by value.  */
 /* { dg-do run } */
 /* { dg-options "-O2" } */
+/* { dg-options "-O2 -G0" { target { nios2-*-* } } } */
 
 #define T(N)					\
 struct S##N { unsigned char i[N]; };		\
diff --git a/gcc/testsuite/gcc.dg/torture/fp-double-convert-float-1.c b/gcc/testsuite/gcc.dg/torture/fp-double-convert-float-1.c
new file mode 100644
index 0000000..ec23274
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/fp-double-convert-float-1.c
@@ -0,0 +1,41 @@
+/* PR57245 */
+/* { dg-do run } */
+/* { dg-require-effective-target fenv } */
+/* { dg-additional-options "-frounding-math" } */
+
+#include <fenv.h>
+#include <stdlib.h>
+
+int
+main ()
+{
+#if __DBL_MANT_DIG__ == 53 && __FLT_MANT_DIG__ == 24
+#ifdef FE_UPWARD
+  fesetround (FE_UPWARD);
+  float f = 1.3;
+  if (f != 0x1.4ccccep+0f)
+    __builtin_abort ();
+#endif
+#ifdef FE_TONEAREST
+  fesetround (FE_TONEAREST);
+  /* Use different actual values so the bogus CSE we perform does not
+     break things.  */
+  f = 1.33;
+  if (f != 0x1.547ae2p+0f)
+    abort ();
+#endif
+#ifdef FE_DOWNWARD
+  fesetround (FE_DOWNWARD);
+  f = 1.333;
+  if (f != 0x1.553f7cp+0f)
+    abort ();
+#endif
+#ifdef FE_TOWARDZERO
+  fesetround (FE_TOWARDZERO);
+  f = 1.3333;
+  if (f != 0x1.555326p+0f)
+    abort ();
+#endif
+#endif
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/fp-uint64-convert-double-1.c b/gcc/testsuite/gcc.dg/torture/fp-uint64-convert-double-1.c
new file mode 100644
index 0000000..fadad8c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/fp-uint64-convert-double-1.c
@@ -0,0 +1,74 @@
+/* PR84407 */
+/* { dg-do run } */
+/* { dg-require-effective-target fenv } */
+/* { dg-additional-options "-frounding-math -fexcess-precision=standard" } */
+
+#include <fenv.h>
+#include <stdlib.h>
+
+void __attribute__((noipa))
+fooa ()
+{
+#if __DBL_MANT_DIG__ == 53
+#ifdef FE_TONEAREST
+  fesetround(FE_TONEAREST);
+  __UINT64_TYPE__ x = 0x7fffffffffffffff;
+  double f = x;
+  if (f != 0x1p+63)
+    abort ();
+#endif
+#endif
+}
+
+void __attribute__((noipa))
+foob ()
+{
+#if __DBL_MANT_DIG__ == 53
+#ifdef FE_DOWNWARD
+  fesetround(FE_DOWNWARD);
+  __UINT64_TYPE__ x = 0x7fffffffffffffff;
+  double f = x;
+  if (f != 0x1.fffffffffffffp+62)
+    abort ();
+#endif
+#endif
+}
+
+void __attribute__((noipa))
+fooc ()
+{
+#if __DBL_MANT_DIG__ == 53
+#ifdef FE_UPWARD
+  fesetround(FE_UPWARD);
+  __UINT64_TYPE__ x = 0x7fffffffffffffff;
+  double f = x;
+  if (f != 0x1p+63)
+    abort ();
+#endif
+#endif
+}
+
+void __attribute__((noipa))
+food ()
+{
+#if __DBL_MANT_DIG__ == 53
+#ifdef FE_TOWARDZERO
+  fesetround(FE_TOWARDZERO);
+  __UINT64_TYPE__ x = 0x7fffffffffffffff;
+  double f = x;
+  if (f != 0x1.fffffffffffffp+62)
+    abort ();
+#endif
+#endif
+}
+
+
+int
+main ()
+{
+  fooa ();
+  foob ();
+  fooc ();
+  food ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/fp-uint64-convert-double-2.c b/gcc/testsuite/gcc.dg/torture/fp-uint64-convert-double-2.c
new file mode 100644
index 0000000..952f96b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/fp-uint64-convert-double-2.c
@@ -0,0 +1,75 @@
+/* PR84407 */
+/* { dg-do run } */
+/* { dg-require-effective-target fenv } */
+/* { dg-additional-options "-frounding-math" } */
+
+#include <fenv.h>
+#include <stdlib.h>
+
+void __attribute__((noipa))
+fooa ()
+{
+#if __DBL_MANT_DIG__ == 53
+#ifdef FE_TONEAREST
+  fesetround(FE_TONEAREST);
+  /* Large enough constant to trigger unsigned_float.  */
+  __UINT64_TYPE__ x = 0x8000000000000001;
+  double f = x;
+  if (f != 0x1p+63)
+    abort ();
+#endif
+#endif
+}
+
+void __attribute__((noipa))
+foob ()
+{
+#if __DBL_MANT_DIG__ == 53
+#ifdef FE_DOWNWARD
+  fesetround(FE_DOWNWARD);
+  __UINT64_TYPE__ x = 0x8000000000000001;
+  double f = x;
+  if (f != 0x1p+63)
+    abort ();
+#endif
+#endif
+}
+
+void __attribute__((noipa))
+fooc ()
+{
+#if __DBL_MANT_DIG__ == 53
+#ifdef FE_UPWARD
+  fesetround(FE_UPWARD);
+  __UINT64_TYPE__ x = 0x8000000000000001;
+  double f = x;
+  if (f != 0x1.0000000000001p+63)
+    abort ();
+#endif
+#endif
+}
+
+void __attribute__((noipa))
+food ()
+{
+#if __DBL_MANT_DIG__ == 53
+#ifdef FE_TOWARDZERO
+  fesetround(FE_TOWARDZERO);
+  __UINT64_TYPE__ x = 0x8000000000000001;
+  double f = x;
+  if (f != 0x1p+63)
+    abort ();
+#endif
+#endif
+}
+
+
+int
+main ()
+{
+  fooa ();
+  foob ();
+  fooc ();
+  food ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr102920.c b/gcc/testsuite/gcc.dg/torture/pr102920.c
new file mode 100644
index 0000000..aa27ac5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr102920.c
@@ -0,0 +1,25 @@
+/* { dg-do run } */
+/* { dg-additional-options "-funswitch-loops" } */
+
+unsigned short a = 42;
+unsigned short b = 1;
+long int c = 1;
+unsigned char var_120;
+unsigned char var_123;
+
+void __attribute__((noipa)) test(unsigned short a, unsigned short b, long c)
+{
+  for (char i = 0; i < (char)c; i += 5)
+    if (!b)
+      var_120 = a;
+    else
+      var_123 = a;
+}
+
+int main()
+{
+  test(a, b, c);
+  if (var_123 != 42)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr103181.c b/gcc/testsuite/gcc.dg/torture/pr103181.c
new file mode 100644
index 0000000..6bc705a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr103181.c
@@ -0,0 +1,24 @@
+/* { dg-do run } */
+
+typedef unsigned char __attribute__((__vector_size__ (2))) U;
+typedef unsigned short S;
+typedef unsigned int __attribute__((__vector_size__ (64))) V;
+
+V v;
+U a, b, c;
+
+U
+foo (S s)
+{
+  v += __builtin_bswap16 (s) || (S) (a / ((U){3, 0}));
+  return b + c;
+}
+
+int
+main (void)
+{
+  U x = foo (4);
+  if (x[0] || x[1])
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr103188.c b/gcc/testsuite/gcc.dg/torture/pr103188.c
new file mode 100644
index 0000000..0412f6f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr103188.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+
+int a, b, c, d = 10, e = 1, f, g, h, i;
+int main()
+{
+  int j = -1;
+k:
+  h = c;
+l:
+  c = ~c;
+  if (e)
+  m:
+    a = 0;
+  if (j > 1)
+    goto m;
+  if (!e)
+    goto l;
+  if (c)
+    goto p;
+n:
+  goto m;
+o:
+  if (f) {
+    if (g)
+      goto k;
+    j = 0;
+  p:
+    if (d)
+      goto o;
+    goto n;
+  }
+  if (i)
+    goto l;
+  for (; a < 1; a++)
+    while (a > d)
+      b++;
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr103204.c b/gcc/testsuite/gcc.dg/torture/pr103204.c
new file mode 100644
index 0000000..b08b686
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr103204.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+
+int a[1][1];
+int b, c;
+void d()
+{
+  int e;
+  int f[0];
+  int *g;
+  for (; e;)
+    c = b;
+  if (a[0][b] = (__UINTPTR_TYPE__)f != 3)
+    g = &f[b];
+  int h;
+  *g = *g = *g = h;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr103219.c b/gcc/testsuite/gcc.dg/torture/pr103219.c
new file mode 100644
index 0000000..cb9023e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr103219.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+
+int f();
+void g();
+int a, b, c, e;
+int d[10];
+int main()
+{
+  if (c)
+    if (f())
+      {
+	g();
+	if (e) {
+	    a = 0;
+	    for (; a != 6; a = a + 2)
+	      {
+		b = 0;
+		for (; b <= 3; b++)
+		  d[b] &= 1;
+	      }
+	}
+      }
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr103237.c b/gcc/testsuite/gcc.dg/torture/pr103237.c
new file mode 100644
index 0000000..f2399f9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr103237.c
@@ -0,0 +1,24 @@
+/* { dg-do run } */
+/* { dg-additional-options "-ftree-vectorize" } */
+
+int g1;
+unsigned int g2 = -1U;
+static void __attribute__((noipa))
+func_1()
+{
+  int *l_1 = &g1;
+  for (int g3a = 0; g3a != 4; g3a++)
+    for (int l_2 = 0; l_2 <= 3; l_2++)
+      {
+        unsigned int *l_3 = &g2;
+        *l_1 = *l_3 ^= 1;
+      }
+}
+int
+main()
+{
+  func_1();
+  if (g1 != -1)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/bittest.c b/gcc/testsuite/gcc.dg/tree-ssa/bittest.c
new file mode 100644
index 0000000..7d712ca
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/bittest.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+
+
+void bar (void);
+
+void
+foo(unsigned int abc123)
+{
+  unsigned int xyzpdq = (1 << abc123);
+  if ((xyzpdq & 0x800) != 0)
+    bar();
+}
+
+void
+baz(unsigned int abc123)
+{
+  unsigned int xyzpdq = (1 << abc123);
+  if ((xyzpdq & 0x800) == 0)
+    bar();
+}
+
+/* What we want to verify is that the bit test against xyzpdq is
+   replaced with a test against abc123 which avoids the shifting
+   and bit ops.  */
+/* { dg-final { scan-tree-dump-not "xyzpdq" "optimized"} } */
+/* { dg-final { scan-tree-dump-times "if .abc123" 2 "optimized"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-16.c b/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-16.c
index d6cbfe3..f975e16 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-16.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-16.c
@@ -130,16 +130,18 @@ void test_width_var (void)
 
   {
     /* Create an unsigned range with a lower bound greater than 1 and
-       an upper bound in excess of INT_MAX and verify that the lower
-       bound isn't used as the minimum output (since the excessive
-       upper bound wraps around zero).  It's possible to constrain
-       the upper bound on the output more, based on the upper bound
-       of the width here, but not worth the trouble.  */
-    extern unsigned w;
+       an upper bound in excess of INT_MAX and verify that after conversion
+       to signed int the lower bound isn't used as the minimum output (since
+       the excessive upper bound wraps around zero).  Passing the precision
+       as unsigned, without the cast to signed int, is undedined.
+       It's possible to constrain the upper bound on the output more, based
+       on the upper bound of the width here, but not worth the trouble.  */
+    unsigned w = W (unsigned);
     if (w < 5 || (unsigned)-1 - 7 < w)
       w = 5;
+    int val = V (int);
 
-    T ("%*u", w, V (int));   /* { dg-warning "between 1 and 2147483648 " } */
+    T ("%*u", (int)w, val);   /* { dg-warning "between 1 and 2147483648 " } */
   }
 
   {
@@ -234,13 +236,17 @@ void test_precision_var (void)
   {
     /* Similar to the corresponding width case, create an unsigned range
        with a lower bound greater than 1 and an upper bound in excess of
-       INT_MAX and verify that the lower bound isn't used as the minimum
-       output (since the excessive upper bound wraps around zero).  */
-    extern unsigned p;
+       INT_MAX and verify that after conversion to int the lower bound isn't
+       used as the minimum output (since the excessive upper bound wraps
+       around zero).  Passing the precision as unsigned, without a cast to
+       signed int, is undefined.  */
+    unsigned p = V (unsigned);
     if (p < 7 || (unsigned)-1 - 9 < p)
       p = 7;
 
-    T ("%.*u", p, V (int));   /* { dg-warning "up to 2147483647 " } */
+    int val = V (int);
+
+    T ("%.*u", (int)p, val);   /* { dg-warning "up to 2147483647 " } */
   }
 
   {
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-22.c b/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-22.c
index 685a4fd..82eb585 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-22.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-22.c
@@ -18,7 +18,18 @@ void g (char *s1, char *s2)
   if (n + d + 1 >= 1025)
     return;
 
-  sprintf (b, "%s.%s", s1, s2);     // { dg-bogus "\\\[-Wformat-overflow" }
+  /* Ranger can find ranges here:
+     [1] n_6: size_t [0, 1023]
+     [2] d_8: size_t [0, 1023]
+
+     Whereas evrp can't really:
+     [1] n_6: size_t [0, 9223372036854775805]
+     [2] d_8: size_t [0, 9223372036854775805]
+
+     This is causing the sprintf warning pass to issue a false
+     positive here.  */
+
+  sprintf (b, "%s.%s", s1, s2);     // { dg-bogus "\\\[-Wformat-overflow" "" { xfail *-*-* } }
 
   f (b);
 }
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-23.c b/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-23.c
index 7fb9651..112b08a 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-23.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/builtin-sprintf-warn-23.c
@@ -214,12 +214,14 @@ void test_struct_member_array (struct S3 *s3, int i)
   T (d, "%s", d);       /* { dg-warning "overlaps" } */
   T (d, "%s", d + 0);   /* { dg-warning "overlaps" } */
   T (d, "%s", d + 1);   /* { dg-warning "may overlap" } */
-  T (d, "%s", d + 2);   /* { dg-warning "may overlap" } */
+  /* Since d below points to char[4], strlen(d + 2) must be at most 1
+     and so the call cannot overlap. */
+  T (d, "%s", d + 2);
   T (d, "%s", d + i);   /* { dg-warning "may overlap" } */
 
   T (d, "%s", &d[0]);   /* { dg-warning "overlaps" } */
   T (d, "%s", &d[1]);   /* { dg-warning "may overlap" } */
-  T (d, "%s", &d[2]);   /* { dg-warning "may overlap" } */
+  T (d, "%s", &d[2]);
   T (d, "%s", &d[i]);   /* { dg-warning "may overlap" } */
 
   T (d + 0, "%s", d);   /* { dg-warning "overlaps" } */
@@ -236,7 +238,7 @@ void test_struct_member_array (struct S3 *s3, int i)
 
   T (d, "%s", s);       /* { dg-warning "overlaps" } */
   T (d, "%s", s + 1);   /* { dg-warning "may overlap" } */
-  T (d, "%s", s + 2);   /* { dg-warning "may overlap" } */
+  T (d, "%s", s + 2);
   T (d, "%s", s + i);   /* { dg-warning "may overlap" } */
 
   s = s3->s2_1.s_1.b;
@@ -324,7 +326,7 @@ void test_struct_member_array (struct S3 *s3, int i)
 
   T (d, "%s", s);       /* { dg-warning "overlaps" } */
   T (d, "%s", s + 1);   /* { dg-warning "may overlap" } */
-  T (d, "%s", s + 2);   /* { dg-warning "may overlap" } */
+  T (d, "%s", s + 2);
   T (d, "%s", s + i);   /* { dg-warning "may overlap" } */
 
   s = s3->s2_2.s_2.a;
@@ -368,7 +370,7 @@ void test_struct_member_array (struct S3 *s3, int i)
 
   T (d, "%s", s);       /* { dg-warning "overlaps" } */
   T (d, "%s", s + 1);   /* { dg-warning "may overlap" } */
-  T (d, "%s", s + 2);   /* { dg-warning "may overlap" } */
+  T (d, "%s", s + 2);
   T (d, "%s", s + i);   /* { dg-warning "may overlap" } */
 
   s = s3->s2_2.s_2.a;
@@ -394,12 +396,12 @@ void test_struct_member_array_array (struct S3 *s3, int i)
   T (d, "%s", s);       /* { dg-warning "overlaps" } */
   T (d, "%s", s + 0);   /* { dg-warning "overlaps" } */
   T (d, "%s", s + 1);   /* { dg-warning "may overlap" } */
-  T (d, "%s", s + 2);   /* { dg-warning "may overlap" } */
+  T (d, "%s", s + 2);
   T (d, "%s", s + i);   /* { dg-warning "may overlap" } */
 
   T (d, "%s", &s[0]);   /* { dg-warning "overlaps" } */
   T (d, "%s", &s[1]);   /* { dg-warning "may overlap" } */
-  T (d, "%s", &s[2]);   /* { dg-warning "may overlap" } */
+  T (d, "%s", &s[2]);
   T (d, "%s", &s[i]);   /* { dg-warning "may overlap" } */
 
   T (d + 0, "%s", s);   /* { dg-warning "overlaps" } */
@@ -566,12 +568,12 @@ void test_union_member_array (union U *un, int i)
   T (d, "%s", d);       /* { dg-warning "overlaps" } */
   T (d, "%s", d + 0);   /* { dg-warning "overlaps" } */
   T (d, "%s", d + 1);   /* { dg-warning "may overlap" } */
-  T (d, "%s", d + 2);   /* { dg-warning "may overlap" } */
+  T (d, "%s", d + 2);
   T (d, "%s", d + i);   /* { dg-warning "may overlap" } */
 
   T (d, "%s", &d[0]);   /* { dg-warning "overlaps" } */
   T (d, "%s", &d[1]);   /* { dg-warning "may overlap" } */
-  T (d, "%s", &d[2]);   /* { dg-warning "may overlap" } */
+  T (d, "%s", &d[2]);
   T (d, "%s", &d[i]);   /* { dg-warning "may overlap" } */
 
   T (d + 0, "%s", d);   /* { dg-warning "overlaps" } */
@@ -588,7 +590,7 @@ void test_union_member_array (union U *un, int i)
 
   T (d, "%s", s);       /* { dg-warning "overlaps" } */
   T (d, "%s", s + 1);   /* { dg-warning "may overlap" } */
-  T (d, "%s", s + 2);   /* { dg-warning "may overlap" } */
+  T (d, "%s", s + 2);
   T (d, "%s", s + i);   /* { dg-warning "may overlap" } */
 
   s = un->s2_1.s_1.b;
@@ -616,7 +618,7 @@ void test_union_member_array (union U *un, int i)
 
   T (d, "%s", s);       /* { dg-warning "overlaps" } */
   T (d, "%s", s + 1);   /* { dg-warning "may overlap" } */
-  T (d, "%s", s + 2);   /* { dg-warning "may overlap" } */
+  T (d, "%s", s + 2);
   T (d, "%s", s + i);   /* { dg-warning "may overlap" } */
 
   s = un->s2_2.s_1.b;
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/evrp9.c b/gcc/testsuite/gcc.dg/tree-ssa/evrp9.c
index 6e7828e..fb7c319 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/evrp9.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/evrp9.c
@@ -24,5 +24,7 @@ foo (unsigned int x, unsigned int y)
     bar ();
 }
 
-/* { dg-final { scan-tree-dump-not "== 5" "evrp" } } */
-/* { dg-final { scan-tree-dump-not "== 6" "evrp" } } */
+/* { dg-final { scan-tree-dump-times "Folding predicate minv_.* == 5 to 0" 1 "evrp" } } */
+/* { dg-final { scan-tree-dump-times "Folding predicate minv_.* == 6 to 0" 1 "evrp" } } */
+/* { dg-final { scan-tree-dump-times "Folding predicate maxv_.* == 5 to 0" 1 "evrp" } } */
+/* { dg-final { scan-tree-dump-times "Folding predicate maxv_.* == 6 to 0" 1 "evrp" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ldist-rawmemchr-1.c b/gcc/testsuite/gcc.dg/tree-ssa/ldist-rawmemchr-1.c
index 6abfd27..bf6335f 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ldist-rawmemchr-1.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ldist-rawmemchr-1.c
@@ -1,5 +1,6 @@
 /* { dg-do run { target s390x-*-* } } */
 /* { dg-options "-O2 -ftree-loop-distribution -fdump-tree-ldist-details" } */
+/* { dg-additional-options "-march=z13 -mzarch" { target s390x-*-* } } */
 /* { dg-final { scan-tree-dump-times "generated rawmemchrQI" 2 "ldist" { target s390x-*-* } } } */
 /* { dg-final { scan-tree-dump-times "generated rawmemchrHI" 2 "ldist" { target s390x-*-* } } } */
 /* { dg-final { scan-tree-dump-times "generated rawmemchrSI" 2 "ldist" { target s390x-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ldist-rawmemchr-2.c b/gcc/testsuite/gcc.dg/tree-ssa/ldist-rawmemchr-2.c
index 00d6ea0..83f5a35 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ldist-rawmemchr-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ldist-rawmemchr-2.c
@@ -1,5 +1,6 @@
 /* { dg-do run { target s390x-*-* } } */
 /* { dg-options "-O2 -ftree-loop-distribution -fdump-tree-ldist-details" } */
+/* { dg-additional-options "-march=z13 -mzarch" { target s390x-*-* } } */
 /* { dg-final { scan-tree-dump-times "generated rawmemchrQI" 2 "ldist" { target s390x-*-* } } } */
 /* { dg-final { scan-tree-dump-times "generated rawmemchrHI" 2 "ldist" { target s390x-*-* } } } */
 /* { dg-final { scan-tree-dump-times "generated rawmemchrSI" 2 "ldist" { target s390x-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ldist-strlen-1.c b/gcc/testsuite/gcc.dg/tree-ssa/ldist-strlen-1.c
index 918b600..aeb04b9 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ldist-strlen-1.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ldist-strlen-1.c
@@ -1,5 +1,6 @@
 /* { dg-do run } */
 /* { dg-options "-O2 -ftree-loop-distribution -fdump-tree-ldist-details" } */
+/* { dg-additional-options "-march=z13 -mzarch" { target s390x-*-* } } */
 /* { dg-final { scan-tree-dump-times "generated strlenQI\n" 4 "ldist" } } */
 /* { dg-final { scan-tree-dump-times "generated strlenHI\n" 4 "ldist" { target s390x-*-* } } } */
 /* { dg-final { scan-tree-dump-times "generated strlenSI\n" 4 "ldist" { target s390x-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ldist-strlen-3.c b/gcc/testsuite/gcc.dg/tree-ssa/ldist-strlen-3.c
index 370fd5e..0652857 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ldist-strlen-3.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ldist-strlen-3.c
@@ -1,5 +1,6 @@
 /* { dg-do compile } */
 /* { dg-options "-O2 -ftree-loop-distribution -fdump-tree-ldist-details" } */
+/* { dg-additional-options "-march=z13 -mzarch" { target s390x-*-* } } */
 /* { dg-final { scan-tree-dump-times "generated strlenSI\n" 1 "ldist" { target s390x-*-* } } } */
 
 extern int s[];
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-10.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-10.c
new file mode 100644
index 0000000..4a6d9e5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-10.c
@@ -0,0 +1,20 @@
+/* { dg-do link } */
+/* { dg-options "-O2 -fdump-tree-modref1"  } */
+void linker_error ();
+__attribute__((noinline))
+int *
+test (int **a)
+{
+  return *a;
+}
+int
+main()
+{
+	int val;
+	int *a=&val;
+	int *b = test (&a);
+	if (b == (int *)&a)
+		linker_error ();
+	return 0;
+}
+/* { dg-final { scan-tree-dump "no_direct_clobber no_indirect_clobber no_direct_escape no_indirect_escape not_returned_directly no_indirect_read" "modref1"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-11.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-11.c
new file mode 100644
index 0000000..10ebe1f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-11.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-modref1 -fno-ipa-pure-const"  } */
+struct linkedlist {
+  struct linkedlist *next;
+};
+struct linkedlist *
+find_last (struct linkedlist *l)
+{
+  while (l->next)
+   l = l->next;
+  return l;
+}
+/* { dg-final { scan-tree-dump "parm 0 flags: no_direct_clobber no_indirect_clobber no_direct_escape no_indirect_escape" "modref1"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-12.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-12.c
new file mode 100644
index 0000000..f8ce047
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-12.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized"  } */
+void foo ();
+int
+test()
+{
+  struct {int a,b;} a = {0,0};
+  __attribute__ ((noinline))
+  void nested ()
+  {
+	  a.b++;
+  }
+  nested ();
+  return a.a;
+}
+/* { dg-final { scan-tree-dump "return 0" "optimized"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-13.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-13.c
new file mode 100644
index 0000000..276711e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-13.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-release_ssa"  } */
+struct wrap {
+	void **array;
+};
+__attribute__ ((noinline))
+void
+write_array (struct wrap *ptr)
+{
+	ptr->array[0]=0;
+}
+int
+test ()
+{
+	void *arrayval;
+	struct wrap w = {&arrayval};
+	write_array (&w);
+	return w.array == &arrayval;
+}
+/* We should deterine that write_array writes to PTR only indirectly.  */
+/* { dg-final { scan-tree-dump "return 1" "release_ssa"  } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-14.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-14.c
new file mode 100644
index 0000000..986248f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-14.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wsuggest-attribute=const" } */
+struct a {int a,b,c;};
+__attribute__ ((noinline))
+int init (struct a *a)
+{
+  a->a=1;
+  a->b=2;
+  a->c=3;
+}
+int const_fn () /* { dg-warning "function might be candidate for attribute 'const" } */
+{
+  struct a a;
+  init (&a);
+  return a.a + a.b + a.c;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-5.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-5.c
index fde3177..0bee79d 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/modref-5.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-5.c
@@ -24,4 +24,4 @@ main()
     __builtin_abort ();
   return 0;
 }
-/* { dg-final { scan-tree-dump "parm 1 flags: nodirectescape" "modref1"  } } */
+/* { dg-final { scan-tree-dump "parm 1 flags: no_direct_clobber no_direct_escape" "modref1"  } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-6.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-6.c
index 2d97a49..7146389 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/modref-6.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-6.c
@@ -28,10 +28,10 @@ int test2()
    return a;
 }
 /* Flags for normal call.  */
-/* { dg-final { scan-tree-dump "parm 0 flags: direct noclobber noescape nodirectescape not_returned" "modref1"  } } */
+/* { dg-final { scan-tree-dump "parm 0 flags: no_direct_clobber no_indirect_clobber no_direct_escape no_indirect_escape not_returned_directly not_returned_indirectly no_indirect_read" "modref1"  } } */
 /* Flags for pure call.  */
-/* { dg-final { scan-tree-dump "parm 0 flags: direct not_returned" "modref1"  } } */
+/* { dg-final { scan-tree-dump "parm 0 flags: not_returned_directly not_returned_indirectly no_indirect_read" "modref1"  } } */
 /* Flags for const call.  */
-/* { dg-final { scan-tree-dump "parm 0 flags: not_returned" "modref1"  } } */
+/* { dg-final { scan-tree-dump "parm 0 flags: not_returned_directly" "modref1"  } } */
 /* Overall we want to make "int a" non escaping.  */
 /* { dg-final { scan-tree-dump "return 42" "optimized"  } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-8.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-8.c
index 15ae4ac..4a18e34 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/modref-8.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-8.c
@@ -1,4 +1,4 @@
-/* { dg-options "-O2 --param modref-max-adjustments=8 -fdump-tree-modref1"  } */
+/* { dg-options "-O2 --param modref-max-adjustments=8 -fdump-tree-modref1 -fno-optimize-sibling-calls"  } */
 /* { dg-do compile } */
 void
 set (char *p)
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-1.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-1.c
new file mode 100644
index 0000000..1f80cc5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-1.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-dse1-details"  } */
+volatile int *ptr;
+struct a {
+	int a,b,c;
+} a;
+__attribute__((noinline))
+static int init (struct a*a)
+{
+	a->a=0;
+	a->b=1;
+}
+__attribute__((noinline))
+static int use (struct a*a)
+{
+	if (a->c != 3)
+		*ptr=5;
+}
+
+void
+main(void)
+{
+	struct a a;
+	init (&a);
+	a.c=3;
+	use (&a);
+}
+/* { dg-final { scan-tree-dump "Deleted dead store: init" "dse1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-2.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-2.c
new file mode 100644
index 0000000..e41d065
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-2.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-dse2-details"  } */
+volatile int *ptr;
+struct a {
+	int a,b,c;
+} a;
+__attribute__((noinline))
+static int init (struct a*a)
+{
+	a->a=0;
+	a->b=1;
+	a->c=1;
+}
+__attribute__((noinline))
+static int use (struct a*a)
+{
+	if (a->c != 3)
+		*ptr=5;
+}
+
+void
+main(void)
+{
+	struct a a;
+	init (&a);
+	a.c=3;
+	use (&a);
+}
+/* Only DSE2 is tracking live bytes needed to figure out that store to c is
+   also dead above.  */
+/* { dg-final { scan-tree-dump "Deleted dead store: init" "dse2" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-3.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-3.c
new file mode 100644
index 0000000..c69e423
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-3.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-dse1-details"  } */
+__attribute__ ((noinline))
+void write (int *a)
+{
+	*a=1;
+	a[1]=2;
+}
+int test ()
+{
+	int a;
+	a=2;
+	write (&a);
+	return a;
+}
+int test2 (int *a)
+{
+	*a=2;
+	write (a);
+	return *a;
+}
+/* { dg-final { scan-tree-dump-times "Deleted dead store" 2 "dse1"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-5.c b/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-5.c
new file mode 100644
index 0000000..ad35b70
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/modref-dse-5.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-dse2-details"  } */
+struct a {int a,b,c;};
+__attribute__ ((noinline))
+void
+kill_me (struct a *a)
+{
+  a->a=0;
+  a->b=0;
+  a->c=0;
+}
+__attribute__ ((noinline))
+int
+wrap(int b, struct a *a)
+{
+   kill_me (a);
+   return b;
+}
+__attribute__ ((noinline))
+void
+my_pleasure (struct a *a)
+{
+  a->a=1;
+  a->c=2;
+}
+__attribute__ ((noinline))
+int
+wrap2(int b, struct a *a)
+{
+   my_pleasure (a);
+   return b;
+}
+
+int
+set (struct a *a)
+{
+  wrap (0, a);
+  int ret = wrap2 (0, a);
+  //int ret = my_pleasure (a);
+  a->b=1;
+  return ret;
+}
+/* { dg-final { scan-tree-dump "Deleted dead store: wrap" "dse2" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/phi-opt-26.c b/gcc/testsuite/gcc.dg/tree-ssa/phi-opt-26.c
new file mode 100644
index 0000000..21aa66e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/phi-opt-26.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O -fgimple -fdump-tree-phiopt1" } */
+
+int __GIMPLE (ssa,startwith("phiopt"))
+foo (int a, int b, int flag)
+{
+  int res;
+
+  __BB(2):
+  if (flag_2(D) != 0)
+    goto __BB6;
+  else
+    goto __BB4;
+
+  __BB(4):
+  if (a_3(D) > b_4(D))
+    goto __BB7;
+  else
+    goto __BB6;
+
+  __BB(6):
+  goto __BB7;
+
+  __BB(7):
+  res_1 = __PHI (__BB4: a_3(D), __BB6: b_4(D));
+  return res_1;
+}
+
+/* We should be able to detect MAX despite the extra edge into
+   the middle BB.  */
+/* { dg-final { scan-tree-dump "MAX" "phiopt1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-1.c b/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-1.c
index 59663dd..be504dd 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-1.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-Ofast -fdump-tree-vrp-thread1" } */
+/* { dg-options "-Ofast -fdump-tree-dom2" } */
 
 void g (int);
 void g1 (int);
@@ -27,4 +27,9 @@ f (long a, long b, long c, long d, long x)
   g (a);
 }
 
-/* { dg-final { scan-tree-dump-times "Removing basic block" 1 "vrp-thread1" } } */
+/* This is actually a regression.  The backward threader cannot thread
+   the above scenario, but it is being caught by the DOM threader
+   which still uses the forward threader.  We should implement this
+   optimization in the backward threader before killing the forward
+   threader.  Similarly for the other phi_on_compare-*.c tests.  */
+/* { dg-final { scan-tree-dump-times "Removing basic block" 1 "dom2" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-2.c b/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-2.c
index 0c2f6e0..b186570 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-2.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-Ofast -fdump-tree-vrp-thread1" } */
+/* { dg-options "-Ofast -fdump-tree-dom2" } */
 
 void g (void);
 void g1 (void);
@@ -20,4 +20,4 @@ f (long a, long b, long c, long d, int x)
     }
 }
 
-/* { dg-final { scan-tree-dump-times "Removing basic block" 1 "vrp-thread1" } } */
+/* { dg-final { scan-tree-dump-times "Removing basic block" 1 "dom2" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-3.c b/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-3.c
index 6a3d359..b48ecbf 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-3.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-3.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-Ofast -fdump-tree-vrp-thread1" } */
+/* { dg-options "-Ofast -fdump-tree-dom2" } */
 
 void g (void);
 void g1 (void);
@@ -22,4 +22,4 @@ f (long a, long b, long c, long d, int x)
     }
 }
 
-/* { dg-final { scan-tree-dump-times "Removing basic block" 1 "vrp-thread1" } } */
+/* { dg-final { scan-tree-dump-times "Removing basic block" 1 "dom2" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-4.c b/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-4.c
index 9bc4c6d..1e09f89 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-4.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/phi_on_compare-4.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-Ofast -fdump-tree-vrp-thread1" } */
+/* { dg-options "-Ofast -fdump-tree-dom2" } */
 
 void g (int);
 void g1 (int);
@@ -37,4 +37,4 @@ f (long a, long b, long c, long d, int x)
   g (c + d);
 }
 
-/* { dg-final { scan-tree-dump-times "Removing basic block" 1 "vrp-thread1" } } */
+/* { dg-final { scan-tree-dump-times "Removing basic block" 1 "dom2" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr102880.c b/gcc/testsuite/gcc.dg/tree-ssa/pr102880.c
new file mode 100644
index 0000000..0306dee
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr102880.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+
+void foo(void);
+
+static int b, c, d, e, f, ah;
+static short g, ai, am, aq, as;
+static char an, at, av, ax, ay;
+static char a(char h, char i) { return i == 0 || h && i == 1 ? 0 : h % i; }
+static void ae(int h) {
+  if (a(b, h))
+    foo();
+
+}
+int main() {
+  ae(1);
+  ay = a(0, ay);
+  ax = a(g, aq);
+  at = a(0, as);
+  av = a(c, 1);
+  an = a(am, f);
+  int al = e || ((a(1, ah) && b) & d) == 2;
+  ai = al;
+}
+
+/* We should eliminate the call to foo.  */
+/* { dg-final { scan-tree-dump-not "foo" "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr102906.c b/gcc/testsuite/gcc.dg/tree-ssa/pr102906.c
new file mode 100644
index 0000000..1846f0b6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr102906.c
@@ -0,0 +1,17 @@
+// { dg-do compile }
+// { dg-options "-Os -fdump-tree-ch-details" }
+
+extern unsigned int foo (int*) __attribute__((pure));
+
+unsigned int
+tr2 (int array[], int n)
+{
+  unsigned int sum = 0;
+  int x;
+  if (n > 0)
+    for (x = 0; x < n; x++)
+      sum += foo (&array[x]);
+  return sum;
+}
+
+// { dg-final { scan-tree-dump-not "Not duplicating.*optimizing for size" "ch2" } }
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr102951.c b/gcc/testsuite/gcc.dg/tree-ssa/pr102951.c
new file mode 100644
index 0000000..42b6fe6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr102951.c
@@ -0,0 +1,41 @@
+/* PR tree-optimization/102951 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-ccp1" } */
+/* { dg-final { scan-tree-dump-times "return \&a\\\[1\\\];" 2 "ccp1" } } */
+/* { dg-final { scan-tree-dump-times "return \&a\\\[4\\\];" 2 "ccp1" } } */
+/* { dg-final { scan-tree-dump-not "MIN_EXPR" "ccp1" } } */
+/* { dg-final { scan-tree-dump-not "MAX_EXPR" "ccp1" } } */
+
+extern int a[5];
+
+int *
+foo (void)
+{
+  int *p1 = &a[1];
+  int *p2 = &a[2];
+  return p1 < p2 ? p1 : p2;
+}
+
+int *
+bar (void)
+{
+  int *p1 = &a[1];
+  int *p2 = &a[2];
+  return p1 <= p2 ? p1 : p2;
+}
+
+int *
+baz (void)
+{
+  int *p1 = &a[3];
+  int *p2 = &a[4];
+  return p1 > p2 ? p1 : p2;
+}
+
+int *
+qux (void)
+{
+  int *p1 = &a[3];
+  int *p2 = &a[4];
+  return p1 >= p2 ? p1 : p2;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr103245-1.c b/gcc/testsuite/gcc.dg/tree-ssa/pr103245-1.c
new file mode 100644
index 0000000..68ddead
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr103245-1.c
@@ -0,0 +1,25 @@
+/* PR tree-optimization/103245 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times " = ABSU_EXPR <v_\[0-9]*\\\(D\\\)>;" 1 "optimized" } } */
+
+unsigned
+f1 (int v)
+{
+  unsigned int d_6;
+  int b_5;
+  int a_4;
+  _Bool _1;
+  unsigned int v1_2;
+  unsigned int _7;
+  int _9;
+
+  _1 = v < 0;
+  a_4 = (int) _1;
+  b_5 = -a_4;
+  _9 = b_5 | 1;
+  d_6 = (unsigned int) _9;
+  v1_2 = (unsigned int) v;
+  _7 = v1_2 * d_6;
+  return _7;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr20701.c b/gcc/testsuite/gcc.dg/tree-ssa/pr20701.c
index 2f91458..f05076c 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr20701.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr20701.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-vrp1 -fno-early-inlining -fdelete-null-pointer-checks" } */
+/* { dg-options "-O2 -fdump-tree-vrp1 -fno-early-inlining -fdelete-null-pointer-checks -fno-thread-jumps" } */
 
 typedef struct {
   int code;
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr20702.c b/gcc/testsuite/gcc.dg/tree-ssa/pr20702.c
index c8968577..8112967 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr20702.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr20702.c
@@ -4,7 +4,7 @@
    immediate successors of the basic block.  */
 
 /* { dg-do compile } */
-/* { dg-options "-O2 -fno-tree-dominator-opts -fdisable-tree-evrp -fdump-tree-vrp1-details -fdelete-null-pointer-checks" } */
+/* { dg-options "-O2 -fno-thread-jumps -fdisable-tree-evrp -fdump-tree-vrp1-details -fdelete-null-pointer-checks" } */
 
 extern void bar (int);
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr21001.c b/gcc/testsuite/gcc.dg/tree-ssa/pr21001.c
index 4ea5f21..f9216a4 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr21001.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr21001.c
@@ -5,7 +5,7 @@
    range information out of the conditional.  */
 
 /* { dg-do compile } */
-/* { dg-options "-O2 -fno-tree-dominator-opts -fno-tree-fre -fdisable-tree-evrp -fdump-tree-vrp1-details" } */
+/* { dg-options "-O2 -fno-tree-fre -fdisable-tree-evrp -fno-thread-jumps -fdump-tree-vrp1-details" } */
 /* { dg-additional-options "-fdisable-tree-ethread -fdisable-tree-thread1" } */
 
 int
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr21086.c b/gcc/testsuite/gcc.dg/tree-ssa/pr21086.c
index aadd53e..9b93d39 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr21086.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr21086.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdisable-tree-evrp -fdump-tree-vrp1 -fdump-tree-dce2 -fdelete-null-pointer-checks" } */
+/* { dg-options "-O2 -fno-thread-jumps -fdisable-tree-evrp -fdump-tree-vrp1 -fdump-tree-dce2 -fdelete-null-pointer-checks" } */
 
 int
 foo (int *p)
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr21090.c b/gcc/testsuite/gcc.dg/tree-ssa/pr21090.c
index 3909adb..92a8768 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr21090.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr21090.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdisable-tree-evrp -fdump-tree-vrp1 -fdelete-null-pointer-checks" } */
+/* { dg-options "-O2 -fno-thread-jumps -fdisable-tree-evrp -fdump-tree-vrp1 -fdelete-null-pointer-checks" } */
 
 int g, h;
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr21294.c b/gcc/testsuite/gcc.dg/tree-ssa/pr21294.c
index b9edabc..8c8f447 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr21294.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr21294.c
@@ -4,8 +4,7 @@
    allows us to eliminate the second "if" statement.  */
 
 /* { dg-do compile } */
-/* { dg-options "-O2 -fno-tree-dominator-opts -fdisable-tree-evrp -fdump-tree-vrp1-details" } */
-/* { dg-additional-options "-fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fno-tree-dominator-opts -fdisable-tree-evrp -fdisable-tree-ethread -fdisable-tree-threadfull1 -fdump-tree-vrp1-details" } */
 
 struct f {
   int i;
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr21417.c b/gcc/testsuite/gcc.dg/tree-ssa/pr21417.c
index b934c9c..36d9cb0 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr21417.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr21417.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdisable-tree-thread3 -fdump-tree-thread4-details" } */
+/* { dg-options "-O2 -fdump-tree-thread2-details" } */
 
 struct tree_common 
 { 
@@ -49,5 +49,5 @@ L23:
 /* We should thread the backedge to the top of the loop; ie we only
    execute the if (expr->common.code != 142) test once per loop
    iteration.  */
-/* { dg-final { scan-tree-dump-times "jump thread" 1 "thread4" } } */
+/* { dg-final { scan-tree-dump-times "jump thread" 1 "thread2" } } */
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr21458-2.c b/gcc/testsuite/gcc.dg/tree-ssa/pr21458-2.c
index f8d7353..9610570 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr21458-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr21458-2.c
@@ -16,4 +16,4 @@ foo (int a)
     }
 }
 
-/* { dg-final { scan-tree-dump-times "Predicate evaluates to: 1" 1 "evrp" } } */
+/* { dg-final { scan-tree-dump-times "Folding predicate.* to 1" 1 "evrp" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr21559.c b/gcc/testsuite/gcc.dg/tree-ssa/pr21559.c
index 43f046e..83b7c80 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr21559.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr21559.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-evrp-details -fdump-tree-vrp-thread1-details" } */
+/* { dg-options "-O2 -fdump-tree-evrp-details" } */
 
 static int blocksize = 4096;
 
@@ -34,8 +34,3 @@ void foo (void)
 
 /* First, we should simplify the bits < 0 test within the loop.  */
 /* { dg-final { scan-tree-dump-times "Simplified relational" 1 "evrp" } } */
-
-/* We used to check for 3 threaded jumps here, but they all would
-   rotate the loop.  */
-
-
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr21563.c b/gcc/testsuite/gcc.dg/tree-ssa/pr21563.c
index 72dce83..504b3cc 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr21563.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr21563.c
@@ -2,7 +2,7 @@
    Make sure VRP folds the second "if" statement.  */
 
 /* { dg-do compile } */
-/* { dg-options "-O2 -fno-tree-dominator-opts -fdisable-tree-evrp -fdump-tree-vrp1-details -fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fno-thread-jumps -fdisable-tree-evrp -fdump-tree-vrp1-details" } */
 
 int
 foo (int a)
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr23744.c b/gcc/testsuite/gcc.dg/tree-ssa/pr23744.c
index ba3fda3..f70f254 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr23744.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr23744.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fno-tree-ccp -fdisable-tree-evrp -fdump-tree-vrp1-details" } */
+/* { dg-options "-O2 -fno-tree-ccp -fdisable-tree-evrp -fdump-tree-vrp1" } */
 
 void h (void);
 
@@ -17,4 +17,5 @@ int g (int i, int j)
     return 1;
 }
 
-/* { dg-final { scan-tree-dump-times "gimple_simplified" 1 "vrp1" } } */
+/* { dg-final { scan-tree-dump-times "return 0" 1 "vrp1" } } */
+/* { dg-final { scan-tree-dump-not "return 1" "vrp1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr25382.c b/gcc/testsuite/gcc.dg/tree-ssa/pr25382.c
index d747655..8634c0a 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr25382.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr25382.c
@@ -3,7 +3,7 @@
    Check that VRP now gets ranges from BIT_AND_EXPRs.  */
 
 /* { dg-do compile } */
-/* { dg-options "-O2 -fno-tree-ccp -fdisable-tree-evrp -fdump-tree-vrp1" } */
+/* { dg-options "-O2 -fno-thread-jumps -fno-tree-ccp -fdisable-tree-evrp -fdump-tree-vrp1" } */
 
 int
 foo (int a)
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr49039.c b/gcc/testsuite/gcc.dg/tree-ssa/pr49039.c
index a2044d0..102b073 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr49039.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr49039.c
@@ -1,6 +1,6 @@
 /* PR tree-optimization/49039 */
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdisable-tree-evrp -fdump-tree-vrp1 -fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fdisable-tree-evrp -fdump-tree-vrp1 -fno-thread-jumps" } */
 
 extern void bar (void);
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr58480.c b/gcc/testsuite/gcc.dg/tree-ssa/pr58480.c
index 42898e7..f11623b 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr58480.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr58480.c
@@ -1,5 +1,5 @@
 /* { dg-do compile { target { ! keeps_null_pointer_checks } } } */
-/* { dg-options "-O2 -fdisable-tree-evrp -fdump-tree-vrp1 -fdelete-null-pointer-checks" } */
+/* { dg-options "-O2 -fno-thread-jumps -fdisable-tree-evrp -fdump-tree-vrp1 -fdelete-null-pointer-checks" } */
 
 extern void eliminate (void);
 extern void* f1 (void *a, void *b) __attribute__((nonnull));
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr59597.c b/gcc/testsuite/gcc.dg/tree-ssa/pr59597.c
index 764b3fe..0f66aae 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr59597.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr59597.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-Ofast -fdump-tree-vrp-thread1-details" } */
+/* { dg-options "-Ofast -fdisable-tree-cunrolli -fdump-tree-threadfull1-details" } */
 
 typedef unsigned short u16;
 typedef unsigned char u8;
@@ -56,7 +56,8 @@ main (int argc, char argv[])
   return crc;
 }
 
-/* None of the threads we can get in vrp-thread1 are valid.  They all
-   cross or rotate loops.  */
-/* { dg-final { scan-tree-dump-not "Registering jump thread" "vrp-thread1" } } */
-/* { dg-final { scan-tree-dump-not "joiner" "vrp-thread1" } } */
+/* We used to have no threads in vrp-thread1 because all the attempted
+   ones would cross loops.  Now we get 30+ threads before VRP because
+   of loop unrolling.  A better option is to disable unrolling and
+   test for the original 4 threads that this test was testing.  */
+/* { dg-final { scan-tree-dump-times "Registering jump thread" 4 "threadfull1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr61839_1.c b/gcc/testsuite/gcc.dg/tree-ssa/pr61839_1.c
index 0229a82..f5af7a1 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr61839_1.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr61839_1.c
@@ -1,6 +1,6 @@
 /* PR tree-optimization/61839.  */
 /* { dg-do run } */
-/* { dg-options "-O2 -fdump-tree-vrp-thread1 -fdisable-tree-evrp -fdump-tree-optimized -fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fdisable-tree-evrp -fdisable-tree-ethread -fdisable-tree-threadfull1 -fdump-tree-vrp1 -fdump-tree-optimized" } */
 /* { dg-require-effective-target int32plus } */
 
 __attribute__ ((noinline))
@@ -38,11 +38,11 @@ int main ()
 }
 
 /* Scan for c = 972195717) >> [0, 1] in function foo.  */
-/* { dg-final { scan-tree-dump-times "486097858 : 972195717" 1  "vrp-thread1" } } */
+/* { dg-final { scan-tree-dump-times "486097858 : 972195717" 1  "vrp1" } } */
 
 /* Previously we were checking for two ?: with constant PHI arguments,
    but now we collapse them into one.  */
 /* Scan for c = 972195717) >> [2, 3] in function bar.  */
-/* { dg-final { scan-tree-dump-times "243048929 : 121524464" 1  "vrp-thread1" } } */
+/* { dg-final { scan-tree-dump-times "243048929 : 121524464" 1  "vrp1" } } */
 
 /* { dg-final { scan-tree-dump-times "486097858" 0  "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr61839_3.c b/gcc/testsuite/gcc.dg/tree-ssa/pr61839_3.c
index 7be1873..bc2126f 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr61839_3.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr61839_3.c
@@ -1,6 +1,6 @@
 /* PR tree-optimization/61839.  */
 /* { dg-do run } */
-/* { dg-options "-O2 -fdump-tree-vrp-thread1 -fdump-tree-optimized -fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fdump-tree-vrp -fdump-tree-optimized -fdisable-tree-ethread -fdisable-tree-threadfull1" } */
 
 __attribute__ ((noinline))
 int foo (int a, unsigned b)
@@ -22,5 +22,5 @@ int main ()
 }
 
 /* Scan for c [12, 13] << 8 in function foo.  */
-/* { dg-final { scan-tree-dump-times "3072 : 3328" 1  "vrp-thread1" } } */
+/* { dg-final { scan-tree-dump-times "3072 : 3328" 1  "vrp1" } } */
 /* { dg-final { scan-tree-dump-times "3072" 0  "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr66752-3.c b/gcc/testsuite/gcc.dg/tree-ssa/pr66752-3.c
index ba7025a..e9bb744 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr66752-3.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr66752-3.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-thread1-details -fdump-tree-thread4" } */
+/* { dg-options "-O2 -fdump-tree-threadfull1-details -fdump-tree-thread2" } */
 
 extern int status, pt;
 extern int count;
@@ -35,7 +35,7 @@ foo (int N, int c, int b, int *a)
 /* There are 2 jump threading opportunities (which don't cross loops),
    all of which will be realized, which will eliminate testing of
    FLAG, completely.  */
-/* { dg-final { scan-tree-dump-times "Registering jump" 2 "thread1"} } */
+/* { dg-final { scan-tree-dump-times "Registering jump" 2 "threadfull1"} } */
 
 /* We used to remove references to FLAG by DCE2, but this was
    depending on early threaders threading through loop boundaries
@@ -43,4 +43,4 @@ foo (int N, int c, int b, int *a)
    run after loop optimizations , can successfully eliminate the
    references to FLAG.  Verify that ther are no references by the late
    threading passes.  */
-/* { dg-final { scan-tree-dump-not "if .flag" "thread4"} } */
+/* { dg-final { scan-tree-dump-not "if .flag" "thread2"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr68198.c b/gcc/testsuite/gcc.dg/tree-ssa/pr68198.c
index af8b7a5..1250729 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr68198.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr68198.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-thread1-details -fdisable-tree-ethread" } */
+/* { dg-options "-O2 -fdump-tree-threadfull1-details -fdisable-tree-ethread" } */
 
 extern void abort (void);
 
@@ -38,4 +38,4 @@ c_finish_omp_clauses (tree clauses)
 }
 
 /* There are 3 jump threading opportunities.  */
-/* { dg-final { scan-tree-dump-times "Registering jump" 3 "thread1"} } */
+/* { dg-final { scan-tree-dump-times "Registering jump" 3 "threadfull1"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr69270-3.c b/gcc/testsuite/gcc.dg/tree-ssa/pr69270-3.c
index 89735f6..5ffd5f7 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr69270-3.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr69270-3.c
@@ -3,7 +3,7 @@
 
 /* We're looking for a constant argument a PHI node.  There
    should only be one if we unpropagate correctly.  */
-/* { dg-final { scan-tree-dump-times ", 1" 1 "uncprop1"} } */
+/* { dg-final { scan-tree-dump-times "<1\|, 1" 1 "uncprop1"} } */
 
 typedef long unsigned int size_t;
 typedef union gimple_statement_d *gimple;
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr77445-2.c b/gcc/testsuite/gcc.dg/tree-ssa/pr77445-2.c
index f2a5e78..b3db1bc 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr77445-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr77445-2.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdisable-tree-evrp -fdump-tree-thread-details-blocks-stats" } */
+/* { dg-options "-O2 -fdisable-tree-evrp -fdump-tree-thread-details-blocks-stats -fdump-tree-threadfull1-blocks-stats -fdump-tree-threadfull2-blocks-stats" } */
 typedef enum STATES {
 	START=0,
 	INVALID,
@@ -123,8 +123,8 @@ enum STATES FMS( u8 **in , u32 *transitions) {
    aarch64 has the highest CASE_VALUES_THRESHOLD in GCC.  It's high enough
    to change decisions in switch expansion which in turn can expose new
    jump threading opportunities.  Skip the later tests on aarch64.  */
-/* { dg-final { scan-tree-dump "Jumps threaded: \[7-9\]" "thread2" } } */
+/* { dg-final { scan-tree-dump "Jumps threaded: \[7-9\]" "thread1" } } */
 /* { dg-final { scan-tree-dump-not "optimizing for size" "thread1" } } */
-/* { dg-final { scan-tree-dump-not "optimizing for size" "thread2" } } */
-/* { dg-final { scan-tree-dump-not "optimizing for size" "thread3" { target { ! aarch64*-*-* } } } } */
-/* { dg-final { scan-tree-dump-not "optimizing for size" "thread4" { target { ! aarch64*-*-* } } } } */ 
+/* { dg-final { scan-tree-dump-not "optimizing for size" "threadfull1" } } */
+/* { dg-final { scan-tree-dump-not "optimizing for size" "thread2" { target { ! aarch64*-*-* } } } } */
+/* { dg-final { scan-tree-dump-not "optimizing for size" "threadfull2" { target { ! aarch64*-*-* } } } } */ 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr77445.c b/gcc/testsuite/gcc.dg/tree-ssa/pr77445.c
index 883a63d..72bf0ec 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr77445.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr77445.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-thread3-details-blocks -fno-early-inlining -fno-tree-vrp -fno-tree-dominator-opts" } */
+/* { dg-options "-O2 -fno-early-inlining -fno-tree-vrp -fno-tree-dominator-opts -fdump-tree-thread2-details-blocks" } */
 
 static int a;
 static int b;
@@ -25,5 +25,5 @@ main (int argc)
   if (b)
     test2 ();
 }
-/* { dg-final { scan-tree-dump-times "Registering jump thread" 2 "thread3" } } */
-/* { dg-final { scan-tree-dump-not "Invalid sum" "thread3" } } */
+/* { dg-final { scan-tree-dump-times "Registering jump thread" 2 "thread2" } } */
+/* { dg-final { scan-tree-dump-not "Invalid sum" "thread2" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pta-callused.c b/gcc/testsuite/gcc.dg/tree-ssa/pta-callused.c
index aa639b4..b9a57d8 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pta-callused.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pta-callused.c
@@ -22,5 +22,5 @@ int bar (int b)
   return *foo (&q);
 }
 
-/* { dg-final { scan-tree-dump "CALLUSED\\(\[0-9\]+\\) = { NONLOCAL f.* i q }" "alias" } } */
+/* { dg-final { scan-tree-dump "CALLUSED\\(\[0-9\]+\\) = { ESCAPED NONLOCAL f.* i q }" "alias" } } */
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-1.c b/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-1.c
index c3ccb5d..c6c2b2b 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-1.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-thread1-details --param logical-op-non-short-circuit=1" } */
+/* { dg-options "-O2 -fdump-tree-threadfull1-details --param logical-op-non-short-circuit=1" } */
 
 // Copied from ssa-dom-thread-11.c
 
@@ -17,4 +17,4 @@ mark_target_live_regs (int b, int block, int bb_tick)
 /* When the first two conditionals in the first IF are true, but
    the third conditional is false, then there's a jump threading
    opportunity to bypass the second IF statement.  */
-/* { dg-final { scan-tree-dump-times "Registering.*jump thread" 1 "thread1"} } */
+/* { dg-final { scan-tree-dump-times "Registering.*jump thread" 1 "threadfull1"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-2.c b/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-2.c
index d2689b6..163d9e1 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-2.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-2.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */ 
-/* { dg-options "-O2 -fdump-tree-thread2-details -w" } */
+/* { dg-options "-O2 -fdump-tree-thread1-details -w" } */
 
 // Copied from ssa-dom-thread-12.c.
 
@@ -36,4 +36,4 @@ scan_function (gimple stmt)
    that stmt->num_ops - 3 != 0.  When that test is false, we can derive
    a value for stmt->num_ops.  That in turn allows us to thread the jump
    for the conditional at the start of the call to gimple_op.  */
-/* { dg-final { scan-tree-dump-times "Registering.*jump thread" 1 "thread2"} } */
+/* { dg-final { scan-tree-dump-times "Registering.*jump thread" 1 "thread1"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-4.c b/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-4.c
index d5aa2e8..dde43ff 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-4.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ranger-threader-4.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
-/* { dg-additional-options "-O2 -fdump-tree-vrp-details -fdump-tree-thread1-details --param logical-op-non-short-circuit=1" }  */
-/* { dg-final { scan-tree-dump-times "Registering jump" 8 "thread1" } }  */
+/* { dg-additional-options "-O2 -fdump-tree-threadfull1-details --param logical-op-non-short-circuit=1" }  */
+/* { dg-final { scan-tree-dump-times "Registering jump" 8 "threadfull1" } }  */
 
 /* Copied from ssa-thread-14.  */
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dce-9.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dce-9.c
new file mode 100644
index 0000000..e1ffa7f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dce-9.c
@@ -0,0 +1,10 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-cddce1" } */
+
+int main()
+{
+  while(1)
+    for(int i=0; i<9000000; i++){}
+}
+
+/* { dg-final { scan-tree-dump-not "if" "cddce1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-1.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-1.c
index 759d9ab..5b2ac53 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-1.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */ 
-/* { dg-options "-O2 -fno-tree-vrp -fdump-tree-dom2-details" } */
+/* { dg-options "-O2 -fno-tree-vrp -fdisable-tree-threadfull1 -fdump-tree-dom2-details" } */
 void t(void);
 void q(void);
 void q1(void);
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-11.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-11.c
index 73969bb..bcc9ccf 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-11.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-11.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-dom2-details --param logical-op-non-short-circuit=1 -fdisable-tree-thread1 -fdisable-tree-thread2 -fdisable-tree-vrp-thread1 " } */
+/* { dg-options "-O2 -fdump-tree-dom2-details --param logical-op-non-short-circuit=1 -fdisable-tree-thread1 -fdisable-tree-thread2 -fdisable-tree-threadfull1" } */
 
 static int *bb_ticks;
 extern void frob (void);
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-12.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-12.c
index bad5e0a..e68acbb 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-12.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-12.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */ 
-/* { dg-options "-O2 -fdump-tree-dom2-details -w -fdisable-tree-thread2" } */
+/* { dg-options "-O2 -fdump-tree-dom2-details -w -fdisable-tree-thread1" } */
 typedef long unsigned int size_t;
 union tree_node;
 typedef union tree_node *tree;
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-14.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-14.c
index a25fe8b..8497ed6 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-14.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-14.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-options "-O2 -fdump-tree-dom2-details -w --param logical-op-non-short-circuit=1" } */
-/* { dg-additional-options "-fdisable-tree-thread1 -fdisable-tree-ethread -fdisable-tree-thread2" } */
+/* { dg-additional-options "-fdisable-tree-thread1 -fdisable-tree-ethread -fdisable-tree-threadfull1" } */
 
 enum optab_methods
 {
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-16.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-16.c
index 1b677f4..e8555f2 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-16.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-16.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-dom2-details -w --param logical-op-non-short-circuit=1 -fdisable-tree-vrp-thread1" } */
+/* { dg-options "-O2 -fdump-tree-dom2-details -w --param logical-op-non-short-circuit=1 -fdisable-tree-threadfull1" } */
 unsigned char
 validate_subreg (unsigned int offset, unsigned int isize, unsigned int osize, int zz, int qq)
 {
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-2b.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-2b.c
index 823ada9..74a55aa 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-2b.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-2b.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */ 
-/* { dg-options "-O2 -fdump-tree-thread3-stats -fdump-tree-dom2-stats -fdisable-tree-ethread" } */
+/* { dg-options "-O2 -fdump-tree-thread2-stats -fdump-tree-dom2-stats -fdisable-tree-ethread" } */
 
 void foo();
 void bla();
@@ -26,4 +26,4 @@ void thread_latch_through_header (void)
    case.  And we want to thread through the header as well.  These
    are both caught by threading in DOM.  */
 /* { dg-final { scan-tree-dump-not "Jumps threaded" "dom2"} } */
-/* { dg-final { scan-tree-dump-times "Jumps threaded: 1" 1 "thread3"} } */
+/* { dg-final { scan-tree-dump-times "Jumps threaded: 1" 1 "thread2"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-4.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-4.c
deleted file mode 100644
index 9cd4635..0000000
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-4.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/* { dg-do compile } */ 
-/* { dg-options "-O2 -fdump-tree-vrp-thread2-details -fdump-tree-dom2-details -std=gnu89 --param logical-op-non-short-circuit=1" } */
-struct bitmap_head_def;
-typedef struct bitmap_head_def *bitmap;
-typedef const struct bitmap_head_def *const_bitmap;
-typedef unsigned long BITMAP_WORD;
-typedef struct bitmap_element_def
-{
-  struct bitmap_element_def *next;
-  unsigned int indx;
-} bitmap_element;
-
-
-
-
-
-
-
-
-
-unsigned char
-bitmap_ior_and_compl (bitmap dst, const_bitmap a, const_bitmap b,
-		      const_bitmap kill)
-{
-  unsigned char changed = 0;
-
-  bitmap_element *dst_elt;
-  const bitmap_element *a_elt, *b_elt, *kill_elt, *dst_prev;
-
-  while (a_elt || b_elt)
-    {
-      unsigned char new_element = 0;
-
-      if (b_elt)
-	while (kill_elt && kill_elt->indx < b_elt->indx)
-	  kill_elt = kill_elt->next;
-
-      if (b_elt && kill_elt && kill_elt->indx == b_elt->indx
-	  && (!a_elt || a_elt->indx >= b_elt->indx))
-	{
-	  bitmap_element tmp_elt;
-	  unsigned ix;
-
-	  BITMAP_WORD ior = 0;
-
-	      changed = bitmap_elt_ior (dst, dst_elt, dst_prev,
-					a_elt, &tmp_elt, changed);
-
-	}
-
-    }
-
-
-  return changed;
-}
-/* We used to catch 3 jump threads in vrp-thread1, but they all
-   rotated the loop, so they were disallowed.  This in turn created
-   other opportunities for the other threaders which result in the the
-   post-loop threader (vrp-thread2) catching more.  */
-/* { dg-final { scan-tree-dump-times "Registering jump thread" 5 "vrp-thread2" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-7.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-7.c
index 1da00a6..b64e71d 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-7.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-dom-thread-7.c
@@ -1,14 +1,18 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-thread1-stats -fdump-tree-thread2-stats -fdump-tree-dom2-stats -fdump-tree-thread3-stats -fdump-tree-dom3-stats -fdump-tree-vrp2-stats -fno-guess-branch-probability" } */
+/* { dg-options "-O2 -fdump-tree-dom2-stats -fdump-tree-thread2-stats -fdump-tree-dom3-stats -fno-guess-branch-probability" } */
 
-/* { dg-final { scan-tree-dump "Jumps threaded: 12"  "thread3" } } */
 /* { dg-final { scan-tree-dump-not "Jumps threaded"  "dom2" } } */
 
+/* We were previously checking for no threads in vrp-thread2, but now
+   that we have merged the post and pre threaders, we get a dozen
+   threads before VRP2.  */
+
 /* aarch64 has the highest CASE_VALUES_THRESHOLD in GCC.  It's high enough
    to change decisions in switch expansion which in turn can expose new
    jump threading opportunities.  Skip the later tests on aarch64.  */
 /* { dg-final { scan-tree-dump-not "Jumps threaded"  "dom3" { target { ! aarch64*-*-* } } } } */
-/* { dg-final { scan-tree-dump-not "Jumps threaded"  "vrp-thread2" { target { ! aarch64*-*-* } } } } */
+/* { dg-final { scan-tree-dump "Jumps threaded: 7"  "thread2" { target { ! aarch64*-*-* } } } } */
+/* { dg-final { scan-tree-dump "Jumps threaded: 18"  "thread2" { target { aarch64*-*-* } } } } */
 
 enum STATE {
   S0=0,
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-fre-97.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-fre-97.c
new file mode 100644
index 0000000..2f09c8b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-fre-97.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* ethread threading does not yet catch this but it might at some point.  */
+/* { dg-options "-O -fdump-tree-fre1-details -fno-thread-jumps" } */
+
+int foo (int b, int x)
+{
+  int i, j;
+  if (b)
+    i = x;
+  if (b)
+    j = x;
+  return j == i;
+}
+
+/* Even with different undefs we should CSE a PHI node with the
+   same controlling condition.  */
+
+/* { dg-final { scan-tree-dump "Replaced redundant PHI node" "fre1" } } */
+/* { dg-final { scan-tree-dump "return 1;" "fre1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-11.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-11.c
deleted file mode 100644
index 672a54e..0000000
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-11.c
+++ /dev/null
@@ -1,50 +0,0 @@
-/* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-vrp2-details --param logical-op-non-short-circuit=1" } */
-/* { dg-additional-options "-fdisable-tree-ethread -fdisable-tree-thread1 -fdisable-tree-thread2" } */
-/* { dg-final { scan-tree-dump-not "IRREDUCIBLE_LOOP" "vrp2" } } */
-
-void abort (void);
-typedef struct bitmap_head_def *bitmap;
-typedef const struct bitmap_head_def *const_bitmap;
-typedef struct bitmap_obstack
-{
-  struct bitmap_obstack *next;
-  unsigned int indx;
-}
-bitmap_element;
-typedef struct bitmap_head_def
-{
-  bitmap_element *first;
-}
-bitmap_head;
-static __inline__ unsigned char
-bitmap_elt_ior (bitmap dst, bitmap_element * dst_elt,
-		bitmap_element * dst_prev, const bitmap_element * a_elt,
-		const bitmap_element * b_elt)
-{
-  ((void) (!(a_elt || b_elt) ? abort (), 0 : 0));
-}
-
-unsigned char
-bitmap_ior_and_compl (bitmap dst, const_bitmap a, const_bitmap b,
-		      const_bitmap kill)
-{
-  bitmap_element *dst_elt = dst->first;
-  const bitmap_element *a_elt = a->first;
-  const bitmap_element *b_elt = b->first;
-  const bitmap_element *kill_elt = kill->first;
-  bitmap_element *dst_prev = ((void *) 0);
-  while (a_elt || b_elt)
-    {
-      if (b_elt && kill_elt && kill_elt->indx == b_elt->indx
-	  && (!a_elt || a_elt->indx >= b_elt->indx));
-      else
-	{
-	  bitmap_elt_ior (dst, dst_elt, dst_prev, a_elt, b_elt);
-	  if (a_elt && b_elt && a_elt->indx == b_elt->indx)
-	    ;
-	  else if (a_elt && (!b_elt || a_elt->indx <= b_elt->indx))
-	    a_elt = a_elt->next;
-	}
-    }
-}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-12.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-12.c
deleted file mode 100644
index 08c0b8d..0000000
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-12.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-thread3-details -fdump-tree-thread4-details -fno-finite-loops --param early-inlining-insns=14 -fno-inline-functions" } */
-/* { dg-final { scan-tree-dump "Registering jump thread" "thread3" } } */
-/* { dg-final { scan-tree-dump "Registering jump thread" "thread4" } } */
-
-typedef struct bitmap_head_def *bitmap;
-typedef const struct bitmap_head_def *const_bitmap;
-typedef struct VEC_int_base
-{
-}
-VEC_int_base;
-typedef struct VEC_int_heap
-{
-  VEC_int_base base;
-}
-VEC_int_heap;
-typedef unsigned long BITMAP_WORD;
-typedef struct bitmap_element_def
-{
-  struct bitmap_element_def *next;
-  unsigned int indx;
-}
-bitmap_element;
-typedef struct bitmap_head_def
-{
-}
-bitmap_head;
-typedef struct
-{
-  bitmap_element *elt1;
-  bitmap_element *elt2;
-  BITMAP_WORD bits;
-}
-bitmap_iterator;
-static __inline__ void
-bmp_iter_and_compl_init (bitmap_iterator * bi, const_bitmap map1,
-			 const_bitmap map2, unsigned start_bit,
-			 unsigned *bit_no)
-{
-}
-
-static __inline__ void
-bmp_iter_next (bitmap_iterator * bi, unsigned *bit_no)
-{
-}
-
-static __inline__ unsigned char
-bmp_iter_and_compl (bitmap_iterator * bi, unsigned *bit_no)
-{
-  if (bi->bits)
-    {
-      while (bi->elt2 && bi->elt2->indx < bi->elt1->indx)
-	bi->elt2 = bi->elt2->next;
-    }
-}
-
-extern int VEC_int_base_length (VEC_int_base *);
-bitmap
-compute_idf (bitmap def_blocks, bitmap_head * dfs)
-{
-  bitmap_iterator bi;
-  unsigned bb_index, i;
-  VEC_int_heap *work_stack;
-  bitmap phi_insertion_points;
-  while ((VEC_int_base_length (((work_stack) ? &(work_stack)->base : 0))) > 0)
-    {
-      for (bmp_iter_and_compl_init
-	   (&(bi), (&dfs[bb_index]), (phi_insertion_points), (0), &(i));
-	   bmp_iter_and_compl (&(bi), &(i)); bmp_iter_next (&(bi), &(i)))
-	{
-	}
-    }
-}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-14.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-14.c
index 8c5cc82..51fc065 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-14.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-14.c
@@ -1,7 +1,6 @@
 /* { dg-do compile } */
-/* { dg-additional-options "-O2 -fdump-tree-vrp-thread1-details --param logical-op-non-short-circuit=1" }  */
-/* { dg-additional-options "-fdisable-tree-thread1" } */
-/* { dg-final { scan-tree-dump-times "Threaded jump" 8 "vrp-thread1" } }  */
+/* { dg-additional-options "-O2 --param logical-op-non-short-circuit=1 -fdump-tree-threadfull1-details" }  */
+/* { dg-final { scan-tree-dump-times "Registering jump thread" 8 "threadfull1" } }  */
 
 void foo (void);
 void bar (void);
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-backedge.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-backedge.c
new file mode 100644
index 0000000..ef9a3e8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-thread-backedge.c
@@ -0,0 +1,32 @@
+// { dg-do compile }
+// { dg-options "-O2 -fdisable-tree-ethread -fdisable-tree-thread1 -fdisable-tree-thread2 -fno-tree-dominator-opts -fdump-tree-threadfull2-details" }
+
+// Test that we can thread jumps across the backedge of a loop through
+// the switch statement to a particular case.
+//
+// Just in case, we disable all the jump threaders before loop
+// optimizations to make sure we get a clean stab at this.
+
+int foo (unsigned int x, int s)
+{
+  while (s != 999)
+    {
+      switch (s)
+	{
+	case 0:
+	  if (x)
+	    s = 1;
+	  break;
+	case 1:
+	  if (x)
+	    s = 999;
+	  break;
+	default:
+	  break;
+	}
+      x++;
+    }
+  return s;
+}
+
+// { dg-final { scan-tree-dump "Registering jump thread:.*normal \\(back\\)" "threadfull2" } }
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-vrp-thread-1.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-vrp-thread-1.c
index 86d07ef..25c2d3a 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-vrp-thread-1.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-vrp-thread-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-vrp-thread1-details -fdelete-null-pointer-checks" } */
+/* { dg-options "-O2 -fdump-tree-threadfull1-details -fdelete-null-pointer-checks" } */
 /* { dg-skip-if "" keeps_null_pointer_checks } */
 
 void oof (void);
@@ -29,5 +29,5 @@ build_omp_regions_1 (basic_block bb, struct omp_region *parent,
 
 /* ARM Cortex-M defined LOGICAL_OP_NON_SHORT_CIRCUIT to false,
    so skip below test.  */
-/* { dg-final { scan-tree-dump-times "Threaded" 1 "vrp-thread1" { target { ! arm_cortex_m } } } } */
+/* { dg-final { scan-tree-dump-times "Registering jump thread" 1 "threadfull1" { target { ! arm_cortex_m } } } } */
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp02.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp02.c
index 2285c55..6e9c8df 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp02.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp02.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-vrp1 -fdelete-null-pointer-checks -fdisable-tree-evrp -fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fdump-tree-vrp1 -fdelete-null-pointer-checks -fdisable-tree-evrp -fno-thread-jumps" } */
 
 struct A
 {
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp03.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp03.c
index 1d7ea4e8..4cbaca4 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp03.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp03.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdisable-tree-evrp -fdump-tree-vrp1 -fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fdisable-tree-evrp -fdump-tree-vrp1 -fno-thread-jumps" } */
 
 struct A
 {
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp05.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp05.c
index c17cd1b..7f38e8d 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp05.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp05.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-vrp1 -fno-early-inlining -fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fdump-tree-vrp1 -fno-early-inlining -fno-thread-jumps" } */
 
 
 inline int ten()
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp06.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp06.c
index acb03c2..898477e 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp06.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp06.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdisable-tree-evrp -fdump-tree-vrp1 -fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fdisable-tree-evrp -fdump-tree-vrp1 -fno-thread-jumps" } */
 
 int baz (void);
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp07.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp07.c
index 31a5415..ec5f6cc 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp07.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp07.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fno-tree-fre -fdisable-tree-evrp -fdump-tree-vrp1-details -fdelete-null-pointer-checks -fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fno-tree-fre -fdisable-tree-evrp -fdump-tree-vrp1-details -fdelete-null-pointer-checks -fno-thread-jumps" } */
 
 int
 foo (int i, int *p)
@@ -34,6 +34,3 @@ foo (int i, int *p)
 /* { dg-final { scan-tree-dump-times "Folding predicate p_.*to 1" 1 "vrp1" } } */
 /* { dg-final { scan-tree-dump-times "Folding predicate p_.*to 0" 1 "vrp1" { target { ! keeps_null_pointer_checks } } } } */
 /* { dg-final { scan-tree-dump-times "Folding predicate p_.*to 0" 0 "vrp1" { target {   keeps_null_pointer_checks } } } } */
-
-/* { dg-final { scan-tree-dump-times "PREDICATE: p_\[0-9\]" 2 "vrp1" { target { ! keeps_null_pointer_checks } } } } */
-/* { dg-final { scan-tree-dump-times "PREDICATE: p_\[0-9\]" 1 "vrp1" { target {   keeps_null_pointer_checks } } } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp08.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp08.c
index c2da30b..c9d9023 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp08.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp08.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fno-tree-fre -fdisable-tree-evrp -fdump-tree-vrp1-details -fdelete-null-pointer-checks" } */
+/* { dg-options "-O2 -fno-tree-fre -fdisable-tree-evrp -fdump-tree-vrp1-details -fno-thread-jumps -fdelete-null-pointer-checks" } */
 
 /* Compile with -fno-tree-fre -O2 to prevent CSEing *p.  */
 int
@@ -20,5 +20,4 @@ foo (int a, int *p)
 }
 /* Target disabling -fdelete-null-pointer-checks should not fold checks */
 /* { dg-final { scan-tree-dump-times "Folding predicate p_.*to 1" 1 "vrp1" { target { ! keeps_null_pointer_checks } } } } */
-/* { dg-final { scan-tree-dump-times "PREDICATE: p_.* ne_expr 0" 1 "vrp1" { target { ! keeps_null_pointer_checks } } } } */
 /* { dg-final { scan-tree-dump-times "Folding predicate p_.*to 1" 0 "vrp1" { target {   keeps_null_pointer_checks } } } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp09.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp09.c
index fad0051..3541696 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp09.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp09.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fno-tree-fre -fdisable-tree-evrp -fdump-tree-vrp1 -std=gnu89 -fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fno-tree-fre -fdisable-tree-evrp -fdump-tree-vrp1 -std=gnu89 -fno-thread-jumps" } */
 
 foo (int *p)
 {
@@ -27,4 +27,5 @@ L78:
     }
 }
 
-/* { dg-final { scan-tree-dump-times "Folding predicate p_.. != 0B to 1" 2 "vrp1" } } */
+/* { dg-final { scan-tree-dump-times "Folding predicate p_" 2 "vrp1" } } */
+/* { dg-final { scan-tree-dump-times "!= 0B to 1" 2 "vrp1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp106.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp106.c
index f25ea9c..dc5021a 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp106.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp106.c
@@ -1,6 +1,6 @@
 /* PR tree-optimization/18046  */
-/* { dg-options "-O2 -fdump-tree-vrp-thread1-details" }  */
-/* { dg-final { scan-tree-dump-times "Threaded jump" 1 "vrp-thread1" } }  */
+/* { dg-options "-O2 -fdump-tree-ethread-details" }  */
+/* { dg-final { scan-tree-dump-times "Registering jump thread" 1 "ethread" } }  */
 /* During VRP we expect to thread the true arm of the conditional through the switch
    and to the BB that corresponds to the 7 ... 9 case label.  */
 extern void foo (void);
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp113.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp113.c
index ab8d91e..dfe4989 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp113.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp113.c
@@ -13,5 +13,3 @@ int f(int a) {
       case 7: return 19;
     }
 }
-
-/* { dg-final { scan-tree-dump "return 3;" "vrp1" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp20.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp20.c
index f9df67f..e582299 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp20.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp20.c
@@ -23,5 +23,5 @@ int g (int b) {
 	return 1;
 }
 
-/* { dg-final { scan-tree-dump "Folding predicate a_. == 0 to 0" "vrp1" } } */
-/* { dg-final { scan-tree-dump "Folding predicate b_. != 0 to 1" "vrp1" } } */
+/* { dg-final { scan-tree-dump "Folding predicate a_\.\* == 0 to 0" "vrp1" } } */
+/* { dg-final { scan-tree-dump "Folding predicate b_\.\* != 0 to 1" "vrp1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp33.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp33.c
index 88833eb..470675e 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp33.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp33.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-vrp1 -fno-tree-fre -fdisable-tree-evrp -fdisable-tree-ethread -fdisable-tree-thread1" } */
+/* { dg-options "-O2 -fdump-tree-vrp1 -fno-tree-fre -fdisable-tree-evrp -fno-thread-jumps" } */
 
 /* This is from PR14052.  */
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp55.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp55.c
index a478a69..0ef57d9 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp55.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp55.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdump-tree-vrp-thread1-blocks-vops-details -fdelete-null-pointer-checks" } */
+/* { dg-options "-O2 -fdump-tree-ethread-details -fdelete-null-pointer-checks" } */
 
 void arf (void);
 
@@ -12,6 +12,6 @@ fu (char *p, int x)
     arf ();
 }
 
-/* { dg-final { scan-tree-dump-times "Threaded jump" 1 "vrp-thread1" { target { ! keeps_null_pointer_checks } } } } */
-/* { dg-final { scan-tree-dump-times "Threaded jump" 0 "vrp-thread1" { target {   keeps_null_pointer_checks } } } } */
+/* { dg-final { scan-tree-dump-times "Registering jump thread" 1 "ethread" { target { ! keeps_null_pointer_checks } } } } */
+/* { dg-final { scan-tree-dump-times "Registering jump thread" 0 "ethread" { target {   keeps_null_pointer_checks } } } } */
 
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp92.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp92.c
index 66d74e9..9e53547 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp92.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp92.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fdisable-tree-evrp -fno-tree-fre -fdump-tree-vrp1-details -fdisable-tree-ethread" } */
+/* { dg-options "-O2 -fdisable-tree-evrp -fno-tree-fre -fdump-tree-vrp1 -fdisable-tree-ethread" } */
 
 void bar (void);
 int foo (int i, int j)
@@ -18,5 +18,6 @@ int foo (int i, int j)
   return j;
 }
 
-/* { dg-final { scan-tree-dump "res_.: int \\\[1, 1\\\]" "vrp1" } } */
-/* { dg-final { scan-tree-dump-not "Threaded" "vrp1" } } */
+/* { dg-final { scan-tree-dump-not "PHI" "vrp1" } } */
+/* { dg-final { scan-tree-dump-not "goto" "vrp1" } } */
+/* { dg-final { scan-tree-dump-not "if" "vrp1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp98-1.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp98-1.c
new file mode 100644
index 0000000..daa3f07
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp98-1.c
@@ -0,0 +1,41 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target int128 } */
+/* { dg-options "-Os -fdump-tree-evrp-details" } */
+
+#include <stdint.h>
+#include <limits.h>
+
+typedef unsigned int word __attribute__((mode(word)));
+typedef unsigned __int128 bigger_than_word;
+
+int
+foo (bigger_than_word a, word b, uint8_t c)
+{
+  /* Must fold use of t1 into use of b, as b is no wider than word_mode. */
+  const uint8_t t1 = b % UCHAR_MAX;
+
+  /* Must NOT fold use of t2 into use of a, as a is wider than word_mode. */
+  const uint8_t t2 = a % UCHAR_MAX;
+
+  /* Must fold use of t3 into use of c, as c is narrower than t3. */
+  const uint32_t t3 = (const uint32_t)(c >> 1);
+
+  uint16_t ret = 0;
+
+  if (t1 == 1)
+    ret = 20;
+  else if (t2 == 2)
+    ret = 30;
+  else if (t3 == 3)
+    ret = 40;
+  /* Th extra condition below is necessary to prevent a prior pass from
+     folding away the cast. Ignored in scan-tree-dump. */
+  else if (t3 == 4)
+    ret = 50;
+
+  return ret;
+}
+
+/* { dg-final { scan-tree-dump "Folded into: if \\(_\[0-9\]+ == 1\\)" "evrp" } } */
+/* { dg-final { scan-tree-dump-not "Folded into: if \\(_\[0-9\]+ == 2\\)" "evrp" } } */
+/* { dg-final { scan-tree-dump "Folded into: if \\(_\[0-9\]+ == 3\\)" "evrp" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vrp98.c b/gcc/testsuite/gcc.dg/tree-ssa/vrp98.c
index 982f091..78d3bba 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/vrp98.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/vrp98.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target int128 } */
-/* { dg-options "-Os -fdump-tree-vrp1-details" } */
+/* { dg-options "-Os -fdisable-tree-evrp -fdump-tree-vrp1-details" } */
 
 #include <stdint.h>
 #include <limits.h>
diff --git a/gcc/testsuite/gcc.dg/ubsan/pr81981.c b/gcc/testsuite/gcc.dg/ubsan/pr81981.c
index 8a6597c..d201efb 100644
--- a/gcc/testsuite/gcc.dg/ubsan/pr81981.c
+++ b/gcc/testsuite/gcc.dg/ubsan/pr81981.c
@@ -16,6 +16,6 @@ foo (int i)
       u[0] = i;
     }
 
-  v = u[0];		/* { dg-warning "may be used uninitialized" } */
+  v = u[0];		/* { dg-warning "may be used uninitialized" "" { xfail *-*-* } } */
   return t[0];		/* { dg-warning "may be used uninitialized" } */
 }
diff --git a/gcc/testsuite/gcc.dg/uninit-pr89230-1.c b/gcc/testsuite/gcc.dg/uninit-pr89230-1.c
index 1c07c4f..dfc87a5 100644
--- a/gcc/testsuite/gcc.dg/uninit-pr89230-1.c
+++ b/gcc/testsuite/gcc.dg/uninit-pr89230-1.c
@@ -8,7 +8,8 @@ struct S { int i, j; };
 
 int g (void)
 {
-  struct S *p = f (), *q;
+  struct S *p = f ();
+  struct S *q; // { dg-bogus "may be used uninitialized" "uninitialized" { xfail *-*-* } }
 
   if (p->i || !(q = f ()) || p->j != q->i)
    {
diff --git a/gcc/testsuite/gcc.dg/uninit-pred-7_a.c b/gcc/testsuite/gcc.dg/uninit-pred-7_a.c
index c2ba2a4..434d90a 100644
--- a/gcc/testsuite/gcc.dg/uninit-pred-7_a.c
+++ b/gcc/testsuite/gcc.dg/uninit-pred-7_a.c
@@ -23,7 +23,7 @@ int foo (int n, int l, int m, int r)
       blah(v); /* { dg-bogus "uninitialized" "bogus warning" } */
 
   if ( l )
-      blah(v); /* { dg-bogus "uninitialized" "bogus warning" } */
+      blah(v); /* { dg-bogus "uninitialized" "bogus warning" { xfail *-*-* } } */
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.dg/uninit-pred-9_b.c b/gcc/testsuite/gcc.dg/uninit-pred-9_b.c
index 8c2d28c..53c4a53 100644
--- a/gcc/testsuite/gcc.dg/uninit-pred-9_b.c
+++ b/gcc/testsuite/gcc.dg/uninit-pred-9_b.c
@@ -17,10 +17,10 @@ int foo (int n, int l, int m, int r)
 
   if (l > 100)
     if ( (n <= 9) &&  (m < 100)  && (r < 19) )
-      blah(v); /* { dg-bogus "uninitialized" "bogus warning" } */
+      blah(v); /* { dg-bogus "uninitialized" "bogus warning" { xfail powerpc64*-*-* cris-*-* } } */
 
   if ( (n <= 8) &&  (m < 99)  && (r < 19) )
-      blah(v); /* { dg-bogus "uninitialized" "pr101674" { xfail powerpc64*-*-* mmix-*-* cris-*-* } } */
+      blah(v); /* { dg-bogus "uninitialized" "pr101674" { xfail mmix-*-* } } */
 
   return 0;
 }
diff --git a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-byte.c b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-byte.c
deleted file mode 100644
index aadee7f..0000000
--- a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-byte.c
+++ /dev/null
@@ -1,12 +0,0 @@
-/* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_byte } */
-/* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
-
-#define TYPE int8_t
-#define N 16
-#include <stdint.h>
-#include "complex-add-pattern-template.c"
-
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { xfail aarch64_sve2 } } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { xfail aarch64_sve2 } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-int.c b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-int.c
index 8eba24d..cead05f 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-int.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-int.c
@@ -1,12 +1,15 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_int } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
 
 #define TYPE int32_t
 #define N 16
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { xfail aarch64_sve2 } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { target { vect_complex_add_byte } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { target { vect_complex_add_int } && ! target { aarch64_sve2 } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-long.c b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-long.c
index 9275ff1..0d21f57 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-long.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-long.c
@@ -1,12 +1,17 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_long } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+#define UNROLL
 
 #define TYPE int64_t
 #define N 16
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { xfail aarch64_sve2 } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { target { vect_complex_add_long } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { target { vect_complex_add_long } && ! target { aarch64_sve2 } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" { target { vect_long_long } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" { target { vect_long_long } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" { target { vect_long_long } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-short.c b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-short.c
index 8cbbdb8..8928386 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-short.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-short.c
@@ -1,12 +1,17 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_short } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+#define UNROLL
 
 #define TYPE int16_t
 #define N 16
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { xfail aarch64_sve2 } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { target { vect_complex_add_short } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { target { vect_complex_add_short } && ! target { aarch64_sve2 } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-byte.c b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-byte.c
deleted file mode 100644
index b753914..0000000
--- a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-byte.c
+++ /dev/null
@@ -1,12 +0,0 @@
-/* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_byte } */
-/* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
-
-#define TYPE uint8_t
-#define N 16
-#include <stdint.h>
-#include "complex-add-pattern-template.c"
-
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { xfail aarch64_sve2 } } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { xfail aarch64_sve2 } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-int.c b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-int.c
index 270c49e..4ab4b9a 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-int.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-int.c
@@ -1,12 +1,17 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_int } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+#define UNROLL
 
 #define TYPE uint32_t
 #define N 16
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { xfail aarch64_sve2 } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { target { vect_complex_add_int } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { target { vect_complex_add_int } && ! target { aarch64_sve2 } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-long.c b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-long.c
index 88144e5..38aa9c0 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-long.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-long.c
@@ -1,12 +1,16 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_long } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+#define UNROLL
 
 #define TYPE uint64_t
 #define N 16
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { xfail aarch64_sve2 } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { target { vect_complex_add_long } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { target { vect_complex_add_long } && ! target { aarch64_sve2 } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" { target { vect_long_long } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" { target { vect_long_long } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-short.c b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-short.c
index 445af39..8846c98 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-short.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/bb-slp-complex-add-pattern-unsigned-short.c
@@ -1,12 +1,17 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_short } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+#define UNROLL
 
 #define TYPE uint16_t
 #define N 16
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { xfail aarch64_sve2 } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { target { vect_complex_add_short } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { target { vect_complex_add_short } && ! target { aarch64_sve2 } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/complex-add-pattern-template.c b/gcc/testsuite/gcc.dg/vect/complex/complex-add-pattern-template.c
index a99a929..658af29 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/complex-add-pattern-template.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/complex-add-pattern-template.c
@@ -1,5 +1,8 @@
 void add90 (TYPE a[restrict N], TYPE b[restrict N], TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i+=2)
     {
       c[i] = a[i] - b[i+1];
@@ -11,6 +14,9 @@ void add90 (TYPE a[restrict N], TYPE b[restrict N], TYPE c[restrict N])
 
 void add270 (TYPE a[restrict N], TYPE b[restrict N], TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i+=2)
     {
       c[i] = a[i] + b[i+1];
@@ -22,6 +28,9 @@ void add270 (TYPE a[restrict N], TYPE b[restrict N], TYPE c[restrict N])
 
 void addMixed (TYPE a[restrict N], TYPE b[restrict N], TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i+=4)
     {
       c[i] = a[i] - b[i+1];
@@ -34,6 +43,9 @@ void addMixed (TYPE a[restrict N], TYPE b[restrict N], TYPE c[restrict N])
 void add90HandUnrolled (TYPE a[restrict N], TYPE b[restrict N],
 			TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < (N /2); i+=4)
     {
       c[i] = a[i] - b[i+1];
@@ -48,6 +60,9 @@ void add90HandUnrolled (TYPE a[restrict N], TYPE b[restrict N],
 void add90Hybrid (TYPE a[restrict N], TYPE b[restrict N], TYPE c[restrict N],
 		  TYPE d[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i+=2)
     {
       c[i] = a[i] - b[i+1];
@@ -57,4 +72,4 @@ void add90Hybrid (TYPE a[restrict N], TYPE b[restrict N], TYPE c[restrict N],
     }
 }
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 2 "vect" } } */
\ No newline at end of file
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 2 "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/complex-add-template.c b/gcc/testsuite/gcc.dg/vect/complex/complex-add-template.c
index 32c81e6..f37ab98 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/complex-add-template.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/complex-add-template.c
@@ -3,6 +3,9 @@
 void add0 (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 	   _Complex TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i++)
     c[i] = a[i] + b[i];
 }
@@ -10,6 +13,9 @@ void add0 (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 void add90snd (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 	       _Complex TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i++)
     c[i] = a[i] + (b[i] * I);
 }
@@ -19,6 +25,9 @@ void add90snd (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 void add180snd (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 	        _Complex TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i++)
     c[i] = a[i] + (b[i] * I * I);
 }
@@ -26,6 +35,9 @@ void add180snd (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 void add270snd (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 	        _Complex TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i++)
     c[i] = a[i] + (b[i] * I * I * I);
 }
@@ -35,6 +47,9 @@ void add270snd (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 void add90fst (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 	       _Complex TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i++)
     c[i] = (a[i] * I) + b[i];
 }
@@ -44,6 +59,9 @@ void add90fst (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 void add180fst (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 	        _Complex TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i++)
     c[i] = (a[i] * I * I) + b[i];
 }
@@ -51,6 +69,9 @@ void add180fst (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 void add270fst (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 	        _Complex TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i++)
     c[i] = (a[i] * I * I * I) + b[i];
 }
@@ -60,6 +81,9 @@ void add270fst (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 void addconjfst (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 		 _Complex TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i++)
     c[i] = ~a[i] + b[i];
 }
@@ -67,6 +91,9 @@ void addconjfst (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 void addconjsnd (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 		 _Complex TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i++)
     c[i] = a[i] + ~b[i];
 }
@@ -74,6 +101,9 @@ void addconjsnd (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 void addconjboth (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
 		  _Complex TYPE c[restrict N])
 {
+#if defined (UNROLL)
+#pragma GCC unroll 16
+#endif
   for (int i=0; i < N; i++)
     c[i] = ~a[i] + ~b[i];
 }
diff --git a/gcc/testsuite/gcc.dg/vect/complex/complex-operations-run.c b/gcc/testsuite/gcc.dg/vect/complex/complex-operations-run.c
index a0348a7..14ac512 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/complex-operations-run.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/complex-operations-run.c
@@ -1,7 +1,7 @@
 /* { dg-do run } */
 /* { dg-require-effective-target vect_complex_add_double } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
 
 #include <stdio.h>
 #include <complex.h>
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-double.c
index 7bbb61a..2e611b7 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-double.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-double.c
@@ -1,12 +1,16 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_double } */
+/* { dg-require-effective-target vect_double } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
+
+#define UNROLL
 
 #define TYPE double
 #define N 16
 #include "complex-add-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { target { vect_complex_add_double } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { target { vect_complex_add_double } } } } */
 
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-float.c
index cf99f1d..1e63a5f 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-float.c
@@ -1,11 +1,15 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_float } */
+/* { dg-require-effective-target vect_float } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
+
+#define UNROLL
 
 #define TYPE float
 #define N 16
 #include "complex-add-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { target { vect_complex_add_float } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { target { vect_complex_add_float } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-half-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-half-float.c
index 9f535dd..0ae49c1 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-half-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-half-float.c
@@ -1,7 +1,8 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_half } */
+/* { dg-require-effective-target float16 } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
 /* { dg-add-options arm_v8_3a_fp16_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE _Float16
 #define N 16
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-double.c
index e121113..3f5619b 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-double.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-double.c
@@ -1,11 +1,16 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_double } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
+/* { dg-require-effective-target vect_double } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+#define UNROLL
 
 #define TYPE double
 #define N 16
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { target { vect_complex_add_double } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { target { vect_complex_add_double } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-float.c
index 8565833..a961a85 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-float.c
@@ -1,11 +1,15 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_float } */
+/* { dg-require-effective-target vect_float } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
+
+#define UNROLL
 
 #define TYPE float
 #define N 16
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { target { vect_complex_add_float } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { target { vect_complex_add_float } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-half-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-half-float.c
index 857ee9d..885fd97 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-half-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-add-pattern-half-float.c
@@ -1,11 +1,16 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_half } */
+/* { dg-require-effective-target float16 } */
 /* { dg-add-options arm_v8_3a_fp16_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
+
+#define UNROLL
 
 #define TYPE _Float16
 #define N 16
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { xfail arm*-*-* } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "slp1" { target { vect_complex_add_half } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "slp1" { target { vect_complex_add_half } && ! target { arm*-*-* } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-double.c
index d9d13c2..462063a 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-double.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-double.c
@@ -1,9 +1,10 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_double } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE double
 #define N 16
 #include "complex-mla-template.c"
 
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA_CONJ" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA" "slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-float.c
index ac680cb..a88adc8 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-float.c
@@ -1,8 +1,10 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_float } */
-/* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
+/* { dg-add-options arm_v8_3a_fp16_complex_neon } */
 
 #define TYPE float
 #define N 16
 #include "complex-mla-template.c"
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA_CONJ" "slp1" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA" "slp1" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-half-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-half-float.c
index d0a48d0..5100920 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-half-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mla-half-float.c
@@ -1,9 +1,12 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_half } */
+/* { dg-require-effective-target float16 } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
 /* { dg-add-options arm_v8_3a_fp16_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE _Float16
 #define N 16
 #include "complex-mla-template.c"
 
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA_CONJ" "slp1" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA" "slp1"  { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-double.c
index d9d13c2..a434fd1 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-double.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-double.c
@@ -1,9 +1,12 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_double } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE double
 #define N 16
-#include "complex-mla-template.c"
+#include "complex-mls-template.c"
 
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS_CONJ" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS" "slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-float.c
index ac680cb..b7ccbbd 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-float.c
@@ -1,8 +1,11 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_float } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE float
 #define N 16
-#include "complex-mla-template.c"
+#include "complex-mls-template.c"
+
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS_CONJ" "slp1"  { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS" "slp1"  { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-half-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-half-float.c
index d0a48d0..217401b 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-half-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mls-half-float.c
@@ -1,9 +1,12 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_half } */
+/* { dg-require-effective-target float16 } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
 /* { dg-add-options arm_v8_3a_fp16_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE _Float16
 #define N 16
-#include "complex-mla-template.c"
+#include "complex-mls-template.c"
 
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS_CONJ" "slp1"  { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS" "slp1"  { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-double.c
index ab8313f..f7e9386 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-double.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-double.c
@@ -1,9 +1,10 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_double } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE double
 #define N 16
 #include "complex-mul-template.c"
 
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL_CONJ" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL" slp1" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-float.c
index 49bf961..0dc9c52 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-float.c
@@ -1,8 +1,11 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_float } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE float
 #define N 16
 #include "complex-mul-template.c"
+
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL_CONJ" "slp1"  { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL" "slp1"  { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-half-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-half-float.c
index f5c23fb..309b168 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-half-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-bb-slp-complex-mul-half-float.c
@@ -1,9 +1,12 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_half } */
+/* { dg-require-effective-target float16 } */
+/* { dg-additional-options "-fno-tree-loop-vectorize" } */
 /* { dg-add-options arm_v8_3a_fp16_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE _Float16
 #define N 16
 #include "complex-mul-template.c"
 
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL_CONJ" "slp1"  { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL" "slp1"  { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-double.c
index 0d4d3ce..f935405 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-double.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-double.c
@@ -1,11 +1,12 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_double } */
+/* { dg-require-effective-target vect_double } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE double
 #define N 200
 #include "complex-add-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 2 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 2 "vect" } } */
\ No newline at end of file
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 2 "vect" { target { vect_complex_add_double } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 2 "vect"  { target { vect_complex_add_double } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-float.c
index b986696..71f391d 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-float.c
@@ -1,11 +1,12 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_float } */
+/* { dg-require-effective-target vect_float } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE float
 #define N 200
 #include "complex-add-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 2 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 2 "vect" } } */
\ No newline at end of file
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 2 "vect" { target { vect_complex_add_float } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 2 "vect" { target { vect_complex_add_float } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-half-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-half-float.c
index c3dca57..e5b826f 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-half-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-half-float.c
@@ -1,11 +1,10 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_half } */
+/* { dg-require-effective-target float16 } */
 /* { dg-add-options arm_v8_3a_fp16_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE _Float16
 #define N 200
 #include "complex-add-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 2 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 2 "vect" } } */
\ No newline at end of file
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 2 "vect" { target { vect_complex_add_half } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 2 "vect" { target { vect_complex_add_half } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-double.c
index df4d3f6..5b70d83 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-double.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-double.c
@@ -1,11 +1,13 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_double } */
+/* { dg-require-effective-target vect_double } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE double
 #define N 200
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 4 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 4 "vect" { target { vect_complex_add_double } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" { target { vect_complex_add_double } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-float.c
index 6df5c6d..3ef0564 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-float.c
@@ -1,11 +1,13 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_float } */
+/* { dg-require-effective-target vect_float } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE float
 #define N 200
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 4 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 4 "vect" { target { vect_complex_add_float } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" { target { vect_complex_add_float } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-half-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-half-float.c
index a725112..06a9216 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-half-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-add-pattern-half-float.c
@@ -1,12 +1,16 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_half } */
+/* { dg-require-effective-target vect_float } */
+/* { dg-require-effective-target float16 } */
 /* { dg-add-options arm_v8_3a_fp16_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE _Float16
 #define N 200
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 4 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 4 "vect" { target { vect_complex_add_half } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" { target { vect_complex_add_half } } } } */
 
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "slp1" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-double.c
index c85ff07..3ec46e5 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-double.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-double.c
@@ -1,8 +1,9 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_double } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE double
 #define N 200
 #include "complex-mla-template.c"
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-float.c
index a17b91b..af59323 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-float.c
@@ -1,8 +1,9 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target vect_complex_add_float } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE float
 #define N 200
 #include "complex-mla-template.c"
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-half-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-half-float.c
index fa81985..34146f3 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-half-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mla-half-float.c
@@ -1,8 +1,8 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_half } */
 /* { dg-add-options arm_v8_3a_fp16_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE _Float16
 #define N 200
 #include "complex-mla-template.c"
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-double.c
index c85ff07..0982a2b 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-double.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-double.c
@@ -1,8 +1,10 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_double } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE double
 #define N 200
-#include "complex-mla-template.c"
+#include "complex-mls-template.c"
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-float.c
index a17b91b..a069533 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-float.c
@@ -1,8 +1,10 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_float } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE float
 #define N 200
-#include "complex-mla-template.c"
+#include "complex-mls-template.c"
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMA" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-half-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-half-float.c
index fa81985..89ac54c 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-half-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mls-half-float.c
@@ -1,8 +1,9 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_half } */
 /* { dg-add-options arm_v8_3a_fp16_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE _Float16
 #define N 200
-#include "complex-mla-template.c"
+#include "complex-mls-template.c"
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_FMS" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-double.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-double.c
index 77c01a8..56a8ea4 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-double.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-double.c
@@ -1,8 +1,8 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_double } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE double
 #define N 200
 #include "complex-mul-template.c"
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-float.c
index a8b44f2..969416d 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-float.c
@@ -1,8 +1,8 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_float } */
 /* { dg-add-options arm_v8_3a_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE float
 #define N 200
 #include "complex-mul-template.c"
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-half-float.c b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-half-float.c
index d57df82..da1b921 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-half-float.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/fast-math-complex-mul-half-float.c
@@ -1,8 +1,8 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_half } */
 /* { dg-add-options arm_v8_3a_fp16_complex_neon } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
 
 #define TYPE _Float16
 #define N 200
 #include "complex-mul-template.c"
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL_CONJ" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_MUL" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-byte.c b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-byte.c
index 438d219..bf7c135 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-byte.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-byte.c
@@ -1,12 +1,14 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_byte } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
 
 #define TYPE int8_t
 #define N 200
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" { target { vect_complex_add_byte } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" { target { vect_complex_add_byte } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-int.c b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-int.c
index 04c27dc..2fd2a9f 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-int.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-int.c
@@ -1,12 +1,14 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_int } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
 
 #define TYPE int32_t
 #define N 200
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" { target { vect_complex_add_int } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" { target { vect_complex_add_int } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-long.c b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-long.c
index 27988ad..7097715 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-long.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-long.c
@@ -1,12 +1,15 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_long } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+#define UNROLL
 
 #define TYPE int64_t
 #define N 200
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" { target { vect_complex_add_long } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" { target { vect_complex_add_long } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" { target { vect_long_long } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" { target { vect_long_long } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-short.c b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-short.c
index 88d225d..8cbb8ac 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-short.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-short.c
@@ -1,12 +1,14 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_short } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
 
 #define TYPE int16_t
 #define N 200
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" { target { vect_complex_add_short } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" { target { vect_complex_add_short } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-byte.c b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-byte.c
index 59109c0..9477f0d 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-byte.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-byte.c
@@ -1,12 +1,14 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_byte } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
 
 #define TYPE uint8_t
 #define N 200
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" { target { vect_complex_add_byte } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" { target { vect_complex_add_byte } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-int.c b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-int.c
index 44d9635..41f6004 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-int.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-int.c
@@ -1,12 +1,14 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_int } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
 
 #define TYPE uint32_t
 #define N 200
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" { target { vect_complex_add_int } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" { target { vect_complex_add_int } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-long.c b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-long.c
index 6671457..7708ac4 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-long.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-long.c
@@ -1,12 +1,15 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_long } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
+
+#define UNROLL
 
 #define TYPE uint64_t
 #define N 200
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" { target { vect_complex_add_long } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" { target { vect_complex_add_long } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" { target { vect_long_long } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" { target { vect_long_long } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-short.c b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-short.c
index 841adf8..6260be4 100644
--- a/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-short.c
+++ b/gcc/testsuite/gcc.dg/vect/complex/vect-complex-add-pattern-unsigned-short.c
@@ -1,12 +1,14 @@
 /* { dg-do compile } */
-/* { dg-require-effective-target vect_complex_add_short } */
 /* { dg-require-effective-target stdint_types } */
-/* { dg-add-options arm_v8_1m_mve_fp } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-add-options arm_v8_3a_complex_neon } */
 
 #define TYPE uint16_t
 #define N 200
 #include <stdint.h>
 #include "complex-add-pattern-template.c"
 
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" } } */
-/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT90" 1 "vect" { target { vect_complex_add_short } } } } */
+/* { dg-final { scan-tree-dump-times "stmt.*COMPLEX_ADD_ROT270" 1 "vect" { target { vect_complex_add_short } } } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT270" "vect" } } */
+/* { dg-final { scan-tree-dump "Found COMPLEX_ADD_ROT90" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr101145_1.c b/gcc/testsuite/gcc.dg/vect/pr101145_1.c
index 9332b2c..24a9da6 100644
--- a/gcc/testsuite/gcc.dg/vect/pr101145_1.c
+++ b/gcc/testsuite/gcc.dg/vect/pr101145_1.c
@@ -1,4 +1,4 @@
-/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_char_add } */
 /* { dg-additional-options "-O3" } */
 #define TYPE signed char
 #define MIN -128
diff --git a/gcc/testsuite/gcc.dg/vect/pr101145_2.c b/gcc/testsuite/gcc.dg/vect/pr101145_2.c
index fa2c6be..9091f60 100644
--- a/gcc/testsuite/gcc.dg/vect/pr101145_2.c
+++ b/gcc/testsuite/gcc.dg/vect/pr101145_2.c
@@ -1,4 +1,4 @@
-/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_char_add } */
 /* { dg-additional-options "-O3" } */
 #define TYPE unsigned char
 #define MIN 0
diff --git a/gcc/testsuite/gcc.dg/vect/pr101145_3.c b/gcc/testsuite/gcc.dg/vect/pr101145_3.c
index 9f43c82..001e5f3 100644
--- a/gcc/testsuite/gcc.dg/vect/pr101145_3.c
+++ b/gcc/testsuite/gcc.dg/vect/pr101145_3.c
@@ -1,4 +1,4 @@
-/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_long_long } */
 /* { dg-additional-options "-O3" } */
 #define TYPE int *
 #define MIN ((TYPE)0)
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/license.txt b/gcc/testsuite/gcc.dg/vect/tsvc/license.txt
new file mode 100644
index 0000000..e16b36c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/license.txt
@@ -0,0 +1,29 @@
+Copyright (c) 2011 University of Illinois at Urbana-Champaign.  All rights reserved.
+
+Developed by: Polaris Research Group
+              University of Illinois at Urbana-Champaign
+              http://polaris.cs.uiuc.edu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal with the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+  1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimers.
+  2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimers in the
+     documentation and/or other materials provided with the distribution.
+  3. Neither the names of Polaris Research Group, University of Illinois at
+     Urbana-Champaign, nor the names of its contributors may be used to endorse
+     or promote products derived from this Software without specific prior
+     written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+WITH THE SOFTWARE.
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/tsvc.h b/gcc/testsuite/gcc.dg/vect/tsvc/tsvc.h
new file mode 100644
index 0000000..665ca74
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/tsvc.h
@@ -0,0 +1,1436 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+#define iterations 10000
+#define LEN_1D 32000
+#define LEN_2D 256
+#define ARRAY_ALIGNMENT 64
+
+#pragma GCC push_options
+#pragma GCC optimize "-fno-tree-vectorize"
+
+#include <stdio.h>
+#include <stdlib.h>
+#ifndef __APPLE__
+#include <malloc.h>
+#endif
+#include <string.h>
+#include <math.h>
+
+typedef float real_t;
+#define ABS fabsf
+
+int dummy(real_t[LEN_1D], real_t[LEN_1D], real_t[LEN_1D], real_t[LEN_1D],
+          real_t[LEN_1D], real_t[LEN_2D][LEN_2D], real_t[LEN_2D][LEN_2D],
+          real_t[LEN_2D][LEN_2D], real_t);
+
+__attribute__((aligned(ARRAY_ALIGNMENT)))
+real_t flat_2d_array[LEN_2D * LEN_2D];
+__attribute__((aligned(ARRAY_ALIGNMENT))) real_t x[LEN_1D];
+__attribute__((aligned(ARRAY_ALIGNMENT))) real_t a[LEN_1D], b[LEN_1D],
+    c[LEN_1D], d[LEN_1D], e[LEN_1D], aa[LEN_2D][LEN_2D], bb[LEN_2D][LEN_2D],
+    cc[LEN_2D][LEN_2D], tt[LEN_2D][LEN_2D];
+__attribute__((aligned(ARRAY_ALIGNMENT))) int indx[LEN_1D];
+
+real_t* __restrict__ xx;
+real_t* yy;
+
+void set_1d_array(real_t * arr, int length, real_t value, int stride);
+void set_2d_array(real_t arr[LEN_2D][LEN_2D], real_t value, int stride);
+
+struct args_t {
+    struct timeval t1;
+    struct timeval t2;
+    void * __restrict__ arg_info;
+};
+
+enum {SET1D_RECIP_IDX = -1, SET1D_RECIP_IDX_SQ = -2};
+
+real_t sum1d(real_t arr[LEN_1D]);
+real_t sum2d(real_t arr[LEN_2D][LEN_2D]);
+
+real_t sum_x();
+real_t sum_a();
+real_t sum_b();
+real_t sum_c();
+real_t sum_e();
+
+real_t sum_half_xx();
+
+real_t sum_a_aa();
+
+real_t sum_aa();
+real_t sum_bb();
+real_t sum_cc();
+real_t sum_xx();
+
+real_t sum_aa_bb();
+
+real_t sum_flat_2d_array();
+
+real_t sum1d(real_t arr[LEN_1D]){
+    real_t ret = 0.;
+    for (int i = 0; i < LEN_1D; i++)
+        ret += arr[i];
+    return ret;
+}
+
+real_t sum2d(real_t arr[LEN_2D][LEN_2D]){
+    real_t sum = 0.;
+    for (int i = 0; i < LEN_2D; i++){
+        for (int j = 0; j < LEN_2D; j++){
+            sum += arr[i][j];
+        }
+    }
+
+    return sum;
+}
+
+real_t sum_x()
+{
+    return sum1d(x);
+}
+
+real_t sum_xx()
+{
+    return sum1d(xx);
+}
+
+real_t sum_a()
+{
+    return sum1d(a);
+}
+
+real_t sum_b()
+{
+    return sum1d(b);
+}
+
+real_t sum_a_aa()
+{
+    return sum1d(a) + sum2d(aa);
+}
+
+real_t sum_c()
+{
+    return sum1d(c);
+}
+
+real_t sum_e()
+{
+    return sum1d(e);
+}
+
+real_t sum_aa()
+{
+    return sum2d(aa);
+}
+
+real_t sum_bb()
+{
+    return sum2d(bb);
+}
+
+real_t sum_aa_bb()
+{
+    return sum2d(aa) + sum2d(bb);
+}
+
+real_t sum_cc()
+{
+    return sum2d(cc);
+}
+
+real_t sum_half_xx()
+{
+    real_t temp = 00;
+
+    for (int i = 0; i < LEN_1D/2; i++){
+        temp += xx[i];
+    }
+
+    return temp;
+}
+
+real_t sum_flat_2d_array()
+{
+    real_t sum = 0.;
+
+    for (int i = 0; i < LEN_2D*LEN_2D; i++){
+        sum += flat_2d_array[i];
+    }
+
+    return sum;
+}
+
+
+void set_1d_array(real_t * arr, int length, real_t value, int stride)
+{
+    if (stride == SET1D_RECIP_IDX) {
+        for (int i = 0; i < length; i++) {
+            arr[i] = 1. / (real_t) (i+1);
+        }
+    } else if (stride == SET1D_RECIP_IDX_SQ) {
+        for (int i = 0; i < length; i++) {
+            arr[i] = 1. / (real_t) ((i+1) * (i+1));
+        }
+    } else {
+        for (int i = 0; i < length; i += stride) {
+            arr[i] = value;
+        }
+    }
+}
+
+void set_2d_array(real_t arr[LEN_2D][LEN_2D], real_t value, int stride)
+{
+    for (int i = 0; i < LEN_2D; i++) {
+        set_1d_array(arr[i], LEN_2D, value, stride);
+    }
+}
+
+void init(int** ip, real_t* s1, real_t* s2){
+#if !defined (__APPLE__) && !defined (_AIX)
+    xx = (real_t*) memalign(ARRAY_ALIGNMENT, LEN_1D*sizeof(real_t));
+    *ip = (int *) memalign(ARRAY_ALIGNMENT, LEN_1D*sizeof(real_t));
+#else
+# if defined (__APPLE__) \
+    && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1060
+    /* We have no aligned allocator, but malloc is guaranteed to return
+       alignment suitable for the largest vector item.  */
+    xx = (real_t*) malloc (LEN_1D*sizeof(real_t));
+    *ip = (int *) malloc (LEN_1D*sizeof(real_t));
+# else
+    posix_memalign ((void*)&xx, ARRAY_ALIGNMENT, LEN_1D*sizeof(real_t));
+    posix_memalign ((void*)ip, ARRAY_ALIGNMENT, LEN_1D*sizeof(real_t));
+# endif
+#endif    
+
+    for (int i = 0; i < LEN_1D; i = i+5){
+        (*ip)[i]   = (i+4);
+        (*ip)[i+1] = (i+2);
+        (*ip)[i+2] = (i);
+        (*ip)[i+3] = (i+3);
+        (*ip)[i+4] = (i+1);
+    }
+
+    set_1d_array(a, LEN_1D, 1.,1);
+    set_1d_array(b, LEN_1D, 1.,1);
+    set_1d_array(c, LEN_1D, 1.,1);
+    set_1d_array(d, LEN_1D, 1.,1);
+    set_1d_array(e, LEN_1D, 1.,1);
+    set_1d_array(x, LEN_1D, 1.,1);
+    set_2d_array(aa, 0.,SET1D_RECIP_IDX);
+    set_2d_array(bb, 0.,SET1D_RECIP_IDX);
+    set_2d_array(cc, 0.,SET1D_RECIP_IDX);
+
+    for (int i = 0; i < LEN_1D; i++) {
+        indx[i] = (i+1) % 4+1;
+    }
+
+    *s1 = 1.0;
+    *s2 = 2.0;
+}
+
+int initialise_arrays(const char* name)
+{
+    real_t any=0.;
+    real_t zero=0.;
+    real_t half=.5;
+    real_t one=1.;
+    real_t two=2.;
+    real_t small = .000001;
+    int unit =1;
+    int frac = SET1D_RECIP_IDX;
+    int frac2 = SET1D_RECIP_IDX_SQ;
+
+    if    (!strcmp(name, "s000")) {
+      for (int i = 0; i < LEN_1D; i++) {
+            a[i] = 1+i;
+            b[i] = 2+i;
+            c[i] = 3+i;
+            d[i] = 4+i;
+            e[i] = 5+i;
+          }
+    } else if (!strcmp(name, "s111")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+        set_1d_array(c, LEN_1D, any,frac2);
+        set_1d_array(d, LEN_1D, any,frac2);
+        set_1d_array(e, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s112")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s113")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s114")) {
+        set_2d_array(aa, any,frac);
+        set_2d_array(bb, any,frac2);
+    } else if (!strcmp(name, "s115")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_2d_array(aa,small,unit);
+        set_2d_array(bb,small,unit);
+        set_2d_array(cc,small,unit);
+    } else if (!strcmp(name, "s116")) {
+        set_1d_array(a, LEN_1D, one,unit);
+    } else if (!strcmp(name, "s118")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_2d_array(bb,small,unit);
+    } else if (!strcmp(name, "s119")) {
+        set_2d_array(aa, one,unit);
+        set_2d_array(bb, any,frac2);
+    } else if (!strcmp(name, "s121")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s122")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s123")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s124")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s125")) {
+        set_1d_array(flat_2d_array, LEN_2D*LEN_2D,zero,unit);
+        set_2d_array(aa, one,unit);
+        set_2d_array(bb,half,unit);
+        set_2d_array(cc, two,unit);
+    } else if (!strcmp(name, "s126")) {
+        set_2d_array(bb, one,unit);
+        set_1d_array( flat_2d_array, LEN_2D*LEN_2D,any,frac);
+        set_2d_array(cc, any,frac);
+    } else if (!strcmp(name, "s127")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s128")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, two,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, one,unit);
+    } else if (!strcmp(name, "s131")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s132")) {
+        set_2d_array(aa, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s141")) {
+        set_1d_array( flat_2d_array, LEN_2D*LEN_2D, one,unit);
+        set_2d_array(bb, any,frac2);
+    } else if (!strcmp(name, "s151")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s152")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D,zero,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s161")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array( &b[0], LEN_1D/2, one,2);
+        set_1d_array( &b[1], LEN_1D/2,-one,2);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s162")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s171")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s172")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s173")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s174")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s175")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s176")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s211")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s212")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s221")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s222")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+    } else if (!strcmp(name, "s231")) {
+        set_2d_array(aa, one,unit);
+        set_2d_array(bb, any,frac2);
+    } else if (!strcmp(name, "s232")) {
+        set_2d_array(aa, one,unit);
+        set_2d_array(bb,zero,unit);
+    } else if (!strcmp(name, "s233")) {
+        set_2d_array(aa, any,frac);
+        set_2d_array(bb, any,frac);
+        set_2d_array(cc, any,frac);
+    } else if (!strcmp(name, "s234")) {
+        set_2d_array(aa, one,unit);
+        set_2d_array(bb, any,frac);
+        set_2d_array(cc, any,frac);
+    } else if (!strcmp(name, "s235")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_2d_array(aa, one,unit);
+        set_2d_array(bb, any, frac2);
+    } else if (!strcmp(name, "s241")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, one,unit);
+    } else if (!strcmp(name, "s242")) {
+        set_1d_array(a, LEN_1D,small,unit);
+        set_1d_array(b, LEN_1D,small,unit);
+        set_1d_array(c, LEN_1D,small,unit);
+        set_1d_array(d, LEN_1D,small,unit);
+    } else if (!strcmp(name, "s243")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s244")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D,small,unit);
+        set_1d_array(d, LEN_1D,small,unit);
+    } else if (!strcmp(name, "s251")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s252")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+    } else if (!strcmp(name, "s253")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D,small,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s254")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+    } else if (!strcmp(name, "s255")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+    } else if (!strcmp(name, "s256")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, two,frac);
+        set_2d_array(aa, two,unit);
+        set_2d_array(bb, one,unit);
+    } else if (!strcmp(name, "s257")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_2d_array(aa, two,unit);
+        set_2d_array(bb, one,unit);
+    } else if (!strcmp(name, "s258")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        set_1d_array(b, LEN_1D,zero,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D,zero,unit);
+        set_2d_array(aa, any,frac);
+    } else if (!strcmp(name, "s261")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+        set_1d_array(c, LEN_1D, any,frac2);
+        set_1d_array(d, LEN_1D, one,unit);
+    } else if (!strcmp(name, "s271")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s272")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, two,unit);
+    } else if (!strcmp(name, "s273")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D,small,unit);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s274")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s275")) {
+        set_2d_array(aa, one,unit);
+        set_2d_array(bb,small,unit);
+        set_2d_array(cc,small,unit);
+    } else if (!strcmp(name, "s276")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s277")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array( b, LEN_1D/2, one,unit);
+        set_1d_array( &b[LEN_1D/2], LEN_1D/2,-one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s278")) {
+        set_1d_array( a, LEN_1D/2,-one,unit);
+        set_1d_array( &a[LEN_1D/2], LEN_1D/2,one,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s279")) {
+        set_1d_array( a, LEN_1D/2,-one,unit);
+        set_1d_array( &a[LEN_1D/2], LEN_1D/2,one,unit);
+//        set_1d_array(a, LEN_1D, -one,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s2710")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s2711")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s2712")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s281")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+    } else if (!strcmp(name, "1s281")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, one,unit);
+        set_1d_array(e, LEN_1D, one,unit);
+        set_1d_array(x, LEN_1D, one,unit);
+    } else if (!strcmp(name, "s291")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+    } else if (!strcmp(name, "s292")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+    } else if (!strcmp(name, "s293")) {
+        set_1d_array(a, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s2101")) {
+        set_2d_array(aa, one,unit);
+        set_2d_array(bb, any,frac);
+        set_2d_array(cc, any,frac);
+    } else if (!strcmp(name, "s2102")) {
+        set_2d_array(aa,zero,unit);
+    } else if (!strcmp(name, "s2111")) {
+        set_2d_array(aa, small,unit);
+    } else if (!strcmp(name, "s311")) {
+        set_1d_array(a, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s312")) {
+        set_1d_array(a, LEN_1D,1.000001,unit);
+    } else if (!strcmp(name, "s313")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        set_1d_array(b, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s314")) {
+        set_1d_array(a, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s315")) {
+        set_1d_array(a, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s316")) {
+        set_1d_array(a, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s317")) {
+    } else if (!strcmp(name, "s318")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        a[LEN_1D-1] = -two;
+    } else if (!strcmp(name, "s319")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D,zero,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s3110")) {
+        set_2d_array(aa, any,frac);
+        aa[LEN_2D-1][LEN_2D-1] = two;
+    } else if (!strcmp(name, "s3111")) {
+        set_1d_array(a, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s3112")) {
+        set_1d_array(a, LEN_1D, any,frac2);
+        set_1d_array(b, LEN_1D,zero,unit);
+    } else if (!strcmp(name, "s3113")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        a[LEN_1D-1] = -two;
+    } else if (!strcmp(name, "s321")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D,zero,unit);
+    } else if (!strcmp(name, "s322")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D,zero,unit);
+        set_1d_array(c, LEN_1D,zero,unit);
+    } else if (!strcmp(name, "s323")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s331")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        a[LEN_1D-1] = -one;
+    } else if (!strcmp(name, "s332")) {
+        set_1d_array(a, LEN_1D, any,frac2);
+        a[LEN_1D-1] = two;
+    } else if (!strcmp(name, "s341")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s342")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        set_1d_array(b, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s343")) {
+        set_2d_array(aa, any,frac);
+        set_2d_array(bb, one,unit);
+    } else if (!strcmp(name, "s351")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        c[0] = 1.;
+    } else if (!strcmp(name, "s352")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        set_1d_array(b, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s353")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        c[0] = 1.;
+    } else if (!strcmp(name, "s411")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s412")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s413")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s414")) {
+        set_2d_array(aa, one,unit);
+        set_2d_array(bb, any,frac);
+        set_2d_array(cc, any,frac);
+    } else if (!strcmp(name, "s415")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+        a[LEN_1D-1] = -one;
+    } else if (!strcmp(name, "s421")) {
+        set_1d_array(a, LEN_1D, any,frac2);
+        set_1d_array(flat_2d_array, LEN_1D, one, unit);
+    } else if (!strcmp(name, "s422")) {
+        set_1d_array(flat_2d_array, LEN_1D,one,unit);
+        set_1d_array(a, LEN_1D, any,frac2);
+        set_1d_array(flat_2d_array + LEN_1D, LEN_1D, zero, unit);
+    } else if (!strcmp(name, "s1421")) {
+        set_1d_array(b, LEN_1D, one, unit);
+    } else if (!strcmp(name, "s423")) {
+        set_1d_array(flat_2d_array, LEN_1D,zero,unit);
+        set_1d_array(a, LEN_1D, any,frac2);
+        set_1d_array(flat_2d_array + LEN_1D, LEN_1D, one, unit);
+    } else if (!strcmp(name, "s424")) {
+        set_1d_array(flat_2d_array, LEN_1D,one,unit);
+        set_1d_array(a, LEN_1D, any,frac2);
+        set_1d_array(flat_2d_array, LEN_1D, zero, unit);
+    } else if (!strcmp(name, "s431")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s432")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s441")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(&d[0],             LEN_1D/3  , -one,unit);
+        set_1d_array(&d[LEN_1D/3],      LEN_1D/3  , zero,unit);
+        set_1d_array(&d[(2*LEN_1D/3)],  LEN_1D/3+1, one,unit);
+    } else if (!strcmp(name, "s442")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s443")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s451")) {
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s452")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D,small,unit);
+    } else if (!strcmp(name, "s453")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s471")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, one,unit);
+        set_1d_array(d, LEN_1D, any,frac);
+        set_1d_array(e, LEN_1D, any,frac);
+        set_1d_array(x, LEN_1D, zero, unit);
+    } else if (!strcmp(name, "s481")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s482")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s491")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s4112")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s4113")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "s4114")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s4115")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        set_1d_array(b, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s4116")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        set_2d_array(aa, any,frac);
+    } else if (!strcmp(name, "s4117")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D, any,frac);
+        set_1d_array(d, LEN_1D, any,frac);
+    } else if (!strcmp(name, "s4121")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "va")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "vag")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "vas")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "vif")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "vpv")) {
+        set_1d_array(a, LEN_1D,zero,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "vtv")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, one,unit);
+    } else if (!strcmp(name, "vpvtv")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, any,frac);
+    } else if (!strcmp(name, "vpvts")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, any,frac2);
+    } else if (!strcmp(name, "vpvpv")) {
+        set_1d_array(a, LEN_1D, any,frac2);
+        set_1d_array(b, LEN_1D, one,unit);
+        set_1d_array(c, LEN_1D,-one,unit);
+    } else if (!strcmp(name, "vtvtv")) {
+        set_1d_array(a, LEN_1D, one,unit);
+        set_1d_array(b, LEN_1D, two,unit);
+        set_1d_array(c, LEN_1D,half,unit);
+    } else if (!strcmp(name, "vsumr")) {
+        set_1d_array(a, LEN_1D, any,frac);
+    } else if (!strcmp(name, "vdotr")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        set_1d_array(b, LEN_1D, any,frac);
+    } else if (!strcmp(name, "vbor")) {
+        set_1d_array(a, LEN_1D, any,frac);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, one,frac);
+        set_1d_array(d, LEN_1D, two,frac);
+        set_1d_array(e, LEN_1D,half,frac);
+        set_2d_array(aa, any,frac);
+    } else {
+        set_1d_array(a, LEN_1D, any,frac);
+        set_1d_array(b, LEN_1D, any,frac);
+        set_1d_array(c, LEN_1D, one,frac);
+        set_1d_array(d, LEN_1D, two,frac);
+        set_1d_array(e, LEN_1D,half,frac);
+        set_2d_array(aa, half,frac);
+        set_2d_array(bb, one,frac);
+        set_2d_array(cc, any,frac);
+    }
+
+    return 0;
+}
+
+real_t calc_checksum(const char * name)
+{
+    if (!strcmp(name, "s000")) {
+        return sum_a();
+    } else if (!strcmp(name, "s111")) {
+        return sum_a();
+    } else if (!strcmp(name, "s1111")) {
+        return sum_a();
+    } else if (!strcmp(name, "s112")) {
+        return sum_a();
+    } else if (!strcmp(name, "s1112")) {
+        return sum_a();
+    } else if (!strcmp(name, "s113")) {
+        return sum_a();
+    } else if (!strcmp(name, "s1113")) {
+        return sum_a();
+    } else if (!strcmp(name, "s114")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s115")) {
+        return sum_a();
+    } else if (!strcmp(name, "s1115")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s116")) {
+        return sum_a();
+    } else if (!strcmp(name, "s118")) {
+        return sum_a();
+    } else if (!strcmp(name, "s119")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s1119")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s121")) {
+        return sum_a();
+    } else if (!strcmp(name, "s122")) {
+        return sum_a();
+    } else if (!strcmp(name, "s123")) {
+        return sum_a();
+    } else if (!strcmp(name, "s124")) {
+        return sum_a();
+    } else if (!strcmp(name, "s125")) {
+        return sum_flat_2d_array();
+    } else if (!strcmp(name, "s126")) {
+        return sum_bb();
+    } else if (!strcmp(name, "s127")) {
+        return sum_a();
+    } else if (!strcmp(name, "s128")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s131")) {
+        return sum_a();
+    } else if (!strcmp(name, "s132")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s141")) {
+        return sum_flat_2d_array();
+    } else if (!strcmp(name, "s151")) {
+        return sum_a();
+    } else if (!strcmp(name, "s152")) {
+        return sum_a();
+    } else if (!strcmp(name, "s161")) {
+        return sum_a() + sum_c();
+    } else if (!strcmp(name, "s1161")) {
+        return sum_a() + sum_c();
+    } else if (!strcmp(name, "s162")) {
+        return sum_a();
+    } else if (!strcmp(name, "s171")) {
+        return sum_a();
+    } else if (!strcmp(name, "s172")) {
+        return sum_a();
+    } else if (!strcmp(name, "s173")) {
+        return sum_a();
+    } else if (!strcmp(name, "s174")) {
+        return sum_a();
+    } else if (!strcmp(name, "s175")) {
+        return sum_a();
+    } else if (!strcmp(name, "s176")) {
+        return sum_a();
+    } else if (!strcmp(name, "s211")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s212")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s1213")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s221")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s1221")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s222")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s231")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s232")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s1232")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s233")) {
+        return sum_aa_bb();
+    } else if (!strcmp(name, "s2233")) {
+        return sum_aa_bb();
+    } else if (!strcmp(name, "s235")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s241")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s242")) {
+        return sum_a();
+    } else if (!strcmp(name, "s243")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s244")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s1244")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s2244")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s251")) {
+        return sum_a();
+    } else if (!strcmp(name, "s1251")) {
+        return sum_a();
+    } else if (!strcmp(name, "s2251")) {
+        return sum_a();
+    } else if (!strcmp(name, "s3251")) {
+        return sum_a();
+    } else if (!strcmp(name, "s252")) {
+        return sum_a();
+    } else if (!strcmp(name, "s253")) {
+        return sum_a() + sum_c();
+    } else if (!strcmp(name, "s254")) {
+        return sum_a();
+    } else if (!strcmp(name, "s255")) {
+        return sum_a();
+    } else if (!strcmp(name, "s256")) {
+        return sum_a_aa();
+    } else if (!strcmp(name, "s257")) {
+        return sum_a_aa();
+    } else if (!strcmp(name, "s258")) {
+        return sum_b() + sum_e();
+    } else if (!strcmp(name, "s261")) {
+        return sum_a() + sum_c();
+    } else if (!strcmp(name, "s271")) {
+        return sum_a();
+    } else if (!strcmp(name, "s272")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s273")) {
+        return sum_a() + sum_b() + sum_c();
+    } else if (!strcmp(name, "s274")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s275")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s2275")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s276")) {
+        return sum_a();
+    } else if (!strcmp(name, "s277")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s278")) {
+        return sum_a() + sum_b() + sum_c();
+    } else if (!strcmp(name, "s279")) {
+        return sum_a() + sum_b() + sum_c();
+    } else if (!strcmp(name, "s1279")) {
+        return sum_a() + sum_b() + sum_c();
+    } else if (!strcmp(name, "s2710")) {
+        return sum_a() + sum_b() + sum_c();
+    } else if (!strcmp(name, "s2711")) {
+        return sum_a();
+    } else if (!strcmp(name, "s2712")) {
+        return sum_a();
+    } else if (!strcmp(name, "s281")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s1281")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s291")) {
+        return sum_a();
+    } else if (!strcmp(name, "s292")) {
+        return sum_a();
+    } else if (!strcmp(name, "s293")) {
+        return sum_a();
+    } else if (!strcmp(name, "s2101")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s2102")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s2111")) {
+        return sum_aa();
+    } else if (!strcmp(name, "s311")) {
+        return sum_a();
+    } else if (!strcmp(name, "s31111")) {
+        return sum_a();
+    } else if (!strcmp(name, "s321")) {
+        return sum_a();
+    } else if (!strcmp(name, "s322")) {
+        return sum_a();
+    } else if (!strcmp(name, "s323")) {
+        return sum_a() + sum_b();
+    } else if (!strcmp(name, "s341")) {
+        return sum_a();
+    } else if (!strcmp(name, "s342")) {
+        return sum_a();
+    } else if (!strcmp(name, "s343")) {
+        return sum_flat_2d_array();
+    } else if (!strcmp(name, "s351")) {
+        return sum_a();
+    } else if (!strcmp(name, "s1351")) {
+        return sum_a();
+    } else if (!strcmp(name, "s353")) {
+        return sum_a();
+    } else if (!strcmp(name, "s421")) {
+        return sum_xx();
+    } else if (!strcmp(name, "s1421")) {
+        return sum_half_xx();
+    } else if (!strcmp(name, "s422")) {
+        return sum_xx();
+    } else if (!strcmp(name, "s423")) {
+        return sum_flat_2d_array();
+    } else if (!strcmp(name, "s424")) {
+        return sum_xx();
+    } else if (!strcmp(name, "s431")) {
+        return sum_a();
+    } else if (!strcmp(name, "s441")) {
+        return sum_a();
+    } else if (!strcmp(name, "s442")) {
+        return sum_a();
+    } else if (!strcmp(name, "s443")) {
+        return sum_a();
+    } else if (!strcmp(name, "s451")) {
+        return sum_a();
+    } else if (!strcmp(name, "s452")) {
+        return sum_a();
+    } else if (!strcmp(name, "s453")) {
+        return sum_a();
+    } else if (!strcmp(name, "s471")) {
+        return sum_x() + sum_b();
+    } else if (!strcmp(name, "s481")) {
+        return sum_a();
+    } else if (!strcmp(name, "s482")) {
+        return sum_a();
+    } else if (!strcmp(name, "s491")) {
+        return sum_a();
+    } else if (!strcmp(name, "s4112")) {
+        return sum_a();
+    } else if (!strcmp(name, "s4113")) {
+        return sum_a();
+    } else if (!strcmp(name, "s4114")) {
+        return sum_a();
+    } else if (!strcmp(name, "s4117")) {
+        return sum_a();
+    } else if (!strcmp(name, "s4121")) {
+        return sum_a();
+    } else if (!strcmp(name, "va")) {
+        return sum_a();
+    } else if (!strcmp(name, "vag")) {
+        return sum_a();
+    } else if (!strcmp(name, "vas")) {
+        return sum_a();
+    } else if (!strcmp(name, "vif")) {
+        return sum_a();
+    } else if (!strcmp(name, "vpv")) {
+        return sum_a();
+    } else if (!strcmp(name, "vtv")) {
+        return sum_a();
+    } else if (!strcmp(name, "vpvtv")) {
+        return sum_a();
+    } else if (!strcmp(name, "vpvts")) {
+        return sum_a();
+    } else if (!strcmp(name, "vpvpv")) {
+        return sum_a();
+    } else if (!strcmp(name, "vtvtv")) {
+        return sum_a();
+    } else if (!strcmp(name, "vsumr")) {
+        return sum_a();
+    } else if (!strcmp(name, "vbor")) {
+        return sum_x();
+    } else {
+        fprintf(stderr, "Unknown function name passed to calc_checksum: %s\n", name);
+        exit(1);
+    }
+}
+
+real_t get_expected_result(const char * name) 
+{
+    if (!strcmp(name, "s000")) {
+	return 512075584.f;
+    } else if (!strcmp(name, "s111")) {
+	return 32000.410156f;
+    } else if (!strcmp(name, "s1111")) {
+	return 13.352669f;
+    } else if (!strcmp(name, "s112")) {
+	return 81335.929688f;
+    } else if (!strcmp(name, "s1112")) {
+	return 32009.560547f;
+    } else if (!strcmp(name, "s113")) {
+	return 32000.642578f;
+    } else if (!strcmp(name, "s1113")) {
+	return 40010.613281f;
+    } else if (!strcmp(name, "s114")) {
+	return 919.856323f;
+    } else if (!strcmp(name, "s115")) {
+	return 31727.289062f;
+    } else if (!strcmp(name, "s1115")) {
+	return 25487.052734f;
+    } else if (!strcmp(name, "s116")) {
+	return 32000.f;
+    } else if (!strcmp(name, "s118")) {
+	return 32353.884766f;
+    } else if (!strcmp(name, "s119")) {
+	return 86338.984375f;
+    } else if (!strcmp(name, "s1119")) {
+	return 201466.421875f;
+    } else if (!strcmp(name, "s121")) {
+	return 32009.027344f;
+    } else if (!strcmp(name, "s122")) {
+	return 48446.664062f;
+    } else if (!strcmp(name, "s123")) {
+	return 32003.285156f;
+    } else if (!strcmp(name, "s124")) {
+	return 32001.642578f;
+    } else if (!strcmp(name, "s125")) {
+	return 131072.f;
+    } else if (!strcmp(name, "s126")) {
+	return 66955.132812f;
+    } else if (!strcmp(name, "s127")) {
+	return 32003.285156f;
+    } else if (!strcmp(name, "s128")) {
+	return 80000.f;
+    } else if (!strcmp(name, "s131")) {
+	return 32009.027344f;
+    } else if (!strcmp(name, "s132")) {
+	return 65538.5625f;
+    } else if (!strcmp(name, "s141")) {
+	return 3307351.5f;
+    } else if (!strcmp(name, "s151")) {
+	return 32009.027344f;
+    } else if (!strcmp(name, "s152")) {
+	return 44020.523438f;
+    } else if (!strcmp(name, "s161")) {
+	return 64002.054688f;
+    } else if (!strcmp(name, "s1161")) {
+	return 23.546331f;
+    } else if (!strcmp(name, "s162")) {
+	return 32009.023438f;
+    } else if (!strcmp(name, "s171")) {
+	return 48448.019531f;
+    } else if (!strcmp(name, "s172")) {
+	return 48448.019531f;
+    } else if (!strcmp(name, "s173")) {
+	return 32001.626953f;
+    } else if (!strcmp(name, "s174")) {
+	return 32001.626953f;
+    } else if (!strcmp(name, "s175")) {
+	return 32009.023438f;
+    } else if (!strcmp(name, "s176")) {
+	return 32000.f;
+    } else if (!strcmp(name, "s211")) {
+	return 63983.308594f;
+    } else if (!strcmp(name, "s212")) {
+	return 42008.136719f;
+    } else if (!strcmp(name, "s1213")) {
+	return 14.450508f;
+    } else if (!strcmp(name, "s221")) {
+	return 615418176.f;
+    } else if (!strcmp(name, "s1221")) {
+	return 79623.265625f;
+    } else if (!strcmp(name, "s222")) {
+	return 32000.f;
+    } else if (!strcmp(name, "s231")) {
+	return 119107.445312f;
+    } else if (!strcmp(name, "s232")) {
+	return 65536.f;
+    } else if (!strcmp(name, "s1232")) {
+	return 2885.801514f;
+    } else if (!strcmp(name, "s233")) {
+	return 504911.65625f;
+    } else if (!strcmp(name, "s2233")) {
+	return 337652.8125f;
+    } else if (!strcmp(name, "s235")) {
+	return 44810.886719f;
+    } else if (!strcmp(name, "s241")) {
+	return 64000.f;
+    } else if (!strcmp(name, "s242")) {
+	return 1535966208.f;
+    } else if (!strcmp(name, "s243")) {
+	return 138653.21875f;
+    } else if (!strcmp(name, "s244")) {
+	return 64623.015625f;
+    } else if (!strcmp(name, "s1244")) {
+	return 36.141911f;
+    } else if (!strcmp(name, "s2244")) {
+	return 32.852161f;
+    } else if (!strcmp(name, "s251")) {
+	return 32004.367188f;
+    } else if (!strcmp(name, "s1251")) {
+	return 39967.507812f;
+    } else if (!strcmp(name, "s2251")) {
+	return 2.635388f;
+    } else if (!strcmp(name, "s3251")) {
+	return 13.59558f;
+    } else if (!strcmp(name, "s252")) {
+	return 63999.f;
+    } else if (!strcmp(name, "s253")) {
+	return 320115936.f;
+    } else if (!strcmp(name, "s254")) {
+	return 32000.f;
+    } else if (!strcmp(name, "s255")) {
+	return 31953.501953f;
+    } else if (!strcmp(name, "s256")) {
+	return 66207.828125f;
+    } else if (!strcmp(name, "s257")) {
+	return 163072.f;
+    } else if (!strcmp(name, "s258")) {
+	return 14.65278f;
+    } else if (!strcmp(name, "s261")) {
+	return 54894.515625f;
+    } else if (!strcmp(name, "s271")) {
+	return 97793.570312f;
+    } else if (!strcmp(name, "s272")) {
+	return 64000.f;
+    } else if (!strcmp(name, "s273")) {
+	return 96311.546875f;
+    } else if (!strcmp(name, "s274")) {
+	return 320133920.f;
+    } else if (!strcmp(name, "s275")) {
+	return 65536.f;
+    } else if (!strcmp(name, "s2275")) {
+	return 1640158.5f;
+    } else if (!strcmp(name, "s276")) {
+	return 97793.570312f;
+    } else if (!strcmp(name, "s277")) {
+	return 32000.f;
+    } else if (!strcmp(name, "s278")) {
+	return 64012.589844f;
+    } else if (!strcmp(name, "s279")) {
+	return 64014.289062f;
+    } else if (!strcmp(name, "s1279")) {
+	return 32.852161f;
+    } else if (!strcmp(name, "s2710")) {
+	return 96003.28125f;
+    } else if (!strcmp(name, "s2711")) {
+	return 97793.570312f;
+    } else if (!strcmp(name, "s2712")) {
+	return 97793.570312f;
+    } else if (!strcmp(name, "s281")) {
+	return 32000.f;
+    } else if (!strcmp(name, "s1281")) {
+	return INFINITY;
+    } else if (!strcmp(name, "s291")) {
+	return 32000.f;
+    } else if (!strcmp(name, "s292")) {
+	return 31953.501953f;
+    } else if (!strcmp(name, "s293")) {
+	return 31999.998047f;
+    } else if (!strcmp(name, "s2101")) {
+	return 229657.921875f;
+    } else if (!strcmp(name, "s2102")) {
+	return 256.f;
+    } else if (!strcmp(name, "s2111")) {
+	return 34544940.f;
+    } else if (!strcmp(name, "s311")) {
+	return 10.950721f;
+    } else if (!strcmp(name, "s31111")) {
+	return 10.950721f;
+    } else if (!strcmp(name, "s312")) {
+	return 1.030869f;
+    } else if (!strcmp(name, "s313")) {
+	return 1.644824f;
+    } else if (!strcmp(name, "s314")) {
+	return 1.f;
+    } else if (!strcmp(name, "s315")) {
+	return 54857.f;
+    } else if (!strcmp(name, "s316")) {
+	return 0.000031f;
+    } else if (!strcmp(name, "s317")) {
+	return 0.f;
+    } else if (!strcmp(name, "s318")) {
+	return 32002.f;
+    } else if (!strcmp(name, "s319")) {
+	return 43.802898f;
+    } else if (!strcmp(name, "s3110")) {
+	return 514.f;
+    } else if (!strcmp(name, "s13110")) {
+	return 3.f;
+    } else if (!strcmp(name, "s3111")) {
+	return 10.950725f;
+    } else if (!strcmp(name, "s3112")) {
+	return 1.644725f;
+    } else if (!strcmp(name, "s3113")) {
+	return 2.f;
+    } else if (!strcmp(name, "s321")) {
+	return 32000.f;
+    } else if (!strcmp(name, "s322")) {
+	return 32000.f;
+    } else if (!strcmp(name, "s323")) {
+	return 146472.4375f;
+    } else if (!strcmp(name, "s331")) {
+	return 32000.f;
+    } else if (!strcmp(name, "s332")) {
+	return -1.f;
+    } else if (!strcmp(name, "s341")) {
+	return 10.950721f;
+    } else if (!strcmp(name, "s342")) {
+	return 10.950721f;
+    } else if (!strcmp(name, "s343")) {
+	return 1567.932129f;
+    } else if (!strcmp(name, "s351")) {
+	return 2560660224.f;
+    } else if (!strcmp(name, "s1351")) {
+	return 21.901442f;
+    } else if (!strcmp(name, "s352")) {
+	return 1.644808f;
+    } else if (!strcmp(name, "s353")) {
+	return 320084192.f;
+    } else if (!strcmp(name, "s421")) {
+	return 32009.023438f;
+    } else if (!strcmp(name, "s1421")) {
+	return 16000.f;
+    } else if (!strcmp(name, "s422")) {
+	return 3.737715f;
+    } else if (!strcmp(name, "s423")) {
+	return 64006.683594f;
+    } else if (!strcmp(name, "s424")) {
+	return 822.364014f;
+    } else if (!strcmp(name, "s431")) {
+	return 196500.265625f;
+    } else if (!strcmp(name, "s441")) {
+	return 48448.019531f;
+    } else if (!strcmp(name, "s442")) {
+	return 40224.117188f;
+    } else if (!strcmp(name, "s443")) {
+	return 64895.867188f;
+    } else if (!strcmp(name, "s451")) {
+	return 32007.898438f;
+    } else if (!strcmp(name, "s452")) {
+	return 32511.939453f;
+    } else if (!strcmp(name, "s453")) {
+	return 21.901442f;
+    } else if (!strcmp(name, "s471")) {
+	return 64004.925781f;
+    } else if (!strcmp(name, "s481")) {
+	return 48448.019531f;
+    } else if (!strcmp(name, "s482")) {
+	return 48448.019531f;
+    } else if (!strcmp(name, "s491")) {
+	return 32001.640625f;
+    } else if (!strcmp(name, "s4112")) {
+	return 141504.875f;
+    } else if (!strcmp(name, "s4113")) {
+	return 32001.640625f;
+    } else if (!strcmp(name, "s4114")) {
+	return 32000.f;
+    } else if (!strcmp(name, "s4115")) {
+	return 1.038636f;
+    } else if (!strcmp(name, "s4116")) {
+	return 0.753265f;
+    } else if (!strcmp(name, "s4117")) {
+	return 32002.205078f;
+    } else if (!strcmp(name, "s4121")) {
+	return 48448.019531f;
+    } else if (!strcmp(name, "va")) {
+	return 1.644725f;
+    } else if (!strcmp(name, "vag")) {
+	return 1.644725f;
+    } else if (!strcmp(name, "vas")) {
+	return 1.644725f;
+    } else if (!strcmp(name, "vif")) {
+	return 1.644725f;
+    } else if (!strcmp(name, "vpv")) {
+	return 164487.78125f;
+    } else if (!strcmp(name, "vtv")) {
+	return 32000.f;
+    } else if (!strcmp(name, "vpvtv")) {
+	return 97793.570312f;
+    } else if (!strcmp(name, "vpvts")) {
+	return 17522152701952.f;
+    } else if (!strcmp(name, "vpvpv")) {
+	return 1.644725f;
+    } else if (!strcmp(name, "vtvtv")) {
+	return 32000.f;
+    } else if (!strcmp(name, "vsumr")) {
+	return 10.950725f;
+    } else if (!strcmp(name, "vdotr")) {
+	return 1.644824f;
+    } else if (!strcmp(name, "vbor")) {
+	return 31924.046875f;
+    } else {
+        fprintf(stderr, "Unknown function name passed to expected_result: %s\n", name);
+        exit(1);
+    }
+}
+
+typedef real_t(*test_function_t)(struct args_t *);
+
+static _Bool is_checksum_same(real_t expected, real_t value)
+{
+  if (expected == INFINITY)
+      return value == INFINITY;
+  else if(expected == 0.f)
+      return value <= 0.01f;
+  else {
+      real_t fraction = value / expected;
+      return 0.99f <= fraction && fraction <= 1.01f;
+  }
+}
+
+void run(test_function_t vector_func, const char *fname, void * arg_info)
+{
+    struct args_t func_args = {.arg_info=arg_info};
+
+    double result = vector_func(&func_args);
+    double expected_result = get_expected_result(fname);
+
+    if (!is_checksum_same(expected_result, result))
+      {
+	fprintf (stderr, "value: %f, expected: %f\n", result, expected_result);
+	__builtin_abort();
+      }
+}
+
+int
+__attribute__((noipa))
+dummy(float a[LEN_1D], float b[LEN_1D], float c[LEN_1D], float d[LEN_1D], float e[LEN_1D], float aa[LEN_2D][LEN_2D], float bb[LEN_2D][LEN_2D], float cc[LEN_2D][LEN_2D], float s){
+    // --  called in each loop to make all computations appear required
+    return 0;
+}
+
+#pragma GCC pop_options
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s000.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s000.c
new file mode 100644
index 0000000..b5d5faa
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s000.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s000(struct args_t * func_args)
+{
+//    linear dependence testing
+//    no dependence - vectorizable
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = b[i] + 1;
+        }
+        dummy((real_t*)a, (real_t*)b, (real_t*)c, (real_t*)d, (real_t*)e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s000, "s000", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s111.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s111.c
new file mode 100644
index 0000000..37e4684
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s111.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s111(struct args_t * func_args)
+{
+//    linear dependence testing
+//    no dependence - vectorizable
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        for (int i = 1; i < LEN_1D; i += 2) {
+            a[i] = a[i - 1] + b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s111, "s111", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1111.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1111.c
new file mode 100644
index 0000000..cd7e84e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1111.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1111(struct args_t * func_args)
+{
+//    no dependence - vectorizable
+//    jump in data access
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        for (int i = 0; i < LEN_1D/2; i++) {
+            a[2*i] = c[i] * b[i] + d[i] * b[i] + c[i] * c[i] + d[i] * b[i] + d[i] * c[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1111, "s1111", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1112.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1112.c
new file mode 100644
index 0000000..20cb346
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1112.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1112(struct args_t * func_args)
+{
+//    linear dependence testing
+//    loop reversal
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations*3; nl++) {
+        for (int i = LEN_1D - 1; i >= 0; i--) {
+            a[i] = b[i] + (real_t) 1.;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1112, "s1112", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1113.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1113.c
new file mode 100644
index 0000000..55aa8ed
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1113.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1113(struct args_t * func_args)
+{
+//    linear dependence testing
+//    one iteration dependency on a(LEN_1D/2) but still vectorizable
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = a[LEN_1D/2] + b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1113, "s1113", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1115.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1115.c
new file mode 100644
index 0000000..1b2ff0c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1115.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1115(struct args_t * func_args)
+{
+//    linear dependence testing
+//    triangular saxpy loop
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 100*(iterations/LEN_2D); nl++) {
+        for (int i = 0; i < LEN_2D; i++) {
+            for (int j = 0; j < LEN_2D; j++) {
+                aa[i][j] = aa[i][j]*cc[j][i] + bb[i][j];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1115, "s1115", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1119.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1119.c
new file mode 100644
index 0000000..16610fa
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1119.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1119(struct args_t * func_args)
+{
+//    linear dependence testing
+//    no dependence - vectorizable
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 200*(iterations/(LEN_2D)); nl++) {
+        for (int i = 1; i < LEN_2D; i++) {
+            for (int j = 0; j < LEN_2D; j++) {
+                aa[i][j] = aa[i-1][j] + bb[i][j];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1119, "s1119", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s112.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s112.c
new file mode 100644
index 0000000..c8afaf7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s112.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s112(struct args_t * func_args)
+{
+//    linear dependence testing
+//    loop reversal
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 3*iterations; nl++) {
+        for (int i = LEN_1D - 2; i >= 0; i--) {
+            a[i+1] = a[i] + b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s112, "s112", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target { { ! powerpc*-*-* } || has_arch_pwr8 } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s113.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s113.c
new file mode 100644
index 0000000..f22ab4c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s113.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s113(struct args_t * func_args)
+{
+//    linear dependence testing
+//    a(i)=a(1) but no actual dependence cycle
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 1; i < LEN_1D; i++) {
+            a[i] = a[0] + b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s113, "s113", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s114.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s114.c
new file mode 100644
index 0000000..fded659
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s114.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s114(struct args_t * func_args)
+{
+//    linear dependence testing
+//    transpose vectorization
+//    Jump in data access - not vectorizable
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 200*(iterations/(LEN_2D)); nl++) {
+        for (int i = 0; i < LEN_2D; i++) {
+            for (int j = 0; j < i; j++) {
+                aa[i][j] = aa[j][i] + bb[i][j];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s114, "s114", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s115.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s115.c
new file mode 100644
index 0000000..153cbf8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s115.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s115(struct args_t * func_args)
+{
+//    linear dependence testing
+//    triangular saxpy loop
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 1000*(iterations/LEN_2D); nl++) {
+        for (int j = 0; j < LEN_2D; j++) {
+            for (int i = j+1; i < LEN_2D; i++) {
+                a[i] -= aa[j][i] * a[j];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s115, "s115", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s116.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s116.c
new file mode 100644
index 0000000..93d16b3
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s116.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s116(struct args_t * func_args)
+{
+//    linear dependence testing
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations*10; nl++) {
+        for (int i = 0; i < LEN_1D - 5; i += 5) {
+            a[i] = a[i + 1] * a[i];
+            a[i + 1] = a[i + 2] * a[i + 1];
+            a[i + 2] = a[i + 3] * a[i + 2];
+            a[i + 3] = a[i + 4] * a[i + 3];
+            a[i + 4] = a[i + 5] * a[i + 4];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s116, "s116", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1161.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1161.c
new file mode 100644
index 0000000..dfdeaf7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1161.c
@@ -0,0 +1,48 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1161(struct args_t * func_args)
+{
+//    control flow
+//    tests for recognition of loop independent dependences
+//    between statements in mutually exclusive regions.
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D-1; ++i) {
+            if (c[i] < (real_t)0.) {
+                goto L20;
+            }
+            a[i] = c[i] + d[i] * e[i];
+            goto L10;
+L20:
+            b[i] = a[i] + d[i] * d[i];
+L10:
+            ;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1161, "s1161", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s118.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s118.c
new file mode 100644
index 0000000..3896a9a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s118.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s118(struct args_t * func_args)
+{
+//    linear dependence testing
+//    potential dot product recursion
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 200*(iterations/LEN_2D); nl++) {
+        for (int i = 1; i < LEN_2D; i++) {
+            for (int j = 0; j <= i - 1; j++) {
+                a[i] += bb[j][i] * a[i-j-1];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s118, "s118", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s119.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s119.c
new file mode 100644
index 0000000..43391c5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s119.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s119(struct args_t * func_args)
+{
+//    linear dependence testing
+//    no dependence - vectorizable
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 200*(iterations/(LEN_2D)); nl++) {
+        for (int i = 1; i < LEN_2D; i++) {
+            for (int j = 1; j < LEN_2D; j++) {
+                aa[i][j] = aa[i-1][j-1] + bb[i][j];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s119, "s119", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s121.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s121.c
new file mode 100644
index 0000000..98a67ac
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s121.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s121(struct args_t * func_args)
+{
+//    induction variable recognition
+//    loop with possible ambiguity because of scalar store
+
+    initialise_arrays(__func__);
+
+    int j;
+    for (int nl = 0; nl < 3*iterations; nl++) {
+        for (int i = 0; i < LEN_1D-1; i++) {
+            j = i + 1;
+            a[i] = a[j] + b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s121, "s121", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1213.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1213.c
new file mode 100644
index 0000000..f9f3beb
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1213.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1213(struct args_t * func_args)
+{
+//    statement reordering
+//    dependency needing temporary
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 1; i < LEN_1D-1; i++) {
+            a[i] = b[i-1]+c[i];
+            b[i] = a[i+1]*d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1213, "s1213", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s122.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s122.c
new file mode 100644
index 0000000..0aceeb7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s122.c
@@ -0,0 +1,48 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s122(struct args_t * func_args)
+{
+//    induction variable recognition
+//    variable lower and upper bound, and stride
+//    reverse data access and jump in data access
+
+    struct{int a;int b;} * x = func_args->arg_info;
+    int n1 = x->a;
+    int n3 = x->b;
+
+    initialise_arrays(__func__);
+
+    int j, k;
+    for (int nl = 0; nl < iterations; nl++) {
+        j = 1;
+        k = 0;
+        for (int i = n1-1; i < LEN_1D; i += n3) {
+            k += j;
+            a[i] += b[LEN_1D - k];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s122, "s122", &(struct{int a;int b;}){n1, n3});
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1221.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1221.c
new file mode 100644
index 0000000..4ecccfd
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1221.c
@@ -0,0 +1,38 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1221(struct args_t * func_args)
+{
+//    run-time symbolic resolution
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 4; i < LEN_1D; i++) {
+            b[i] = b[i - 4] + a[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1221, "s1221", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s123.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s123.c
new file mode 100644
index 0000000..adb4da8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s123.c
@@ -0,0 +1,47 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s123(struct args_t * func_args)
+{
+//    induction variable recognition
+//    induction variable under an if
+//    not vectorizable, the condition cannot be speculated
+
+    initialise_arrays(__func__);
+
+    int j;
+    for (int nl = 0; nl < iterations; nl++) {
+        j = -1;
+        for (int i = 0; i < (LEN_1D/2); i++) {
+            j++;
+            a[j] = b[i] + d[i] * e[i];
+            if (c[i] > (real_t)0.) {
+                j++;
+                a[j] = c[i] + d[i] * e[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s123, "s123", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1232.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1232.c
new file mode 100644
index 0000000..cfa2e67
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1232.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1232(struct args_t * func_args)
+{
+//    loop interchange
+//    interchanging of triangular loops
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 100*(iterations/LEN_2D); nl++) {
+        for (int j = 0; j < LEN_2D; j++) {
+            for (int i = j; i < LEN_2D; i++) {
+                aa[i][j] = bb[i][j] + cc[i][j];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 1.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1232, "s1232", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s124.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s124.c
new file mode 100644
index 0000000..6a246f0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s124.c
@@ -0,0 +1,47 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s124(struct args_t * func_args)
+{
+//    induction variable recognition
+//    induction variable under both sides of if (same value)
+
+    initialise_arrays(__func__);
+
+    int j;
+    for (int nl = 0; nl < iterations; nl++) {
+        j = -1;
+        for (int i = 0; i < LEN_1D; i++) {
+            if (b[i] > (real_t)0.) {
+                j++;
+                a[j] = b[i] + d[i] * e[i];
+            } else {
+                j++;
+                a[j] = c[i] + d[i] * e[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s124, "s124", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1244.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1244.c
new file mode 100644
index 0000000..94a3041
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1244.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1244(struct args_t * func_args)
+{
+//    node splitting
+//    cycle with ture and anti dependency
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D-1; i++) {
+            a[i] = b[i] + c[i] * c[i] + b[i]*b[i] + c[i];
+            d[i] = a[i] + a[i+1];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1244, "s1244", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s125.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s125.c
new file mode 100644
index 0000000..6790743
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s125.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s125(struct args_t * func_args)
+{
+//    induction variable recognition
+//    induction variable in two loops; collapsing possible
+
+    initialise_arrays(__func__);
+
+    int k;
+    for (int nl = 0; nl < 100*(iterations/(LEN_2D)); nl++) {
+        k = -1;
+        for (int i = 0; i < LEN_2D; i++) {
+            for (int j = 0; j < LEN_2D; j++) {
+                k++;
+                flat_2d_array[k] = aa[i][j] + bb[i][j] * cc[i][j];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s125, "s125", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1251.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1251.c
new file mode 100644
index 0000000..b2e099c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1251.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1251(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    scalar expansion
+
+    initialise_arrays(__func__);
+
+    real_t s;
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            s = b[i]+c[i];
+            b[i] = a[i]+d[i];
+            a[i] = s*e[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1251, "s1251", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s126.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s126.c
new file mode 100644
index 0000000..a1c71b3
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s126.c
@@ -0,0 +1,45 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s126(struct args_t * func_args)
+{
+//    induction variable recognition
+//    induction variable in two loops; recurrence in inner loop
+
+    initialise_arrays(__func__);
+
+    int k;
+    for (int nl = 0; nl < 10*(iterations/LEN_2D); nl++) {
+        k = 1;
+        for (int i = 0; i < LEN_2D; i++) {
+            for (int j = 1; j < LEN_2D; j++) {
+                bb[j][i] = bb[j-1][i] + flat_2d_array[k-1] * cc[j][i];
+                ++k;
+            }
+            ++k;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s126, "s126", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s127.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s127.c
new file mode 100644
index 0000000..6f9a821
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s127.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s127(struct args_t * func_args)
+{
+//    induction variable recognition
+//    induction variable with multiple increments
+
+    initialise_arrays(__func__);
+
+    int j;
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        j = -1;
+        for (int i = 0; i < LEN_1D/2; i++) {
+            j++;
+            a[j] = b[i] + c[i] * d[i];
+            j++;
+            a[j] = b[i] + d[i] * e[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s127, "s127", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1279.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1279.c
new file mode 100644
index 0000000..4a481c7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1279.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1279(struct args_t * func_args)
+{
+//    control flow
+//    vector if/gotos
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (a[i] < (real_t)0.) {
+                if (b[i] > a[i]) {
+                    c[i] += d[i] * e[i];
+                }
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1279, "s1279", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s128.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s128.c
new file mode 100644
index 0000000..7b8874a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s128.c
@@ -0,0 +1,45 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s128(struct args_t * func_args)
+{
+//    induction variables
+//    coupled induction variables
+//    jump in data access
+
+    initialise_arrays(__func__);
+
+    int j, k;
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        j = -1;
+        for (int i = 0; i < LEN_1D/2; i++) {
+            k = j + 1;
+            a[i] = b[k] - d[i];
+            j = k + 1;
+            b[k] = a[i] + c[k];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 1.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s128, "s128", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1281.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1281.c
new file mode 100644
index 0000000..dba95a8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1281.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1281(struct args_t * func_args)
+{
+//    crossing thresholds
+//    index set splitting
+//    reverse data access
+
+    initialise_arrays(__func__);
+
+    real_t x;
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            x = b[i]*c[i] + a[i]*d[i] + e[i];
+            a[i] = x-(real_t)1.0;
+            b[i] = x;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1281, "s1281", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s131.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s131.c
new file mode 100644
index 0000000..73f58a7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s131.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s131(struct args_t * func_args)
+{
+//    global data flow analysis
+//    forward substitution
+
+    initialise_arrays(__func__);
+
+    int m  = 1;
+    for (int nl = 0; nl < 5*iterations; nl++) {
+        for (int i = 0; i < LEN_1D - 1; i++) {
+            a[i] = a[i + m] + b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s131, "s131", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s13110.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s13110.c
new file mode 100644
index 0000000..4d2ef4c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s13110.c
@@ -0,0 +1,51 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s13110(struct args_t * func_args)
+{
+//    reductions
+//    if to max with index reductio 2 dimensions
+
+    initialise_arrays(__func__);
+
+    int xindex, yindex;
+    real_t max, chksum;
+    for (int nl = 0; nl < 100*(iterations/(LEN_2D)); nl++) {
+        max = aa[(0)][0];
+        xindex = 0;
+        yindex = 0;
+        for (int i = 0; i < LEN_2D; i++) {
+            for (int j = 0; j < LEN_2D; j++) {
+                if (aa[i][j] > max) {
+                    max = aa[i][j];
+                    xindex = i;
+                    yindex = j;
+                }
+            }
+        }
+        chksum = max + (real_t) xindex + (real_t) yindex;
+        dummy(a, b, c, d, e, aa, bb, cc, chksum);
+    }
+
+    return max + xindex+1 + yindex+1;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s13110, "s13110", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s132.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s132.c
new file mode 100644
index 0000000..c2f5c2f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s132.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s132(struct args_t * func_args)
+{
+//    global data flow analysis
+//    loop with multiple dimension ambiguous subscripts
+
+    initialise_arrays(__func__);
+
+    int m = 0;
+    int j = m;
+    int k = m+1;
+    for (int nl = 0; nl < 400*iterations; nl++) {
+        for (int i= 1; i < LEN_2D; i++) {
+            aa[j][i] = aa[k][i-1] + b[i] * c[1];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s132, "s132", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1351.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1351.c
new file mode 100644
index 0000000..d020f90
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1351.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1351(struct args_t * func_args)
+{
+//    induction pointer recognition
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 8*iterations; nl++) {
+        real_t* __restrict__ A = a;
+        real_t* __restrict__ B = b;
+        real_t* __restrict__ C = c;
+        for (int i = 0; i < LEN_1D; i++) {
+            *A = *B+*C;
+            A++;
+            B++;
+            C++;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1351, "s1351", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s141.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s141.c
new file mode 100644
index 0000000..b8ef6a6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s141.c
@@ -0,0 +1,45 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s141(struct args_t * func_args)
+{
+//    nonlinear dependence testing
+//    walk a row in a symmetric packed array
+//    element a(i,j) for (int j>i) stored in location j*(j-1)/2+i
+
+    initialise_arrays(__func__);
+
+    int k;
+    for (int nl = 0; nl < 200*(iterations/LEN_2D); nl++) {
+        for (int i = 0; i < LEN_2D; i++) {
+            k = (i+1) * ((i+1) - 1) / 2 + (i+1)-1;
+            for (int j = i; j < LEN_2D; j++) {
+                flat_2d_array[k] += bb[j][i];
+                k += j+1;
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s141, "s141", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1421.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1421.c
new file mode 100644
index 0000000..043989f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s1421.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s1421(struct args_t * func_args)
+{
+//    storage classes and equivalencing
+//    equivalence- no overlap
+
+    initialise_arrays(__func__);
+
+    xx = &b[LEN_1D/2];
+
+    for (int nl = 0; nl < 8*iterations; nl++) {
+        for (int i = 0; i < LEN_1D/2; i++) {
+            b[i] = xx[i] + a[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 1.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s1421, "s1421", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s151.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s151.c
new file mode 100644
index 0000000..46f7181
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s151.c
@@ -0,0 +1,45 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+void s151s(real_t a[LEN_1D], real_t b[LEN_1D],  int m)
+{
+    for (int i = 0; i < LEN_1D-1; i++) {
+        a[i] = a[i + m] + b[i];
+    }
+}
+
+real_t s151(struct args_t * func_args)
+{
+//    interprocedural data flow analysis
+//    passing parameter information into a subroutine
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 5*iterations; nl++) {
+        s151s(a, b,  1);
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s151, "s151", NULL);
+
+  return 0;
+}
+
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s152.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s152.c
new file mode 100644
index 0000000..d9c4186
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s152.c
@@ -0,0 +1,46 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+void s152s(real_t a[LEN_1D], real_t b[LEN_1D], real_t c[LEN_1D], int i)
+{
+    a[i] += b[i] * c[i];
+}
+
+real_t s152(struct args_t * func_args)
+{
+//    interprocedural data flow analysis
+//    collecting information from a subroutine
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            b[i] = d[i] * e[i];
+            s152s(a, b, c, i);
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s152, "s152", NULL);
+
+  return 0;
+}
+
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s161.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s161.c
new file mode 100644
index 0000000..efdf193
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s161.c
@@ -0,0 +1,48 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s161(struct args_t * func_args)
+{
+//    control flow
+//    tests for recognition of loop independent dependences
+//    between statements in mutually exclusive regions.
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations/2; nl++) {
+        for (int i = 0; i < LEN_1D-1; ++i) {
+            if (b[i] < (real_t)0.) {
+                goto L20;
+            }
+            a[i] = c[i] + d[i] * e[i];
+            goto L10;
+L20:
+            c[i+1] = a[i] + d[i] * d[i];
+L10:
+            ;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s161, "s161", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s162.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s162.c
new file mode 100644
index 0000000..521363d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s162.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s162(struct args_t * func_args)
+{
+//    control flow
+//    deriving assertions
+
+    int k = *(int*)func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        if (k > 0) {
+            for (int i = 0; i < LEN_1D-1; i++) {
+                a[i] = a[i + k] + b[i] * c[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s162, "s162", &n1);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s171.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s171.c
new file mode 100644
index 0000000..291ec0a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s171.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s171(struct args_t * func_args)
+{
+//    symbolics
+//    symbolic dependence tests
+
+    int inc = *(int*)func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i * inc] += b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s171, "s171", &n1);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s172.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s172.c
new file mode 100644
index 0000000..077f6e3
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s172.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s172(struct args_t * func_args)
+{
+//    symbolics
+//    vectorizable if n3 .ne. 0
+
+    struct{int a;int b;} * x = func_args->arg_info;
+    int n1 = x->a;
+    int n3 = x->b;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = n1-1; i < LEN_1D; i += n3) {
+            a[i] += b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s172, "s172", &(struct{int a;int b;}){n1, n3});
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s173.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s173.c
new file mode 100644
index 0000000..4bd166b9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s173.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s173(struct args_t * func_args)
+{
+//    symbolics
+//    expression in loop bounds and subscripts
+
+    initialise_arrays(__func__);
+
+    int k = LEN_1D/2;
+    for (int nl = 0; nl < 10*iterations; nl++) {
+        for (int i = 0; i < LEN_1D/2; i++) {
+            a[i+k] = a[i] + b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s173, "s173", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s174.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s174.c
new file mode 100644
index 0000000..a4df4da
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s174.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s174(struct args_t * func_args)
+{
+//    symbolics
+//    loop with subscript that may seem ambiguous
+
+    int M = *(int*)func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 10*iterations; nl++) {
+        for (int i = 0; i < M; i++) {
+            a[i+M] = a[i] + b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s174, "s174", &(struct{int a;}){LEN_1D/2});
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s175.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s175.c
new file mode 100644
index 0000000..4bfb854
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s175.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s175(struct args_t * func_args)
+{
+//    symbolics
+//    symbolic dependence tests
+
+    int inc = *(int*)func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D-1; i += inc) {
+            a[i] = a[i + inc] + b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s175, "s175", &n1);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s176.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s176.c
new file mode 100644
index 0000000..79faf7f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s176.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s176(struct args_t * func_args)
+{
+//    symbolics
+//    convolution
+
+    initialise_arrays(__func__);
+
+    int m = LEN_1D/2;
+    for (int nl = 0; nl < 4*(iterations/LEN_1D); nl++) {
+        for (int j = 0; j < (LEN_1D/2); j++) {
+            for (int i = 0; i < m; i++) {
+                a[i] += b[i+m-j-1] * c[j];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s176, "s176", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2101.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2101.c
new file mode 100644
index 0000000..ad6e586
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2101.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s2101(struct args_t * func_args)
+{
+//    diagonals
+//    main diagonal calculation
+//    jump in data access
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 10*iterations; nl++) {
+        for (int i = 0; i < LEN_2D; i++) {
+            aa[i][i] += bb[i][i] * cc[i][i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s2101, "s2101", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2102.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2102.c
new file mode 100644
index 0000000..dd00eb8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2102.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s2102(struct args_t * func_args)
+{
+//    diagonals
+//    identity matrix, best results vectorize both inner and outer loops
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 100*(iterations/LEN_2D); nl++) {
+        for (int i = 0; i < LEN_2D; i++) {
+            for (int j = 0; j < LEN_2D; j++) {
+                aa[j][i] = (real_t)0.;
+            }
+            aa[i][i] = (real_t)1.;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s2102, "s2102", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s211.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s211.c
new file mode 100644
index 0000000..5053322
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s211.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s211(struct args_t * func_args)
+{
+//    statement reordering
+//    statement reordering allows vectorization
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 1; i < LEN_1D-1; i++) {
+            a[i] = b[i - 1] + c[i] * d[i];
+            b[i] = b[i + 1] - e[i] * d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s211, "s211", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2111.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2111.c
new file mode 100644
index 0000000..42ff35f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2111.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s2111(struct args_t * func_args)
+{
+//    wavefronts, it will make jump in data access
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 100*(iterations/(LEN_2D)); nl++) {
+        for (int j = 1; j < LEN_2D; j++) {
+            for (int i = 1; i < LEN_2D; i++) {
+                aa[j][i] = (aa[j][i-1] + aa[j-1][i])/1.9;
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s2111, "s2111", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s212.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s212.c
new file mode 100644
index 0000000..366b678
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s212.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s212(struct args_t * func_args)
+{
+//    statement reordering
+//    dependency needing temporary
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D-1; i++) {
+            a[i] *= c[i];
+            b[i] += a[i + 1] * d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s212, "s212", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s221.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s221.c
new file mode 100644
index 0000000..7f42c24
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s221.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s221(struct args_t * func_args)
+{
+//    loop distribution
+//    loop that is partially recursive
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations/2; nl++) {
+        for (int i = 1; i < LEN_1D; i++) {
+            a[i] += c[i] * d[i];
+            b[i] = b[i - 1] + a[i] + d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s221, "s221", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s222.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s222.c
new file mode 100644
index 0000000..0cdb8cd
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s222.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s222(struct args_t * func_args)
+{
+//    loop distribution
+//    partial loop vectorizatio recurrence in middle
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations/2; nl++) {
+        for (int i = 1; i < LEN_1D; i++) {
+            a[i] += b[i] * c[i];
+            e[i] = e[i - 1] * e[i - 1];
+            a[i] -= b[i] * c[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s222, "s222", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2233.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2233.c
new file mode 100644
index 0000000..c367c21
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2233.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s2233(struct args_t * func_args)
+{
+//    loop interchange
+//    interchanging with one of two inner loops
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 100*(iterations/LEN_2D); nl++) {
+        for (int i = 1; i < LEN_2D; i++) {
+            for (int j = 1; j < LEN_2D; j++) {
+                aa[j][i] = aa[j-1][i] + cc[j][i];
+            }
+            for (int j = 1; j < LEN_2D; j++) {
+                bb[i][j] = bb[i-1][j] + cc[i][j];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s2233, "s2233", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2244.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2244.c
new file mode 100644
index 0000000..9b0ca74
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2244.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s2244(struct args_t * func_args)
+{
+//    node splitting
+//    cycle with ture and anti dependency
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D-1; i++) {
+            a[i+1] = b[i] + e[i];
+            a[i] = b[i] + c[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s2244, "s2244", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2251.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2251.c
new file mode 100644
index 0000000..980696b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2251.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s2251(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    scalar expansion
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        real_t s = (real_t)0.0;
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = s*e[i];
+            s = b[i]+c[i];
+            b[i] = a[i]+d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s2251, "s2251", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2275.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2275.c
new file mode 100644
index 0000000..e182b83
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2275.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s2275(struct args_t * func_args)
+{
+//    loop distribution is needed to be able to interchange
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 100*(iterations/LEN_2D); nl++) {
+        for (int i = 0; i < LEN_2D; i++) {
+            for (int j = 0; j < LEN_2D; j++) {
+                aa[j][i] = aa[j][i] + bb[j][i] * cc[j][i];
+            }
+            a[i] = b[i] + c[i] * d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s2275, "s2275", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s231.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s231.c
new file mode 100644
index 0000000..da52334
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s231.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s231(struct args_t * func_args)
+{
+//    loop interchange
+//    loop with data dependency
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 100*(iterations/LEN_2D); nl++) {
+        for (int i = 0; i < LEN_2D; ++i) {
+            for (int j = 1; j < LEN_2D; j++) {
+                aa[j][i] = aa[j - 1][i] + bb[j][i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s231, "s231", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s232.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s232.c
new file mode 100644
index 0000000..37bb310
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s232.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s232(struct args_t * func_args)
+{
+//    loop interchange
+//    interchanging of triangular loops
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 100*(iterations/(LEN_2D)); nl++) {
+        for (int j = 1; j < LEN_2D; j++) {
+            for (int i = 1; i <= j; i++) {
+                aa[j][i] = aa[j][i-1]*aa[j][i-1]+bb[j][i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 1.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s232, "s232", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s233.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s233.c
new file mode 100644
index 0000000..10f003a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s233.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s233(struct args_t * func_args)
+{
+//    loop interchange
+//    interchanging with one of two inner loops
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 100*(iterations/LEN_2D); nl++) {
+        for (int i = 1; i < LEN_2D; i++) {
+            for (int j = 1; j < LEN_2D; j++) {
+                aa[j][i] = aa[j-1][i] + cc[j][i];
+            }
+            for (int j = 1; j < LEN_2D; j++) {
+                bb[j][i] = bb[j][i-1] + cc[j][i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s233, "s233", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s235.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s235.c
new file mode 100644
index 0000000..1340b20
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s235.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s235(struct args_t * func_args)
+{
+//    loop interchanging
+//    imperfectly nested loops
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 200*(iterations/LEN_2D); nl++) {
+        for (int i = 0; i < LEN_2D; i++) {
+            a[i] += b[i] * c[i];
+            for (int j = 1; j < LEN_2D; j++) {
+                aa[j][i] = aa[j-1][i] + bb[j][i] * a[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s235, "s235", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s241.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s241.c
new file mode 100644
index 0000000..14b4188
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s241.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s241(struct args_t * func_args)
+{
+//    node splitting
+//    preloading necessary to allow vectorization
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        for (int i = 0; i < LEN_1D-1; i++) {
+            a[i] = b[i] * c[i  ] * d[i];
+            b[i] = a[i] * a[i+1] * d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s241, "s241", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s242.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s242.c
new file mode 100644
index 0000000..924c679
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s242.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s242(struct args_t * func_args)
+{
+//    node splitting
+
+    struct{real_t a;real_t b;} * x = func_args->arg_info;
+    real_t s1 = x->a;
+    real_t s2 = x->b;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations/5; nl++) {
+        for (int i = 1; i < LEN_1D; ++i) {
+            a[i] = a[i - 1] + s1 + s2 + b[i] + c[i] + d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s242, "s242", &(struct{real_t a;real_t b;}){s1, s2});
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s243.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s243.c
new file mode 100644
index 0000000..9361821
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s243.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s243(struct args_t * func_args)
+{
+//    node splitting
+//    false dependence cycle breaking
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D-1; i++) {
+            a[i] = b[i] + c[i  ] * d[i];
+            b[i] = a[i] + d[i  ] * e[i];
+            a[i] = b[i] + a[i+1] * d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s243, "s243", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s244.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s244.c
new file mode 100644
index 0000000..8886876
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s244.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s244(struct args_t * func_args)
+{
+//    node splitting
+//    false dependence cycle breaking
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D-1; ++i) {
+            a[i] = b[i] + c[i] * d[i];
+            b[i] = c[i] + b[i];
+            a[i+1] = b[i] + a[i+1] * d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s244, "s244", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s251.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s251.c
new file mode 100644
index 0000000..a648f94
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s251.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s251(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    scalar expansion
+
+    initialise_arrays(__func__);
+
+    real_t s;
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            s = b[i] + c[i] * d[i];
+            a[i] = s * s;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s251, "s251", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s252.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s252.c
new file mode 100644
index 0000000..f1302b6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s252.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s252(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    loop with ambiguous scalar temporary
+
+    initialise_arrays(__func__);
+
+    real_t t, s;
+    for (int nl = 0; nl < iterations; nl++) {
+        t = (real_t) 0.;
+        for (int i = 0; i < LEN_1D; i++) {
+            s = b[i] * c[i];
+            a[i] = s + t;
+            t = s;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s252, "s252", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s253.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s253.c
new file mode 100644
index 0000000..b14cb57
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s253.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s253(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    scalar expansio assigned under if
+
+    initialise_arrays(__func__);
+
+    real_t s;
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (a[i] > b[i]) {
+                s = a[i] - b[i] * d[i];
+                c[i] += s;
+                a[i] = s;
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s253, "s253", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s254.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s254.c
new file mode 100644
index 0000000..bdc8a01
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s254.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s254(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    carry around variable
+
+    initialise_arrays(__func__);
+
+    real_t x;
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        x = b[LEN_1D-1];
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = (b[i] + x) * (real_t).5;
+            x = b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s254, "s254", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s255.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s255.c
new file mode 100644
index 0000000..c680ee1
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s255.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s255(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    carry around variables, 2 levels
+
+    initialise_arrays(__func__);
+
+    real_t x, y;
+    for (int nl = 0; nl < iterations; nl++) {
+        x = b[LEN_1D-1];
+        y = b[LEN_1D-2];
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = (b[i] + x + y) * (real_t).333;
+            y = x;
+            x = b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s255, "s255", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s256.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s256.c
new file mode 100644
index 0000000..2e18fa8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s256.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s256(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    array expansion
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 10*(iterations/LEN_2D); nl++) {
+        for (int i = 0; i < LEN_2D; i++) {
+            for (int j = 1; j < LEN_2D; j++) {
+                a[j] = (real_t)1.0 - a[j - 1];
+                aa[j][i] = a[j] + bb[j][i]*d[j];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s256, "s256", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s257.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s257.c
new file mode 100644
index 0000000..c6fd86c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s257.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s257(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    array expansion
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 10*(iterations/LEN_2D); nl++) {
+        for (int i = 1; i < LEN_2D; i++) {
+            for (int j = 0; j < LEN_2D; j++) {
+                a[i] = aa[j][i] - a[i-1];
+                aa[j][i] = a[i] + bb[j][i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s257, "s257", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s258.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s258.c
new file mode 100644
index 0000000..34f16a0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s258.c
@@ -0,0 +1,45 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s258(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    wrap-around scalar under an if
+
+    initialise_arrays(__func__);
+
+    real_t s;
+    for (int nl = 0; nl < iterations; nl++) {
+        s = 0.;
+        for (int i = 0; i < LEN_2D; ++i) {
+            if (a[i] > 0.) {
+                s = d[i] * d[i];
+            }
+            b[i] = s * c[i] + d[i];
+            e[i] = (s + (real_t)1.) * aa[0][i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s258, "s258", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s261.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s261.c
new file mode 100644
index 0000000..6edb1a5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s261.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s261(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    wrap-around scalar under an if
+
+    initialise_arrays(__func__);
+
+    real_t t;
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 1; i < LEN_1D; ++i) {
+            t = a[i] + b[i];
+            a[i] = t + c[i-1];
+            t = c[i] * d[i];
+            c[i] = t;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s261, "s261", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s271.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s271.c
new file mode 100644
index 0000000..a454a4d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s271.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s271(struct args_t * func_args)
+{
+//    control flow
+//    loop with singularity handling
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (b[i] > (real_t)0.) {
+                a[i] += b[i] * c[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s271, "s271", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2710.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2710.c
new file mode 100644
index 0000000..19d8484
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2710.c
@@ -0,0 +1,55 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s2710(struct args_t * func_args)
+{
+//    control flow
+//    scalar and vector ifs
+
+    int x = *(int*)func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations/2; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (a[i] > b[i]) {
+                a[i] += b[i] * d[i];
+                if (LEN_1D > 10) {
+                    c[i] += d[i] * d[i];
+                } else {
+                    c[i] = d[i] * e[i] + (real_t)1.;
+                }
+            } else {
+                b[i] = a[i] + e[i] * e[i];
+                if (x > (real_t)0.) {
+                    c[i] = a[i] + d[i] * d[i];
+                } else {
+                    c[i] += e[i] * e[i];
+                }
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s2710, "s2710", &s1);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2711.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2711.c
new file mode 100644
index 0000000..1e6ae03
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2711.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s2711(struct args_t * func_args)
+{
+//    control flow
+//    semantic if removal
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (b[i] != (real_t)0.0) {
+                a[i] += b[i] * c[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s2711, "s2711", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2712.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2712.c
new file mode 100644
index 0000000..f958522
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s2712.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s2712(struct args_t * func_args)
+{
+//    control flow
+//    if to elemental min
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (a[i] >= b[i]) {
+                a[i] += b[i] * c[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s2712, "s2712", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s272.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s272.c
new file mode 100644
index 0000000..cca9adf
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s272.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s272(struct args_t * func_args)
+{
+//    control flow
+//    loop with independent conditional
+
+    int t = *(int*)func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (e[i] >= t) {
+                a[i] += c[i] * d[i];
+                b[i] += c[i] * c[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s272, "s272", &s1);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s273.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s273.c
new file mode 100644
index 0000000..9c75f8d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s273.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s273(struct args_t * func_args)
+{
+//    control flow
+//    simple loop with dependent conditional
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] += d[i] * e[i];
+            if (a[i] < (real_t)0.)
+                b[i] += d[i] * e[i];
+            c[i] += a[i] * d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s273, "s273", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s274.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s274.c
new file mode 100644
index 0000000..9e0260c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s274.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s274(struct args_t * func_args)
+{
+//    control flow
+//    complex loop with dependent conditional
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = c[i] + e[i] * d[i];
+            if (a[i] > (real_t)0.) {
+                b[i] = a[i] + b[i];
+            } else {
+                a[i] = d[i] * e[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s274, "s274", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s275.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s275.c
new file mode 100644
index 0000000..929cf11
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s275.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s275(struct args_t * func_args)
+{
+//    control flow
+//    if around inner loop, interchanging needed
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 10*(iterations/LEN_2D); nl++) {
+        for (int i = 0; i < LEN_2D; i++) {
+            if (aa[0][i] > (real_t)0.) {
+                for (int j = 1; j < LEN_2D; j++) {
+                    aa[j][i] = aa[j-1][i] + bb[j][i] * cc[j][i];
+                }
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s275, "s275", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s276.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s276.c
new file mode 100644
index 0000000..4f62403
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s276.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s276(struct args_t * func_args)
+{
+//    control flow
+//    if test using loop index
+
+    initialise_arrays(__func__);
+
+    int mid = (LEN_1D/2);
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (i+1 < mid) {
+                a[i] += b[i] * c[i];
+            } else {
+                a[i] += b[i] * d[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s276, "s276", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s277.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s277.c
new file mode 100644
index 0000000..5904b63
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s277.c
@@ -0,0 +1,49 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s277(struct args_t * func_args)
+{
+//    control flow
+//    test for dependences arising from guard variable computation.
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D-1; i++) {
+                if (a[i] >= (real_t)0.) {
+                    goto L20;
+                }
+                if (b[i] >= (real_t)0.) {
+                    goto L30;
+                }
+                a[i] += c[i] * d[i];
+L30:
+                b[i+1] = c[i] + d[i] * e[i];
+L20:
+;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s277, "s277", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s278.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s278.c
new file mode 100644
index 0000000..98d7c08
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s278.c
@@ -0,0 +1,47 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s278(struct args_t * func_args)
+{
+//    control flow
+//    if/goto to block if-then-else
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (a[i] > (real_t)0.) {
+                goto L20;
+            }
+            b[i] = -b[i] + d[i] * e[i];
+            goto L30;
+L20:
+            c[i] = -c[i] + d[i] * e[i];
+L30:
+            a[i] = b[i] + c[i] * d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s278, "s278", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s279.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s279.c
new file mode 100644
index 0000000..6f2bfe2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s279.c
@@ -0,0 +1,51 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s279(struct args_t * func_args)
+{
+//    control flow
+//    vector if/gotos
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations/2; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (a[i] > (real_t)0.) {
+                goto L20;
+            }
+            b[i] = -b[i] + d[i] * d[i];
+            if (b[i] <= a[i]) {
+                goto L30;
+            }
+            c[i] += d[i] * e[i];
+            goto L30;
+L20:
+            c[i] = -c[i] + e[i] * e[i];
+L30:
+            a[i] = b[i] + c[i] * d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s279, "s279", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s281.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s281.c
new file mode 100644
index 0000000..d60f2ea
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s281.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s281(struct args_t * func_args)
+{
+//    crossing thresholds
+//    index set splitting
+//    reverse data access
+
+    initialise_arrays(__func__);
+
+    real_t x;
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            x = a[LEN_1D-i-1] + b[i] * c[i];
+            a[i] = x-(real_t)1.0;
+            b[i] = x;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s281, "s281", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s291.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s291.c
new file mode 100644
index 0000000..0b474c2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s291.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s291(struct args_t * func_args)
+{
+//    loop peeling
+//    wrap around variable, 1 level
+
+    initialise_arrays(__func__);
+
+    int im1;
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        im1 = LEN_1D-1;
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = (b[i] + b[im1]) * (real_t).5;
+            im1 = i;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s291, "s291", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s292.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s292.c
new file mode 100644
index 0000000..1745875
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s292.c
@@ -0,0 +1,45 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s292(struct args_t * func_args)
+{
+//    loop peeling
+//    wrap around variable, 2 levels
+//    similar to S291
+
+    initialise_arrays(__func__);
+
+    int im1, im2;
+    for (int nl = 0; nl < iterations; nl++) {
+        im1 = LEN_1D-1;
+        im2 = LEN_1D-2;
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = (b[i] + b[im1] + b[im2]) * (real_t).333;
+            im2 = im1;
+            im1 = i;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s292, "s292", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s293.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s293.c
new file mode 100644
index 0000000..3213948
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s293.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s293(struct args_t * func_args)
+{
+//    loop peeling
+//    a(i)=a(0) with actual dependence cycle, loop is vectorizable
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = a[0];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s293, "s293", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s311.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s311.c
new file mode 100644
index 0000000..11f5ee7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s311.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s311(struct args_t * func_args)
+{
+//    reductions
+//    sum reduction
+
+    initialise_arrays(__func__);
+
+    real_t sum;
+    for (int nl = 0; nl < iterations*10; nl++) {
+        sum = (real_t)0.;
+        for (int i = 0; i < LEN_1D; i++) {
+            sum += a[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, sum);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s311, "s311", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3110.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3110.c
new file mode 100644
index 0000000..b8b2fa8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3110.c
@@ -0,0 +1,52 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s3110(struct args_t * func_args)
+{
+//    reductions
+//    if to max with index reductio 2 dimensions
+//    similar to S315
+
+    initialise_arrays(__func__);
+
+    int xindex, yindex;
+    real_t max, chksum;
+    for (int nl = 0; nl < 100*(iterations/(LEN_2D)); nl++) {
+        max = aa[(0)][0];
+        xindex = 0;
+        yindex = 0;
+        for (int i = 0; i < LEN_2D; i++) {
+            for (int j = 0; j < LEN_2D; j++) {
+                if (aa[i][j] > max) {
+                    max = aa[i][j];
+                    xindex = i;
+                    yindex = j;
+                }
+            }
+        }
+        chksum = max + (real_t) xindex + (real_t) yindex;
+        dummy(a, b, c, d, e, aa, bb, cc, chksum);
+    }
+
+    return max + xindex+1 + yindex+1;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s3110, "s3110", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3111.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3111.c
new file mode 100644
index 0000000..4ff1166
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3111.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s3111(struct args_t * func_args)
+{
+//    reductions
+//    conditional sum reduction
+
+    initialise_arrays(__func__);
+
+    real_t sum;
+    for (int nl = 0; nl < iterations/2; nl++) {
+        sum = 0.;
+        for (int i = 0; i < LEN_1D; i++) {
+            if (a[i] > (real_t)0.) {
+                sum += a[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, sum);
+    }
+
+    return sum;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s3111, "s31111", NULL);
+
+  return 0;
+}
+
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s31111.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s31111.c
new file mode 100644
index 0000000..ad21513
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s31111.c
@@ -0,0 +1,54 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t test(real_t* A){
+  real_t s = (real_t)0.0;
+  for (int i = 0; i < 4; i++)
+    s += A[i];
+  return s;
+}
+
+real_t s31111(struct args_t * func_args)
+{
+//    reductions
+//    sum reduction
+
+    initialise_arrays(__func__);
+
+    real_t sum;
+    for (int nl = 0; nl < 2000*iterations; nl++) {
+        sum = (real_t)0.;
+        sum += test(a);
+        sum += test(&a[4]);
+        sum += test(&a[8]);
+        sum += test(&a[12]);
+        sum += test(&a[16]);
+        sum += test(&a[20]);
+        sum += test(&a[24]);
+        sum += test(&a[28]);
+        dummy(a, b, c, d, e, aa, bb, cc, sum);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s31111, "s31111", NULL);
+
+  return 0;
+}
+
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3112.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3112.c
new file mode 100644
index 0000000..ec81ad8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3112.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s3112(struct args_t * func_args)
+{
+//    reductions
+//    sum reduction saving running sums
+
+    initialise_arrays(__func__);
+
+    real_t sum;
+    for (int nl = 0; nl < iterations; nl++) {
+        sum = (real_t)0.0;
+        for (int i = 0; i < LEN_1D; i++) {
+            sum += a[i];
+            b[i] = sum;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, sum);
+    }
+
+    return sum;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s3112, "s3112", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3113.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3113.c
new file mode 100644
index 0000000..bff35ca
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3113.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s3113(struct args_t * func_args)
+{
+//    reductions
+//    maximum of absolute value
+
+    initialise_arrays(__func__);
+
+    real_t max;
+    for (int nl = 0; nl < iterations*4; nl++) {
+        max = ABS(a[0]);
+        for (int i = 0; i < LEN_1D; i++) {
+            if ((ABS(a[i])) > max) {
+                max = ABS(a[i]);
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, max);
+    }
+
+    return max;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s3113, "s3113", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s312.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s312.c
new file mode 100644
index 0000000..ab0571a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s312.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s312(struct args_t * func_args)
+{
+//    reductions
+//    product reduction
+
+    initialise_arrays(__func__);
+
+    real_t prod;
+    for (int nl = 0; nl < 10*iterations; nl++) {
+        prod = (real_t)1.;
+        for (int i = 0; i < LEN_1D; i++) {
+            prod *= a[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, prod);
+    }
+
+    return prod;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s312, "s312", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s313.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s313.c
new file mode 100644
index 0000000..e58a539
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s313.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s313(struct args_t * func_args)
+{
+//    reductions
+//    dot product
+
+    initialise_arrays(__func__);
+
+    real_t dot;
+    for (int nl = 0; nl < iterations*5; nl++) {
+        dot = (real_t)0.;
+        for (int i = 0; i < LEN_1D; i++) {
+            dot += a[i] * b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, dot);
+    }
+
+    return dot;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s313, "s313", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s314.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s314.c
new file mode 100644
index 0000000..87dd706
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s314.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s314(struct args_t * func_args)
+{
+//    reductions
+//    if to max reduction
+
+    initialise_arrays(__func__);
+
+    real_t x;
+    for (int nl = 0; nl < iterations*5; nl++) {
+        x = a[0];
+        for (int i = 0; i < LEN_1D; i++) {
+            if (a[i] > x) {
+                x = a[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, x);
+    }
+
+    return x;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s314, "s314", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s315.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s315.c
new file mode 100644
index 0000000..8d734a0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s315.c
@@ -0,0 +1,50 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s315(struct args_t * func_args)
+{
+//    reductions
+//    if to max with index reductio 1 dimension
+
+    initialise_arrays(__func__);
+
+    for (int i = 0; i < LEN_1D; i++)
+        a[i] = (i * 7) % LEN_1D;
+
+    real_t x, chksum;
+    int index;
+    for (int nl = 0; nl < iterations; nl++) {
+        x = a[0];
+        index = 0;
+        for (int i = 0; i < LEN_1D; ++i) {
+            if (a[i] > x) {
+                x = a[i];
+                index = i;
+            }
+        }
+        chksum = x + (real_t) index;
+        dummy(a, b, c, d, e, aa, bb, cc, chksum);
+    }
+
+    return index + x + 1;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s315, "s315", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s316.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s316.c
new file mode 100644
index 0000000..4a711bc
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s316.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s316(struct args_t * func_args)
+{
+//    reductions
+//    if to min reduction
+
+    initialise_arrays(__func__);
+
+    real_t x;
+    for (int nl = 0; nl < iterations*5; nl++) {
+        x = a[0];
+        for (int i = 1; i < LEN_1D; ++i) {
+            if (a[i] < x) {
+                x = a[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, x);
+    }
+
+    return x;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s316, "s316", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s317.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s317.c
new file mode 100644
index 0000000..126b452
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s317.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s317(struct args_t * func_args)
+{
+//    reductions
+//    product reductio vectorize with
+//    1. scalar expansion of factor, and product reduction
+//    2. closed form solution: q = factor**n
+
+    initialise_arrays(__func__);
+
+    real_t q;
+    for (int nl = 0; nl < 5*iterations; nl++) {
+        q = (real_t)1.;
+        for (int i = 0; i < LEN_1D/2; i++) {
+            q *= (real_t).99;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, q);
+    }
+
+    return q;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s317, "s317", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s318.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s318.c
new file mode 100644
index 0000000..f4e1a4b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s318.c
@@ -0,0 +1,54 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s318(struct args_t * func_args)
+{
+//    reductions
+//    isamax, max absolute value, increments not equal to 1
+
+    int inc = *(int*)func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    int k, index;
+    real_t max, chksum;
+    for (int nl = 0; nl < iterations/2; nl++) {
+        k = 0;
+        index = 0;
+        max = ABS(a[0]);
+        k += inc;
+        for (int i = 1; i < LEN_1D; i++) {
+            if (ABS(a[k]) <= max) {
+                goto L5;
+            }
+            index = i;
+            max = ABS(a[k]);
+L5:
+            k += inc;
+        }
+        chksum = max + (real_t) index;
+        dummy(a, b, c, d, e, aa, bb, cc, chksum);
+    }
+
+    return max + index + 1;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s318, "s318", &n1);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s319.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s319.c
new file mode 100644
index 0000000..dbb0bd3
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s319.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s319(struct args_t * func_args)
+{
+//    reductions
+//    coupled reductions
+
+    initialise_arrays(__func__);
+
+    real_t sum;
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        sum = 0.;
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = c[i] + d[i];
+            sum += a[i];
+            b[i] = c[i] + e[i];
+            sum += b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, sum);
+    }
+
+    return sum;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s319, "s319", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s321.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s321.c
new file mode 100644
index 0000000..3011e56
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s321.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s321(struct args_t * func_args)
+{
+//    recurrences
+//    first order linear recurrence
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 1; i < LEN_1D; i++) {
+            a[i] += a[i-1] * b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s321, "s321", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s322.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s322.c
new file mode 100644
index 0000000..075e811
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s322.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s322(struct args_t * func_args)
+{
+//    recurrences
+//    second order linear recurrence
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations/2; nl++) {
+        for (int i = 2; i < LEN_1D; i++) {
+            a[i] = a[i] + a[i - 1] * b[i] + a[i - 2] * c[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s322, "s322", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s323.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s323.c
new file mode 100644
index 0000000..c95345d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s323.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s323(struct args_t * func_args)
+{
+//    recurrences
+//    coupled recurrence
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations/2; nl++) {
+        for (int i = 1; i < LEN_1D; i++) {
+            a[i] = b[i-1] + c[i] * d[i];
+            b[i] = a[i] + c[i] * e[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s323, "s323", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3251.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3251.c
new file mode 100644
index 0000000..51ca173
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s3251.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s3251(struct args_t * func_args)
+{
+//    scalar and array expansion
+//    scalar expansion
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D-1; i++){
+            a[i+1] = b[i]+c[i];
+            b[i]   = c[i]*e[i];
+            d[i]   = a[i]*e[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s3251, "s3251", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s331.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s331.c
new file mode 100644
index 0000000..e580df7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s331.c
@@ -0,0 +1,45 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s331(struct args_t * func_args)
+{
+//    search loops
+//    if to last-1
+
+    initialise_arrays(__func__);
+
+    int j;
+    real_t chksum;
+    for (int nl = 0; nl < iterations; nl++) {
+        j = -1;
+        for (int i = 0; i < LEN_1D; i++) {
+            if (a[i] < (real_t)0.) {
+                j = i;
+            }
+        }
+        chksum = (real_t) j;
+        dummy(a, b, c, d, e, aa, bb, cc, chksum);
+    }
+
+    return j+1;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s331, "s331", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s332.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s332.c
new file mode 100644
index 0000000..3fd490b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s332.c
@@ -0,0 +1,52 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s332(struct args_t * func_args)
+{
+//    search loops
+//    first value greater than threshold
+
+    int t = *(int*)func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    int index;
+    real_t value;
+    real_t chksum;
+    for (int nl = 0; nl < iterations; nl++) {
+        index = -2;
+        value = -1.;
+        for (int i = 0; i < LEN_1D; i++) {
+            if (a[i] > t) {
+                index = i;
+                value = a[i];
+                goto L20;
+            }
+        }
+L20:
+        chksum = value + (real_t) index;
+        dummy(a, b, c, d, e, aa, bb, cc, chksum);
+    }
+
+    return value;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s332, "s332", &s1);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s341.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s341.c
new file mode 100644
index 0000000..2aa2fa5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s341.c
@@ -0,0 +1,45 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s341(struct args_t * func_args)
+{
+//    packing
+//    pack positive values
+//    not vectorizable, value of j in unknown at each iteration
+
+    initialise_arrays(__func__);
+
+    int j;
+    for (int nl = 0; nl < iterations; nl++) {
+        j = -1;
+        for (int i = 0; i < LEN_1D; i++) {
+            if (b[i] > (real_t)0.) {
+                j++;
+                a[j] = b[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s341, "s341", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s342.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s342.c
new file mode 100644
index 0000000..a2af1c7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s342.c
@@ -0,0 +1,45 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s342(struct args_t * func_args)
+{
+//    packing
+//    unpacking
+//    not vectorizable, value of j in unknown at each iteration
+
+    initialise_arrays(__func__);
+
+    int j = 0;
+    for (int nl = 0; nl < iterations; nl++) {
+        j = -1;
+        for (int i = 0; i < LEN_1D; i++) {
+            if (a[i] > (real_t)0.) {
+                j++;
+                a[i] = b[j];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s342, "s342", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s343.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s343.c
new file mode 100644
index 0000000..d76c882
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s343.c
@@ -0,0 +1,47 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s343(struct args_t * func_args)
+{
+//    packing
+//    pack 2-d array into one dimension
+//    not vectorizable, value of k in unknown at each iteration
+
+    initialise_arrays(__func__);
+
+    int k;
+    for (int nl = 0; nl < 10*(iterations/LEN_2D); nl++) {
+        k = -1;
+        for (int i = 0; i < LEN_2D; i++) {
+            for (int j = 0; j < LEN_2D; j++) {
+                if (bb[j][i] > (real_t)0.) {
+                    k++;
+                    flat_2d_array[k] = aa[j][i];
+                }
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s343, "s343", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s351.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s351.c
new file mode 100644
index 0000000..b5875d1
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s351.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s351(struct args_t * func_args)
+{
+//    loop rerolling
+//    unrolled saxpy
+
+    initialise_arrays(__func__);
+
+    real_t alpha = c[0];
+    for (int nl = 0; nl < 8*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i += 5) {
+            a[i] += alpha * b[i];
+            a[i + 1] += alpha * b[i + 1];
+            a[i + 2] += alpha * b[i + 2];
+            a[i + 3] += alpha * b[i + 3];
+            a[i + 4] += alpha * b[i + 4];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s351, "s351", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s352.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s352.c
new file mode 100644
index 0000000..11d2412
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s352.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s352(struct args_t * func_args)
+{
+//    loop rerolling
+//    unrolled dot product
+
+    initialise_arrays(__func__);
+
+    real_t dot;
+    for (int nl = 0; nl < 8*iterations; nl++) {
+        dot = (real_t)0.;
+        for (int i = 0; i < LEN_1D; i += 5) {
+            dot = dot + a[i] * b[i] + a[i + 1] * b[i + 1] + a[i + 2]
+                * b[i + 2] + a[i + 3] * b[i + 3] + a[i + 4] * b[i + 4];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, dot);
+    }
+
+    return dot;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s352, "s352", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s353.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s353.c
new file mode 100644
index 0000000..5889858
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s353.c
@@ -0,0 +1,47 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s353(struct args_t * func_args)
+{
+//    loop rerolling
+//    unrolled sparse saxpy
+//    gather is required
+
+    int * __restrict__ ip = func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    real_t alpha = c[0];
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i += 5) {
+            a[i] += alpha * b[ip[i]];
+            a[i + 1] += alpha * b[ip[i + 1]];
+            a[i + 2] += alpha * b[ip[i + 2]];
+            a[i + 3] += alpha * b[ip[i + 3]];
+            a[i + 4] += alpha * b[ip[i + 4]];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s353, "s353", ip);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4112.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4112.c
new file mode 100644
index 0000000..bca9a1b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4112.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s4112(struct args_t * func_args)
+{
+//    indirect addressing
+//    sparse saxpy
+//    gather is required
+
+    struct{int * __restrict__ a;real_t b;} * x = func_args->arg_info;
+    int * __restrict__ ip = x->a;
+    real_t s = x->b;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] += b[ip[i]] * s;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s4112, "s4112", &(struct{int*a;real_t b;}){ip, s1});
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4113.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4113.c
new file mode 100644
index 0000000..ce08d98
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4113.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s4113(struct args_t * func_args)
+{
+//    indirect addressing
+//    indirect addressing on rhs and lhs
+//    gather and scatter is required
+
+    int * __restrict__ ip = func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[ip[i]] = b[ip[i]] + c[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s4113, "s4113", ip);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4114.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4114.c
new file mode 100644
index 0000000..b305900
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4114.c
@@ -0,0 +1,47 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s4114(struct args_t * func_args)
+{
+//    indirect addressing
+//    mix indirect addressing with variable lower and upper bounds
+//    gather is required
+
+    struct{int * __restrict__ a;int b;} * x = func_args->arg_info;
+    int * __restrict__ ip = x->a;
+    int n1 = x->b;
+
+    initialise_arrays(__func__);
+
+    int k;
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = n1-1; i < LEN_1D; i++) {
+            k = ip[i];
+            a[i] = b[i] + c[LEN_1D-k+1-2] * d[i];
+            k += 5;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s4114, "s4114", &(struct{int*a;int b;}){ip, n1});
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4115.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4115.c
new file mode 100644
index 0000000..f85e75e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4115.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s4115(struct args_t * func_args)
+{
+//    indirect addressing
+//    sparse dot product
+//    gather is required
+
+    int * __restrict__ ip = func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    real_t sum;
+    for (int nl = 0; nl < iterations; nl++) {
+        sum = 0.;
+        for (int i = 0; i < LEN_1D; i++) {
+            sum += a[i] * b[ip[i]];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return sum;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s4115, "s4115", ip);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4116.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4116.c
new file mode 100644
index 0000000..95a771e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4116.c
@@ -0,0 +1,49 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s4116(struct args_t * func_args)
+{
+//    indirect addressing
+//    more complicated sparse sdot
+//    gather is required
+
+    struct{int * __restrict__ a;int b;int c;} * x = func_args->arg_info;
+    int * __restrict__ ip = x->a;
+    int j = x->b;
+    int inc = x->c;
+
+    initialise_arrays(__func__);
+
+    real_t sum;
+    int off;
+    for (int nl = 0; nl < 100*iterations; nl++) {
+        sum = 0.;
+        for (int i = 0; i < LEN_2D-1; i++) {
+            off = inc + i;
+            sum += a[off] * aa[j-1][ip[i]];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return sum;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s4116, "s4116", &(struct{int * a; int b; int c;}){ip, LEN_2D/2, n1});
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4117.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4117.c
new file mode 100644
index 0000000..a1d757b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4117.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s4117(struct args_t * func_args)
+{
+//    indirect addressing
+//    seq function
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = b[i] + c[i/2] * d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s4117, "s4117", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4121.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4121.c
new file mode 100644
index 0000000..54aa9d0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s4121.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t f(real_t a, real_t b){
+    return a*b;
+}
+
+real_t s4121(struct args_t * func_args)
+{
+//    statement functions
+//    elementwise multiplication
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] += f(b[i],c[i]);
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s4121, "s4121", NULL);
+
+  return 0;
+}
+
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s421.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s421.c
new file mode 100644
index 0000000..e6c00c5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s421.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s421(struct args_t * func_args)
+{
+//    storage classes and equivalencing
+//    equivalence- no overlap
+
+    initialise_arrays(__func__);
+
+    xx = flat_2d_array;
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        yy = xx;
+        for (int i = 0; i < LEN_1D - 1; i++) {
+            xx[i] = yy[i+1] + a[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 1.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s421, "s421", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s422.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s422.c
new file mode 100644
index 0000000..4d491da
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s422.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s422(struct args_t * func_args)
+{
+//    storage classes and equivalencing
+//    common and equivalence statement
+//    anti-dependence, threshold of 4
+
+    initialise_arrays(__func__);
+
+    xx = flat_2d_array + 4;
+
+    for (int nl = 0; nl < 8*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            xx[i] = flat_2d_array[i + 8] + a[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s422, "s422", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s423.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s423.c
new file mode 100644
index 0000000..7605f05
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s423.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s423(struct args_t * func_args)
+{
+//    storage classes and equivalencing
+//    common and equivalenced variables - with anti-dependence
+
+    // do this again here
+    int vl = 64;
+    xx = flat_2d_array + vl;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D - 1; i++) {
+            flat_2d_array[i+1] = xx[i] + a[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 1.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s423, "s423", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s424.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s424.c
new file mode 100644
index 0000000..211f2d4
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s424.c
@@ -0,0 +1,44 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s424(struct args_t * func_args)
+{
+//    storage classes and equivalencing
+//    common and equivalenced variables - overlap
+//    vectorizeable in strips of 64 or less
+
+    // do this again here
+    int vl = 63;
+    xx = flat_2d_array + vl;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D - 1; i++) {
+            xx[i+1] = flat_2d_array[i] + a[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 1.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s424, "s424", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s431.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s431.c
new file mode 100644
index 0000000..f2b0358
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s431.c
@@ -0,0 +1,43 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s431(struct args_t * func_args)
+{
+//    parameters
+//    parameter statement
+
+    int k1=1;
+    int k2=2;
+    int k=2*k1-k2;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations*10; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = a[i+k] + b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s431, "s431", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s441.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s441.c
new file mode 100644
index 0000000..deab041
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s441.c
@@ -0,0 +1,45 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s441(struct args_t * func_args)
+{
+//    non-logical if's
+//    arithmetic if
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (d[i] < (real_t)0.) {
+                a[i] += b[i] * c[i];
+            } else if (d[i] == (real_t)0.) {
+                a[i] += b[i] * b[i];
+            } else {
+                a[i] += c[i] * c[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s441, "s441", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s442.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s442.c
new file mode 100644
index 0000000..22fad60
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s442.c
@@ -0,0 +1,57 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s442(struct args_t * func_args)
+{
+//    non-logical if's
+//    computed goto
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations/2; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            switch (indx[i]) {
+                case 1:  goto L15;
+                case 2:  goto L20;
+                case 3:  goto L30;
+                case 4:  goto L40;
+            }
+L15:
+            a[i] += b[i] * b[i];
+            goto L50;
+L20:
+            a[i] += c[i] * c[i];
+            goto L50;
+L30:
+            a[i] += d[i] * d[i];
+            goto L50;
+L40:
+            a[i] += e[i] * e[i];
+L50:
+            ;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s442, "s442", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s443.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s443.c
new file mode 100644
index 0000000..2134cc5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s443.c
@@ -0,0 +1,50 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s443(struct args_t * func_args)
+{
+//    non-logical if's
+//    arithmetic if
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (d[i] <= (real_t)0.) {
+                goto L20;
+            } else {
+                goto L30;
+            }
+L20:
+            a[i] += b[i] * c[i];
+            goto L50;
+L30:
+            a[i] += b[i] * b[i];
+L50:
+            ;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s443, "s443", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s451.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s451.c
new file mode 100644
index 0000000..ac0c9ac
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s451.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s451(struct args_t * func_args)
+{
+//    intrinsic functions
+//    intrinsics
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations/5; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = sinf(b[i]) + cosf(c[i]);
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s451, "s451", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s452.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s452.c
new file mode 100644
index 0000000..fe97c4d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s452.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s452(struct args_t * func_args)
+{
+//    intrinsic functions
+//    seq function
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = b[i] + c[i] * (real_t) (i+1);
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s452, "s452", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s453.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s453.c
new file mode 100644
index 0000000..1fc1d66
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s453.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s453(struct args_t * func_args)
+{
+//    induction varibale recognition
+
+    real_t s;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations*2; nl++) {
+        s = 0.;
+        for (int i = 0; i < LEN_1D; i++) {
+            s += (real_t)2.;
+            a[i] = s * b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s453, "s453", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s471.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s471.c
new file mode 100644
index 0000000..b842dda
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s471.c
@@ -0,0 +1,49 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+int s471s(void)
+{
+// --  dummy subroutine call made in s471
+    return 0;
+}
+
+real_t s471(struct args_t * func_args){
+
+//    call statements
+
+    int m = LEN_1D;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations/2; nl++) {
+        for (int i = 0; i < m; i++) {
+            x[i] = b[i] + d[i] * d[i];
+            s471s();
+            b[i] = c[i] + d[i] * e[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s471, "s471", NULL);
+
+  return 0;
+}
+
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s481.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s481.c
new file mode 100644
index 0000000..bf98e17
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s481.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s481(struct args_t * func_args)
+{
+//    non-local goto's
+//    stop statement
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (d[i] < (real_t)0.) {
+                exit (0);
+            }
+            a[i] += b[i] * c[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s481, "s481", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s482.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s482.c
new file mode 100644
index 0000000..c4e2680
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s482.c
@@ -0,0 +1,40 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s482(struct args_t * func_args)
+{
+//    non-local goto's
+//    other loop exit with code before exit
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] += b[i] * c[i];
+            if (c[i] > b[i]) break;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s482, "s482", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s491.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s491.c
new file mode 100644
index 0000000..89ac780
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-s491.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t s491(struct args_t * func_args)
+{
+//    vector semantics
+//    indirect addressing on lhs, store in sequence
+//    scatter is required
+
+    int * __restrict__ ip = func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[ip[i]] = b[i] + c[i] * d[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&s491, "s491", ip);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-va.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-va.c
new file mode 100644
index 0000000..441b5bd
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-va.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t va(struct args_t * func_args)
+{
+//    control loops
+//    vector assignment
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations*10; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&va, "va", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vag.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vag.c
new file mode 100644
index 0000000..9213b27
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vag.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vag(struct args_t * func_args)
+{
+//    control loops
+//    vector assignment, gather
+//    gather is required
+
+    int * __restrict__ ip = func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = b[ip[i]];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vag, "vag", ip);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vas.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vas.c
new file mode 100644
index 0000000..46ec99b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vas.c
@@ -0,0 +1,42 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vas(struct args_t * func_args)
+{
+//    control loops
+//    vector assignment, scatter
+//    scatter is required
+
+    int * __restrict__ ip = func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 2*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[ip[i]] = b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vas, "vas", ip);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vbor.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vbor.c
new file mode 100644
index 0000000..801410d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vbor.c
@@ -0,0 +1,54 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vbor(struct args_t * func_args)
+{
+//    control loops
+//    basic operations rates, isolate arithmetic from memory traffic
+//    all combinations of three, 59 flops for 6 loads and 1 store.
+
+    initialise_arrays(__func__);
+
+    real_t a1, b1, c1, d1, e1, f1;
+    for (int nl = 0; nl < iterations*10; nl++) {
+        for (int i = 0; i < LEN_2D; i++) {
+            a1 = a[i];
+            b1 = b[i];
+            c1 = c[i];
+            d1 = d[i];
+            e1 = e[i];
+            f1 = aa[0][i];
+            a1 = a1 * b1 * c1 + a1 * b1 * d1 + a1 * b1 * e1 + a1 * b1 * f1 +
+                a1 * c1 * d1 + a1 * c1 * e1 + a1 * c1 * f1 + a1 * d1 * e1
+                + a1 * d1 * f1 + a1 * e1 * f1;
+            b1 = b1 * c1 * d1 + b1 * c1 * e1 + b1 * c1 * f1 + b1 * d1 * e1 +
+                b1 * d1 * f1 + b1 * e1 * f1;
+            c1 = c1 * d1 * e1 + c1 * d1 * f1 + c1 * e1 * f1;
+            d1 = d1 * e1 * f1;
+            x[i] = a1 * b1 * c1 * d1;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vbor, "vbor", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vdotr.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vdotr.c
new file mode 100644
index 0000000..80ed9f8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vdotr.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vdotr(struct args_t * func_args)
+{
+//    control loops
+//    vector dot product reduction
+
+    initialise_arrays(__func__);
+
+    real_t dot;
+    for (int nl = 0; nl < iterations*10; nl++) {
+        dot = 0.;
+        for (int i = 0; i < LEN_1D; i++) {
+            dot += a[i] * b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, dot);
+    }
+
+    return dot;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vdotr, "vdotr", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vif.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vif.c
new file mode 100644
index 0000000..152e6c4
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vif.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vif(struct args_t * func_args)
+{
+//    control loops
+//    vector if
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            if (b[i] > (real_t)0.) {
+                a[i] = b[i];
+            }
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vif, "vif", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" { xfail *-*-* } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpv.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpv.c
new file mode 100644
index 0000000..232904f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpv.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vpv(struct args_t * func_args)
+{
+//    control loops
+//    vector plus vector
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations*10; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] += b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vpv, "vpv", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpvpv.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpvpv.c
new file mode 100644
index 0000000..b4d0850
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpvpv.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vpvpv(struct args_t * func_args)
+{
+//    control loops
+//    vector plus vector plus vector
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] += b[i] + c[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vpvpv, "vpvpv", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpvts.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpvts.c
new file mode 100644
index 0000000..f193afa
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpvts.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vpvts(struct args_t * func_args)
+{
+//    control loops
+//    vector plus vector times scalar
+
+    real_t s = *(int*)func_args->arg_info;
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] += b[i] * s;
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vpvts, "vpvts", &s1);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpvtv.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpvtv.c
new file mode 100644
index 0000000..b726b27
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vpvtv.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vpvtv(struct args_t * func_args)
+{
+//    control loops
+//    vector plus vector times vector
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] += b[i] * c[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vpvtv, "vpvtv", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vsumr.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vsumr.c
new file mode 100644
index 0000000..0847eae
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vsumr.c
@@ -0,0 +1,41 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vsumr(struct args_t * func_args)
+{
+//    control loops
+//    vector sum reduction
+
+    initialise_arrays(__func__);
+
+    real_t sum;
+    for (int nl = 0; nl < iterations*10; nl++) {
+        sum = 0.;
+        for (int i = 0; i < LEN_1D; i++) {
+            sum += a[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, sum);
+    }
+
+    return sum;
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vsumr, "vsumr", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vtv.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vtv.c
new file mode 100644
index 0000000..d3457d4
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vtv.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vtv(struct args_t * func_args)
+{
+//    control loops
+//    vector times vector
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < iterations*10; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] *= b[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vtv, "vtv", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vtvtv.c b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vtvtv.c
new file mode 100644
index 0000000..32148ec
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/tsvc/vect-tsvc-vtvtv.c
@@ -0,0 +1,39 @@
+/*  This file is distributed under the University of Illinois Open Source
+    License. See license.txt for details.  */
+
+/* { dg-additional-options "--param vect-epilogues-nomask=0" } */
+/* { dg-require-effective-target vect_float } */
+
+#include "tsvc.h"
+
+real_t vtvtv(struct args_t * func_args)
+{
+//    control loops
+//    vector times vector times vector
+
+    initialise_arrays(__func__);
+
+    for (int nl = 0; nl < 4*iterations; nl++) {
+        for (int i = 0; i < LEN_1D; i++) {
+            a[i] = a[i] * b[i] * c[i];
+        }
+        dummy(a, b, c, d, e, aa, bb, cc, 0.);
+    }
+
+    return calc_checksum(__func__);
+}
+
+int main (int argc, char **argv)
+{
+  int n1 = 1;
+  int n3 = 1;
+  int* ip;
+  real_t s1,s2;
+  init(&ip, &s1, &s2);
+
+  run(&vtvtv, "vtvtv", NULL);
+
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.dg/vect/vect.exp b/gcc/testsuite/gcc.dg/vect/vect.exp
index dca9a4d..7e33398 100644
--- a/gcc/testsuite/gcc.dg/vect/vect.exp
+++ b/gcc/testsuite/gcc.dg/vect/vect.exp
@@ -95,6 +95,9 @@ foreach flags $VECT_ADDITIONAL_FLAGS {
 	[glob -nocomplain $srcdir/$subdir/vect-*.\[cS\]]] \
 	$flags $DEFAULT_VECTCFLAGS
     et-dg-runtest dg-runtest [lsort \
+	[glob -nocomplain $srcdir/$subdir/tsvc/*.\[cS\]]] \
+	$flags $DEFAULT_VECTCFLAGS
+    et-dg-runtest dg-runtest [lsort \
 	[glob -nocomplain $srcdir/$subdir/slp-*.\[cS\]]] \
 	$flags $DEFAULT_VECTCFLAGS
     et-dg-runtest dg-runtest [lsort \
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/bf16_vldN_lane_2.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/bf16_vldN_lane_2.c
index 670cf0b..87b5fc3 100644
--- a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/bf16_vldN_lane_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/bf16_vldN_lane_2.c
@@ -17,7 +17,7 @@ test_vld2q_lane_bf16 (const bfloat16_t *ptr, bfloat16x8x2_t b)
   return vld2q_lane_bf16 (ptr, b, 2);
 }
 
-/* { dg-final { scan-assembler-times "ld2\\t{v2.h - v3.h}\\\[2\\\], \\\[x0\\\]" 2 } } */
+/* { dg-final { scan-assembler-times "ld2\\t{v\[0-9\]+.h - v\[0-9\]+.h}\\\[2\\\], \\\[x0\\\]" 2 } } */
 
 bfloat16x4x3_t
 test_vld3_lane_bf16 (const bfloat16_t *ptr, bfloat16x4x3_t b)
@@ -25,15 +25,13 @@ test_vld3_lane_bf16 (const bfloat16_t *ptr, bfloat16x4x3_t b)
   return vld3_lane_bf16 (ptr, b, 2);
 }
 
-/* { dg-final { scan-assembler-times "ld3\t{v4.h - v6.h}\\\[2\\\], \\\[x0\\\]" 1 } } */
-
 bfloat16x8x3_t
 test_vld3q_lane_bf16 (const bfloat16_t *ptr, bfloat16x8x3_t b)
 {
   return vld3q_lane_bf16 (ptr, b, 2);
 }
 
-/* { dg-final { scan-assembler-times "ld3\t{v1.h - v3.h}\\\[2\\\], \\\[x0\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "ld3\t{v\[0-9\]+.h - v\[0-9\]+.h}\\\[2\\\], \\\[x0\\\]" 2 } } */
 
 bfloat16x4x4_t
 test_vld4_lane_bf16 (const bfloat16_t *ptr, bfloat16x4x4_t b)
@@ -41,12 +39,10 @@ test_vld4_lane_bf16 (const bfloat16_t *ptr, bfloat16x4x4_t b)
   return vld4_lane_bf16 (ptr, b, 2);
 }
 
-/* { dg-final { scan-assembler-times "ld4\t{v4.h - v7.h}\\\[2\\\], \\\[x0\\\]" 1 } } */
-
 bfloat16x8x4_t
 test_vld4q_lane_bf16 (const bfloat16_t *ptr, bfloat16x8x4_t b)
 {
   return vld4q_lane_bf16 (ptr, b, 2);
 }
 
-/* { dg-final { scan-assembler-times "ld4\t{v0.h - v3.h}\\\[2\\\], \\\[x0\\\]" 1 } } */
+/* { dg-final { scan-assembler-times "ld4\t{v\[0-9\]+.h - v\[0-9\]+.h}\\\[2\\\], \\\[x0\\\]" 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-1.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-1.c
new file mode 100644
index 0000000..6baed3a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-1.c
@@ -0,0 +1,11 @@
+/* { dg-do assemble { target aarch64*-*-* } } */
+/* { dg-additional-options "--save-temps" } */
+
+#include <arm_neon.h>
+
+uint8x8_t foo (uint8x8_t a)
+{
+  return vshr_n_u8 (a, 2);
+}
+
+/* { dg-final { scan-assembler-times {\tushr\t.+, 2} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-2.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-2.c
new file mode 100644
index 0000000..df46ae3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-2.c
@@ -0,0 +1,11 @@
+/* { dg-do assemble { target aarch64*-*-* } } */
+/* { dg-additional-options "--save-temps" } */
+
+#include <arm_neon.h>
+
+int16x8_t foo (int16x8_t a)
+{
+  return vshrq_n_s16 (a, 8);
+}
+
+/* { dg-final { scan-assembler-times {\tsshr\t.+, 8} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-3.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-3.c
new file mode 100644
index 0000000..86fcc7a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-3.c
@@ -0,0 +1,11 @@
+/* { dg-do assemble { target aarch64*-*-* } } */
+/* { dg-additional-options "--save-temps" } */
+
+#include <arm_neon.h>
+
+int16x8_t foo (int16x8_t a)
+{
+  return vshrq_n_s16 (a, 16);
+}
+
+/* { dg-final { scan-assembler-times {\tsshr\t.+, 16} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-4.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-4.c
new file mode 100644
index 0000000..11f8fec
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-4.c
@@ -0,0 +1,11 @@
+/* { dg-do assemble { target aarch64*-*-* } } */
+/* { dg-additional-options "--save-temps" } */
+
+#include <arm_neon.h>
+
+int64x1_t foo (int64x1_t a)
+{
+  return vshl_s64 (a, vdup_n_s64(80));
+}
+
+/* { dg-final { scan-assembler-times {\tsshl\t.+, d[0-9]+} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-5.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-5.c
new file mode 100644
index 0000000..e140e5f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-5.c
@@ -0,0 +1,12 @@
+/* { dg-do assemble { target aarch64*-*-* } } */
+/* { dg-additional-options "--save-temps" } */
+/* { dg-skip-if "no optimizations" { *-*-* } { "-O0" } { "" } } */
+
+#include <arm_neon.h>
+
+int64x1_t foo (int64x1_t a)
+{
+  return vshl_s64 (a, vdup_n_s64(-6));
+}
+
+/* { dg-final { scan-assembler-times {\tsshr\t.+, 6} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-6.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-6.c
new file mode 100644
index 0000000..442e316
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-6.c
@@ -0,0 +1,10 @@
+/* { dg-do assemble { target aarch64*-*-* } } */
+/* { dg-additional-options "--save-temps" } */
+
+#include <arm_neon.h>
+
+int32x4_t foo (int32x4_t x) {
+  return vshlq_s32(x, vdupq_n_s32(256));
+}
+
+/* { dg-final { scan-assembler-times {\tsshl\t.+, v[0-9].4s} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-7.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-7.c
new file mode 100644
index 0000000..20b4920
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-7.c
@@ -0,0 +1,12 @@
+/* { dg-do assemble { target aarch64*-*-* } } */
+/* { dg-additional-options "--save-temps" } */
+/* { dg-skip-if "no optimizations" { *-*-* } { "-O0" } { "" } } */
+
+#include <arm_neon.h>
+
+int32x4_t foo (int32x4_t x) {
+  return vshlq_s32(vdupq_n_s32(1), vdupq_n_s32(10));
+}
+
+/* { dg-final { scan-assembler-not {\tsshl\t} } } */
+/* { dg-final { scan-assembler-times {\tmovi\t} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-8.c b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-8.c
new file mode 100644
index 0000000..f586aa7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/advsimd-intrinsics/vshl-opt-8.c
@@ -0,0 +1,10 @@
+/* { dg-do assemble { target aarch64*-*-* } } */
+/* { dg-additional-options "--save-temps" } */
+
+#include <arm_neon.h>
+
+int32x4_t foo (int32x4_t x) {
+  return vshlq_s32(x, vdupq_n_s32(-64));
+}
+
+/* { dg-final { scan-assembler-times {\tsshl\t.+, v[0-9]+.4s} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/auto-init-6.c b/gcc/testsuite/gcc.target/aarch64/auto-init-6.c
index 27c16b3..0456c66 100644
--- a/gcc/testsuite/gcc.target/aarch64/auto-init-6.c
+++ b/gcc/testsuite/gcc.target/aarch64/auto-init-6.c
@@ -1,6 +1,6 @@
 /* Verify pattern initialization for complex type automatic variables.  */
 /* { dg-do compile } */
-/* { dg-options "-ftrivial-auto-var-init=pattern -fdump-rtl-expand" } */
+/* { dg-options "-ftrivial-auto-var-init=pattern" } */
 
 
 _Complex long double result;
@@ -15,4 +15,4 @@ _Complex long double foo()
   return result;
 }
 
-/* { dg-final { scan-rtl-dump-times "0xfffffffffffffffe\\\]\\\) repeated x16" 3 "expand" } } */
+/* { dg-final { scan-assembler-times "word\t-16843010" 14  } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c b/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
index adb787a..c1aeb06 100644
--- a/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/fmla_intrinsic_1.c
@@ -107,10 +107,12 @@ main (int argc, char **argv)
 
 /* vfma_lane_f64.
    vfma_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmadd\\td\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+" 2 } } */
+/* { dg-final { scan-assembler-times "fmadd\\td\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+" 1 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times "fmadd\\td\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+" 2 { target aarch64_little_endian } } } */
 
 /* vfmaq_lane_f64.
    vfmaq_laneq_f64.  */
-/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d, v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d, v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 3 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times "fmla\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d, v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 { target aarch64_little_endian } } } */
 
 
diff --git a/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c b/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
index 865def2..3137ea9 100644
--- a/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/fmls_intrinsic_1.c
@@ -108,10 +108,12 @@ main (int argc, char **argv)
 
 /* vfms_lane_f64.
    vfms_laneq_f64.  */
-/* { dg-final { scan-assembler-times "fmsub\\td\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+" 2 } } */
+/* { dg-final { scan-assembler-times "fmsub\\td\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+" 1 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times "fmsub\\td\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+\, d\[0-9\]+" 2 { target aarch64_little_endian } } } */
 
 /* vfmsq_lane_f64.
    vfmsq_laneq_f64.  */
-/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d, v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d, v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 3 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times "fmls\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d, v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 { target aarch64_little_endian } } } */
 
 
diff --git a/gcc/testsuite/gcc.target/aarch64/fmul_intrinsic_1.c b/gcc/testsuite/gcc.target/aarch64/fmul_intrinsic_1.c
index d01095e..7d4829c 100644
--- a/gcc/testsuite/gcc.target/aarch64/fmul_intrinsic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/fmul_intrinsic_1.c
@@ -107,10 +107,12 @@ main (int argc, char **argv)
 
 /* vmul_lane_f64.
    Vmul_laneq_f64. */
-/* { dg-final { scan-assembler-times "fmul\\td\[0-9\]+, d\[0-9\]+, d\[0-9\]+" 2 } } */
+/* { dg-final { scan-assembler-times "fmul\\td\[0-9\]+, d\[0-9\]+, d\[0-9\]+" 1 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times "fmul\\td\[0-9\]+, d\[0-9\]+, d\[0-9\]+" 2 { target aarch64_little_endian } } } */
 
 /* vmulq_lane_f64.
    vmulq_laneq_f64.  */
-/* { dg-final { scan-assembler-times "fmul\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d, v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 } } */
+/* { dg-final { scan-assembler-times "fmul\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d, v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 3 { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler-times "fmul\\tv\[0-9\]+\.2d, v\[0-9\]+\.2d, v\[0-9\]+\.d\\\[\[0-9\]+\\\]" 2 { target aarch64_little_endian } } } */
 
 
diff --git a/gcc/testsuite/gcc.target/aarch64/neoverse_v1_1.c b/gcc/testsuite/gcc.target/aarch64/neoverse_v1_1.c
new file mode 100644
index 0000000..c1563f0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/neoverse_v1_1.c
@@ -0,0 +1,15 @@
+/* { dg-options "-O2 -mcpu=neoverse-v1" } */
+
+void
+foo (short *restrict x, short y[restrict][128])
+{
+  for (int i = 0; i < 128; ++i)
+    {
+      y[0][i] = x[i * 3 + 0];
+      y[1][i] = x[i * 3 + 1];
+      y[2][i] = x[i * 3 + 2];
+    }
+}
+
+/* This shouldn't be a post-increment.  */
+/* { dg-final { scan-assembler {ld3\t{[^{}]*}, \[x[0-9]+\]\n} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr102376.c b/gcc/testsuite/gcc.target/aarch64/pr102376.c
new file mode 100644
index 0000000..fc830ad
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr102376.c
@@ -0,0 +1,3 @@
+/* { dg-do compile } */
+
+void calculate(void) __attribute__ ((__target__ ("sve"))); /* { dg-error "arch extension 'sve' should be prefixed by '\\+'" } */
diff --git a/gcc/testsuite/gcc.target/aarch64/pr103085.c b/gcc/testsuite/gcc.target/aarch64/pr103085.c
new file mode 100644
index 0000000..dbc9c15
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr103085.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fstack-protector-strong -fPIC" } */
+
+void g(int*);
+void
+f (int x)
+{
+  int arr[10];
+  g (arr);
+}
+
diff --git a/gcc/testsuite/gcc.target/aarch64/shrn-combine-1.c b/gcc/testsuite/gcc.target/aarch64/shrn-combine-1.c
index a285246..334e94a 100644
--- a/gcc/testsuite/gcc.target/aarch64/shrn-combine-1.c
+++ b/gcc/testsuite/gcc.target/aarch64/shrn-combine-1.c
@@ -1,6 +1,8 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
 
+#pragma GCC target "+nosve"
+
 #define TYPE char
 
 void foo (unsigned TYPE * restrict a, TYPE * restrict d, int n)
diff --git a/gcc/testsuite/gcc.target/aarch64/shrn-combine-2.c b/gcc/testsuite/gcc.target/aarch64/shrn-combine-2.c
index 012135b..c90de72 100644
--- a/gcc/testsuite/gcc.target/aarch64/shrn-combine-2.c
+++ b/gcc/testsuite/gcc.target/aarch64/shrn-combine-2.c
@@ -1,6 +1,8 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
 
+#pragma GCC target "+nosve"
+
 #define TYPE short
 
 void foo (unsigned TYPE * restrict a, TYPE * restrict d, int n)
diff --git a/gcc/testsuite/gcc.target/aarch64/shrn-combine-3.c b/gcc/testsuite/gcc.target/aarch64/shrn-combine-3.c
index 8b5b360..a05ecbb 100644
--- a/gcc/testsuite/gcc.target/aarch64/shrn-combine-3.c
+++ b/gcc/testsuite/gcc.target/aarch64/shrn-combine-3.c
@@ -1,6 +1,8 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
 
+#pragma GCC target "+nosve"
+
 #define TYPE int
 
 void foo (unsigned long long * restrict a, TYPE * restrict d, int n)
diff --git a/gcc/testsuite/gcc.target/aarch64/shrn-combine-4.c b/gcc/testsuite/gcc.target/aarch64/shrn-combine-4.c
index fedca76..36ebab7 100644
--- a/gcc/testsuite/gcc.target/aarch64/shrn-combine-4.c
+++ b/gcc/testsuite/gcc.target/aarch64/shrn-combine-4.c
@@ -1,6 +1,8 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
 
+#pragma GCC target "+nosve"
+
 #define TYPE long long
 
 void foo (unsigned TYPE * restrict a, TYPE * restrict d, int n)
diff --git a/gcc/testsuite/gcc.target/aarch64/shrn-combine-5.c b/gcc/testsuite/gcc.target/aarch64/shrn-combine-5.c
index 408e8553..973e577 100644
--- a/gcc/testsuite/gcc.target/aarch64/shrn-combine-5.c
+++ b/gcc/testsuite/gcc.target/aarch64/shrn-combine-5.c
@@ -1,6 +1,8 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
 
+#pragma GCC target "+nosve"
+
 #define TYPE1 char
 #define TYPE2 short
 #define SHIFT 8
diff --git a/gcc/testsuite/gcc.target/aarch64/shrn-combine-6.c b/gcc/testsuite/gcc.target/aarch64/shrn-combine-6.c
index 6211ba3..db36a9c 100644
--- a/gcc/testsuite/gcc.target/aarch64/shrn-combine-6.c
+++ b/gcc/testsuite/gcc.target/aarch64/shrn-combine-6.c
@@ -1,6 +1,8 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
 
+#pragma GCC target "+nosve"
+
 #define TYPE1 short
 #define TYPE2 int
 #define SHIFT 16
diff --git a/gcc/testsuite/gcc.target/aarch64/shrn-combine-7.c b/gcc/testsuite/gcc.target/aarch64/shrn-combine-7.c
index 56cbeac..e7caf3c 100644
--- a/gcc/testsuite/gcc.target/aarch64/shrn-combine-7.c
+++ b/gcc/testsuite/gcc.target/aarch64/shrn-combine-7.c
@@ -1,6 +1,8 @@
 /* { dg-do assemble } */
 /* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
 
+#pragma GCC target "+nosve"
+
 #define TYPE1 int
 #define TYPE2 long long
 #define SHIFT 32
diff --git a/gcc/testsuite/gcc.target/aarch64/shrn-combine-8.c b/gcc/testsuite/gcc.target/aarch64/shrn-combine-8.c
index 6a47f3c..c93c179 100644
--- a/gcc/testsuite/gcc.target/aarch64/shrn-combine-8.c
+++ b/gcc/testsuite/gcc.target/aarch64/shrn-combine-8.c
@@ -6,7 +6,7 @@
 
 uint8x16_t foo (uint16x8_t a, uint16x8_t b)
 {
-  return vrshrn_high_n_u16 (vrshrn_n_u16 (a, 8), b, 8);
+  return vshrn_high_n_u16 (vshrn_n_u16 (a, 8), b, 8);
 }
 
 /* { dg-final { scan-assembler-times {\tuzp2\t} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/shrn-combine-9.c b/gcc/testsuite/gcc.target/aarch64/shrn-combine-9.c
index 929a55c..bdb3c13 100644
--- a/gcc/testsuite/gcc.target/aarch64/shrn-combine-9.c
+++ b/gcc/testsuite/gcc.target/aarch64/shrn-combine-9.c
@@ -6,7 +6,7 @@
 
 uint16x8_t foo (uint32x4_t a, uint32x4_t b)
 {
-  return vrshrn_high_n_u32 (vrshrn_n_u32 (a, 16), b, 16);
+  return vshrn_high_n_u32 (vshrn_n_u32 (a, 16), b, 16);
 }
 
 /* { dg-final { scan-assembler-times {\tuzp2\t} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/signbit-1.c b/gcc/testsuite/gcc.target/aarch64/signbit-1.c
new file mode 100644
index 0000000..a5654d1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/signbit-1.c
@@ -0,0 +1,20 @@
+/* { dg-do assemble } */
+/* { dg-options "-O3 --save-temps" } */
+
+#include <stdint.h>
+
+#pragma GCC target "+nosve"
+
+void fun1(int32_t *x, int n)
+{
+    for (int i = 0; i < (n & -16); i++)
+      x[i] = (-x[i]) >> 31;
+}
+
+void fun2(int32_t *x, int n)
+{
+    for (int i = 0; i < (n & -16); i++)
+      x[i] = (-x[i]) >> 30;
+}
+
+/* { dg-final { scan-assembler-times {\tcmgt\t} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/signbit-2.c b/gcc/testsuite/gcc.target/aarch64/signbit-2.c
new file mode 100644
index 0000000..e4e9afc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/signbit-2.c
@@ -0,0 +1,36 @@
+/* { dg-do assemble } */
+/* { dg-options "-O1 --save-temps" } */
+
+#include <arm_neon.h>
+
+int32x2_t foo1 (int32x2_t a)
+{
+  return vshr_n_s32 (vneg_s32 (a), 31);
+}
+
+int32x4_t foo2 (int32x4_t a)
+{
+  return vshrq_n_s32 (vnegq_s32 (a), 31);
+}
+
+int16x8_t foo3 (int16x8_t a)
+{
+  return vshrq_n_s16 (vnegq_s16 (a), 15);
+}
+
+int16x4_t foo4 (int16x4_t a)
+{
+  return vshr_n_s16 (vneg_s16 (a), 15);
+}
+
+int8x16_t foo5 (int8x16_t a)
+{
+  return vshrq_n_s8 (vnegq_s8 (a), 7);
+}
+
+int8x8_t foo6 (int8x8_t a)
+{
+  return vshr_n_s8 (vneg_s8 (a), 7);
+}
+
+/* { dg-final { scan-assembler-times {\tcmgt\t} 6 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/lowering_tbaa.c b/gcc/testsuite/gcc.target/aarch64/simd/lowering_tbaa.c
new file mode 100644
index 0000000..eaeae21
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/lowering_tbaa.c
@@ -0,0 +1,30 @@
+/* Tests the TBAA information of lowered AArch64 SIMD loads.  */
+/* { dg-do run } */
+/* { dg-options "-save-temps -O2" } */
+
+#include <arm_neon.h>
+
+void __attribute__((noipa))
+g (float *)
+{
+}
+
+int32x4_t __attribute__((noipa))
+f (void)
+{
+  float a[4] = { 1, 2, 3, 4 };
+  g (a);
+  a[0] = a[1] = a[2] = a[3] = 0;
+  void *volatile ptr = a;
+  return vld1q_s32 ((int32_t *) ptr);
+}
+
+int
+main (void)
+{
+  int32x4_t x = f ();
+  int32x4_t y = vdupq_n_s32 (0);
+  if (__builtin_memcmp (&x, &y, 16) != 0)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/stp_1.c b/gcc/testsuite/gcc.target/aarch64/stp_1.c
new file mode 100644
index 0000000..c1a5e6b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/stp_1.c
@@ -0,0 +1,29 @@
+/* { dg-options "-O -fpeephole2" } */
+
+int res[2];
+
+void
+f1 (int x, int y)
+{
+  res[0] = res[1] = x + y;
+}
+
+void
+f2 (int x, int y, int *res)
+{
+  res[0] = res[1] = x + y;
+}
+
+void
+f3 (int x, int y)
+{
+  res[1] = res[0] = x + y;
+}
+
+void
+f4 (int x, int y, int *res)
+{
+  res[1] = res[0] = x + y;
+}
+
+/* { dg-final { scan-assembler-times {\tstp\t} 4 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pcs/struct_3_256.c b/gcc/testsuite/gcc.target/aarch64/sve/pcs/struct_3_256.c
index fdfbec5..af0af29 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pcs/struct_3_256.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pcs/struct_3_256.c
@@ -933,7 +933,11 @@ SEL2 (struct, nonpst1)
 
 /*
 ** test_nonpst1:
+** (
+**	fmov	d0, d3
+** |
 **	mov	v0\.8b, v3\.8b
+** )
 **	ret
 */
 /* { dg-final { scan-assembler-not {\t\.variant_pcs\ttest_nonpst1\n} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-1.c b/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-1.c
index 2c06564..c9a8b82 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-1.c
@@ -1,5 +1,5 @@
-/* { dg-do assemble { target aarch64_asm_sve_ok } } */
-/* { dg-options "-O3 --save-temps" } */
+/* { dg-do compile } */
+/* { dg-options "-O3" } */
 
 /*
 ** f10:
@@ -21,3 +21,4 @@ void f10(double * restrict z, double * restrict w, double * restrict x, double *
 
 /* { dg-final { scan-assembler-not {\tbic\t} } } */
 /* { dg-final { scan-assembler-times {\tnot\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmgt\tp[0-9]+\.d, p[0-9]+/z, z[0-9]+\.d, #0} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-2.c b/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-2.c
index 0c3b78d..8d78f47 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-2.c
@@ -1,5 +1,5 @@
-/* { dg-do assemble { target aarch64_asm_sve_ok } } */
-/* { dg-options "-O3 --save-temps" } */
+/* { dg-do compile } */
+/* { dg-options "-O3" } */
 
 /*
 ** f11:
@@ -21,3 +21,4 @@ void f11(double * restrict z, double * restrict w, double * restrict x, double *
 
 /* { dg-final { scan-assembler-not {\tbic\t} } } */
 /* { dg-final { scan-assembler-times {\tnot\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmgt\tp[0-9]+\.d, p[0-9]+/z, z[0-9]+\.d, #0.0} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-3.c b/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-3.c
index 248f8ab..38fbfef 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-3.c
@@ -1,5 +1,5 @@
-/* { dg-do assemble { target aarch64_asm_sve_ok } } */
-/* { dg-options "-O3 --save-temps" } */
+/* { dg-do compile } */
+/* { dg-options "-O3" } */
 
 /*
 ** f12:
@@ -19,3 +19,4 @@ void f12(int * restrict z, int * restrict w, int * restrict x, int * restrict y,
 
 /* { dg-final { scan-assembler-not {\tbic\t} } } */
 /* { dg-final { scan-assembler-not {\tnot\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b\n} } } */
+/* { dg-final { scan-assembler-times {\tcmple\tp[0-9]+\.s, p[0-9]+/z, z[0-9]+\.s, #0} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-4.c b/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-4.c
index 9620030..0001dd3 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-4.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pred-not-gen-4.c
@@ -1,5 +1,5 @@
-/* { dg-do assemble { target aarch64_asm_sve_ok } } */
-/* { dg-options "-O3 --save-temps" } */
+/* { dg-do compile } */
+/* { dg-options "-O3" } */
 
 #include <math.h>
 
@@ -12,3 +12,4 @@ void f13(double * restrict z, double * restrict w, double * restrict x, double *
 
 /* { dg-final { scan-assembler-not {\tbic\t} } } */
 /* { dg-final { scan-assembler-times {\tnot\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmuo\tp[0-9]+\.d, p[0-9]+/z, z[0-9]+\.d, z[0-9]+\.d} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-cse-codegen.c b/gcc/testsuite/gcc.target/aarch64/vect-cse-codegen.c
new file mode 100644
index 0000000..d025e98
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-cse-codegen.c
@@ -0,0 +1,97 @@
+/* { dg-do compile  { target { lp64 } } } */
+/* { dg-additional-options "-O3 -march=armv8.2-a+crypto -fno-schedule-insns -fno-schedule-insns2 -mcmodel=small" } */
+/* { dg-final { check-function-bodies "**" "" "" { target { le } } } } */
+
+#include <arm_neon.h>
+
+/*
+**test1:
+**	adrp	x[0-9]+, .LC[0-9]+
+**	ldr	q[0-9]+, \[x[0-9]+, #:lo12:.LC[0-9]+\]
+**	add	v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d
+**	str	q[0-9]+, \[x[0-9]+\]
+**	fmov	x[0-9]+, d[0-9]+
+**	orr	x[0-9]+, x[0-9]+, x[0-9]+
+**	ret
+*/
+
+uint64_t
+test1 (uint64_t a, uint64x2_t b, uint64x2_t* rt)
+{
+  uint64_t arr[2] = { 0x0942430810234076UL, 0x0942430810234076UL};
+  uint64_t res = a | arr[0];
+  uint64x2_t val = vld1q_u64 (arr);
+  *rt = vaddq_u64 (val, b);
+  return res;
+}
+
+/*
+**test2:
+**	adrp	x[0-9]+, .LC[0-1]+
+**	ldr	q[0-9]+, \[x[0-9]+, #:lo12:.LC[0-9]+\]
+**	add	v[0-9]+.2d, v[0-9]+.2d, v[0-9]+.2d
+**	str	q[0-9]+, \[x[0-9]+\]
+**	fmov	x[0-9]+, d[0-9]+
+**	orr	x[0-9]+, x[0-9]+, x[0-9]+
+**	ret
+*/
+
+uint64_t
+test2 (uint64_t a, uint64x2_t b, uint64x2_t* rt)
+{
+  uint64x2_t val = vdupq_n_u64 (0x0424303242234076UL);
+  uint64_t arr = vgetq_lane_u64 (val, 0);
+  uint64_t res = a | arr;
+  *rt = vaddq_u64 (val, b);
+  return res;
+}
+
+/*
+**test3:
+**	adrp	x[0-9]+, .LC[0-9]+
+**	ldr	q[0-9]+, \[x[0-9]+, #:lo12:.LC[0-9]+\]
+**	add	v[0-9]+.4s, v[0-9]+.4s, v[0-9]+.4s
+**	str	q[0-9]+, \[x1\]
+**	fmov	w[0-9]+, s[0-9]+
+**	orr	w[0-9]+, w[0-9]+, w[0-9]+
+**	ret
+*/
+
+uint32_t
+test3 (uint32_t a, uint32x4_t b, uint32x4_t* rt)
+{
+  uint32_t arr[4] = { 0x094243, 0x094243, 0x094243, 0x094243 };
+  uint32_t res = a | arr[0];
+  uint32x4_t val = vld1q_u32 (arr);
+  *rt = vaddq_u32 (val, b);
+  return res;
+}
+
+/*
+**test4:
+**	ushr	v[0-9]+.16b, v[0-9]+.16b, 7
+**	mov	x[0-9]+, 16512
+**	movk	x[0-9]+, 0x1020, lsl 16
+**	movk	x[0-9]+, 0x408, lsl 32
+**	movk	x[0-9]+, 0x102, lsl 48
+**	fmov	d[0-9]+, x[0-9]+
+**	pmull	v[0-9]+.1q, v[0-9]+.1d, v[0-9]+.1d
+**	dup	v[0-9]+.2d, v[0-9]+.d\[0\]
+**	pmull2	v[0-9]+.1q, v[0-9]+.2d, v[0-9]+.2d
+**	trn2	v[0-9]+.8b, v[0-9]+.8b, v[0-9]+.8b
+**	umov	w[0-9]+, v[0-9]+.h\[3\]
+**	ret
+*/
+
+uint64_t
+test4 (uint8x16_t input)
+{
+    uint8x16_t bool_input = vshrq_n_u8(input, 7);
+    poly64x2_t mask = vdupq_n_p64(0x0102040810204080UL);
+    poly64_t prodL = vmull_p64((poly64_t)vgetq_lane_p64((poly64x2_t)bool_input, 0),
+                               vgetq_lane_p64(mask, 0));
+    poly64_t prodH = vmull_high_p64((poly64x2_t)bool_input, mask);
+    uint8x8_t res = vtrn2_u8((uint8x8_t)prodL, (uint8x8_t)prodH);
+    return vget_lane_u16((uint16x4_t)res, 3);
+}
+
diff --git a/gcc/testsuite/gcc.target/aarch64/vector_structure_intrinsics.c b/gcc/testsuite/gcc.target/aarch64/vector_structure_intrinsics.c
index 89e9de1..100739a 100644
--- a/gcc/testsuite/gcc.target/aarch64/vector_structure_intrinsics.c
+++ b/gcc/testsuite/gcc.target/aarch64/vector_structure_intrinsics.c
@@ -17,6 +17,14 @@ TEST_TBL (vqtbl2q, int8x16_t, int8x16x2_t, uint8x16_t, s8)
 TEST_TBL (vqtbl2q, uint8x16_t, uint8x16x2_t, uint8x16_t, u8)
 TEST_TBL (vqtbl2q, poly8x16_t, poly8x16x2_t, uint8x16_t, p8)
 
+TEST_TBL (vqtbl3, int8x8_t, int8x16x3_t, uint8x8_t, s8)
+TEST_TBL (vqtbl3, uint8x8_t, uint8x16x3_t, uint8x8_t, u8)
+TEST_TBL (vqtbl3, poly8x8_t, poly8x16x3_t, uint8x8_t, p8)
+
+TEST_TBL (vqtbl3q, int8x16_t, int8x16x3_t, uint8x16_t, s8)
+TEST_TBL (vqtbl3q, uint8x16_t, uint8x16x3_t, uint8x16_t, u8)
+TEST_TBL (vqtbl3q, poly8x16_t, poly8x16x3_t, uint8x16_t, p8)
+
 TEST_TBL (vqtbl4, int8x8_t, int8x16x4_t, uint8x8_t, s8)
 TEST_TBL (vqtbl4, uint8x8_t, uint8x16x4_t, uint8x8_t, u8)
 TEST_TBL (vqtbl4, poly8x8_t, poly8x16x4_t, uint8x8_t, p8)
@@ -25,62 +33,35 @@ TEST_TBL (vqtbl4q, int8x16_t, int8x16x4_t, uint8x16_t, s8)
 TEST_TBL (vqtbl4q, uint8x16_t, uint8x16x4_t, uint8x16_t, u8)
 TEST_TBL (vqtbl4q, poly8x16_t, poly8x16x4_t, uint8x16_t, p8)
 
-#define TEST_TBL3(name, rettype, tbltype, idxtype, ts) \
-  rettype test_ ## name ## _ ## ts (idxtype a, tbltype b) \
-	{ \
-		return name ## _ ## ts (b, a); \
-	}
-
-TEST_TBL3 (vqtbl3, int8x8_t, int8x16x3_t, uint8x8_t, s8)
-TEST_TBL3 (vqtbl3, uint8x8_t, uint8x16x3_t, uint8x8_t, u8)
-TEST_TBL3 (vqtbl3, poly8x8_t, poly8x16x3_t, uint8x8_t, p8)
-
-TEST_TBL3 (vqtbl3q, int8x16_t, int8x16x3_t, uint8x16_t, s8)
-TEST_TBL3 (vqtbl3q, uint8x16_t, uint8x16x3_t, uint8x16_t, u8)
-TEST_TBL3 (vqtbl3q, poly8x16_t, poly8x16x3_t, uint8x16_t, p8)
-
-#define TEST_TBX2(name, rettype, tbltype, idxtype, ts) \
-  rettype test_ ## name ## _ ## ts (rettype a, idxtype b, tbltype c) \
-	{ \
-		return name ## _ ## ts (a, c, b); \
-	}
-
-TEST_TBX2 (vqtbx2, int8x8_t, int8x16x2_t, uint8x8_t, s8)
-TEST_TBX2 (vqtbx2, uint8x8_t, uint8x16x2_t, uint8x8_t, u8)
-TEST_TBX2 (vqtbx2, poly8x8_t, poly8x16x2_t, uint8x8_t, p8)
-
-TEST_TBX2 (vqtbx2q, int8x16_t, int8x16x2_t, uint8x16_t, s8)
-TEST_TBX2 (vqtbx2q, uint8x16_t, uint8x16x2_t, uint8x16_t, u8)
-TEST_TBX2 (vqtbx2q, poly8x16_t, poly8x16x2_t, uint8x16_t, p8)
-
-#define TEST_TBX3(name, rettype, tbltype, idxtype, ts) \
+#define TEST_TBX(name, rettype, tbltype, idxtype, ts) \
   rettype test_ ## name ## _ ## ts (rettype a, tbltype b, idxtype c) \
 	{ \
 		return name ## _ ## ts (a, b, c); \
 	}
 
-TEST_TBX3 (vqtbx3, int8x8_t, int8x16x3_t, uint8x8_t, s8)
-TEST_TBX3 (vqtbx3, uint8x8_t, uint8x16x3_t, uint8x8_t, u8)
-TEST_TBX3 (vqtbx3, poly8x8_t, poly8x16x3_t, uint8x8_t, p8)
+TEST_TBX (vqtbx2, int8x8_t, int8x16x2_t, uint8x8_t, s8)
+TEST_TBX (vqtbx2, uint8x8_t, uint8x16x2_t, uint8x8_t, u8)
+TEST_TBX (vqtbx2, poly8x8_t, poly8x16x2_t, uint8x8_t, p8)
 
-TEST_TBX3 (vqtbx3q, int8x16_t, int8x16x3_t, uint8x16_t, s8)
-TEST_TBX3 (vqtbx3q, uint8x16_t, uint8x16x3_t, uint8x16_t, u8)
-TEST_TBX3 (vqtbx3q, poly8x16_t, poly8x16x3_t, uint8x16_t, p8)
+TEST_TBX (vqtbx2q, int8x16_t, int8x16x2_t, uint8x16_t, s8)
+TEST_TBX (vqtbx2q, uint8x16_t, uint8x16x2_t, uint8x16_t, u8)
+TEST_TBX (vqtbx2q, poly8x16_t, poly8x16x2_t, uint8x16_t, p8)
 
-#define TEST_TBX4(name, rettype, tbltype, idxtype, ts) \
-  rettype test_ ## name ## _ ## ts (rettype a, idxtype b, idxtype dummy1, \
-				    idxtype dummy2, tbltype c) \
-	{ \
-		return name ## _ ## ts (a, c, b); \
-	}
+TEST_TBX (vqtbx3, int8x8_t, int8x16x3_t, uint8x8_t, s8)
+TEST_TBX (vqtbx3, uint8x8_t, uint8x16x3_t, uint8x8_t, u8)
+TEST_TBX (vqtbx3, poly8x8_t, poly8x16x3_t, uint8x8_t, p8)
+
+TEST_TBX (vqtbx3q, int8x16_t, int8x16x3_t, uint8x16_t, s8)
+TEST_TBX (vqtbx3q, uint8x16_t, uint8x16x3_t, uint8x16_t, u8)
+TEST_TBX (vqtbx3q, poly8x16_t, poly8x16x3_t, uint8x16_t, p8)
 
-TEST_TBX4 (vqtbx4, int8x8_t, int8x16x4_t, uint8x8_t, s8)
-TEST_TBX4 (vqtbx4, uint8x8_t, uint8x16x4_t, uint8x8_t, u8)
-TEST_TBX4 (vqtbx4, poly8x8_t, poly8x16x4_t, uint8x8_t, p8)
+TEST_TBX (vqtbx4, int8x8_t, int8x16x4_t, uint8x8_t, s8)
+TEST_TBX (vqtbx4, uint8x8_t, uint8x16x4_t, uint8x8_t, u8)
+TEST_TBX (vqtbx4, poly8x8_t, poly8x16x4_t, uint8x8_t, p8)
 
-TEST_TBX4 (vqtbx4q, int8x16_t, int8x16x4_t, uint8x16_t, s8)
-TEST_TBX4 (vqtbx4q, uint8x16_t, uint8x16x4_t, uint8x16_t, u8)
-TEST_TBX4 (vqtbx4q, poly8x16_t, poly8x16x4_t, uint8x16_t, p8)
+TEST_TBX (vqtbx4q, int8x16_t, int8x16x4_t, uint8x16_t, s8)
+TEST_TBX (vqtbx4q, uint8x16_t, uint8x16x4_t, uint8x16_t, u8)
+TEST_TBX (vqtbx4q, poly8x16_t, poly8x16x4_t, uint8x16_t, p8)
 
 #define TEST_STX(name, tbltype, ptrtype, ts) \
   void test_ ## name ## _ ## ts (ptrtype a, tbltype b) \
@@ -88,21 +69,21 @@ TEST_TBX4 (vqtbx4q, poly8x16_t, poly8x16x4_t, uint8x16_t, p8)
 		name ## _ ## ts (a, b); \
 	}
 
-TEST_STX (vst4q, int8x16x4_t, int8_t*, s8);
-TEST_STX (vst4q, uint8x16x4_t, uint8_t*, u8);
-TEST_STX (vst4q, poly8x16x4_t, poly8_t*, p8);
-TEST_STX (vst4q, int16x8x4_t, int16_t*, s16);
-TEST_STX (vst4q, uint16x8x4_t, uint16_t*, u16);
-TEST_STX (vst4q, poly16x8x4_t, poly16_t*, p16);
-TEST_STX (vst4q, float16x8x4_t, float16_t*, f16);
-TEST_STX (vst4q, bfloat16x8x4_t, bfloat16_t*, bf16);
-TEST_STX (vst4q, int32x4x4_t, int32_t*, s32);
-TEST_STX (vst4q, uint32x4x4_t, uint32_t*, u32);
-TEST_STX (vst4q, float32x4x4_t, float32_t*, f32);
-TEST_STX (vst4q, int64x2x4_t, int64_t*, s64);
-TEST_STX (vst4q, uint64x2x4_t, uint64_t*, u64);
-TEST_STX (vst4q, float64x2x4_t, float64_t*, f64);
-TEST_STX (vst4q, poly64x2x4_t, poly64_t*, p64);
+TEST_STX (vst2, int8x8x2_t, int8_t*, s8);
+TEST_STX (vst2, uint8x8x2_t, uint8_t*, u8);
+TEST_STX (vst2, poly8x8x2_t, poly8_t*, p8);
+TEST_STX (vst2, int16x4x2_t, int16_t*, s16);
+TEST_STX (vst2, uint16x4x2_t, uint16_t*, u16);
+TEST_STX (vst2, poly16x4x2_t, poly16_t*, p16);
+TEST_STX (vst2, float16x4x2_t, float16_t*, f16);
+TEST_STX (vst2, bfloat16x4x2_t, bfloat16_t*, bf16);
+TEST_STX (vst2, int32x2x2_t, int32_t*, s32);
+TEST_STX (vst2, uint32x2x2_t, uint32_t*, u32);
+TEST_STX (vst2, float32x2x2_t, float32_t*, f32);
+TEST_STX (vst2, int64x1x2_t, int64_t*, s64);
+TEST_STX (vst2, uint64x1x2_t, uint64_t*, u64);
+TEST_STX (vst2, float64x1x2_t, float64_t*, f64);
+TEST_STX (vst2, poly64x1x2_t, poly64_t*, p64);
 
 TEST_STX (vst2q, int8x16x2_t, int8_t*, s8);
 TEST_STX (vst2q, uint8x16x2_t, uint8_t*, u8);
@@ -120,49 +101,193 @@ TEST_STX (vst2q, uint64x2x2_t, uint64_t*, u64);
 TEST_STX (vst2q, float64x2x2_t, float64_t*, f64);
 TEST_STX (vst2q, poly64x2x2_t, poly64_t*, p64);
 
-#define TEST_ST3(name, tbltype, ptrtype, ts) \
-  void test_ ## name ## _ ## ts (ptrtype a, int8x8_t dummy, tbltype b) \
+TEST_STX (vst3, int8x8x3_t, int8_t*, s8);
+TEST_STX (vst3, uint8x8x3_t, uint8_t*, u8);
+TEST_STX (vst3, poly8x8x3_t, poly8_t*, p8);
+TEST_STX (vst3, int16x4x3_t, int16_t*, s16);
+TEST_STX (vst3, uint16x4x3_t, uint16_t*, u16);
+TEST_STX (vst3, poly16x4x3_t, poly16_t*, p16);
+TEST_STX (vst3, float16x4x3_t, float16_t*, f16);
+TEST_STX (vst3, bfloat16x4x3_t, bfloat16_t*, bf16);
+TEST_STX (vst3, int32x2x3_t, int32_t*, s32);
+TEST_STX (vst3, uint32x2x3_t, uint32_t*, u32);
+TEST_STX (vst3, float32x2x3_t, float32_t*, f32);
+TEST_STX (vst3, int64x1x3_t, int64_t*, s64);
+TEST_STX (vst3, uint64x1x3_t, uint64_t*, u64);
+TEST_STX (vst3, float64x1x3_t, float64_t*, f64);
+TEST_STX (vst3, poly64x1x3_t, poly64_t*, p64);
+
+TEST_STX (vst3q, int8x16x3_t, int8_t*, s8);
+TEST_STX (vst3q, uint8x16x3_t, uint8_t*, u8);
+TEST_STX (vst3q, poly8x16x3_t, poly8_t*, p8);
+TEST_STX (vst3q, int16x8x3_t, int16_t*, s16);
+TEST_STX (vst3q, uint16x8x3_t, uint16_t*, u16);
+TEST_STX (vst3q, poly16x8x3_t, poly16_t*, p16);
+TEST_STX (vst3q, float16x8x3_t, float16_t*, f16);
+TEST_STX (vst3q, bfloat16x8x3_t, bfloat16_t*, bf16);
+TEST_STX (vst3q, int32x4x3_t, int32_t*, s32);
+TEST_STX (vst3q, uint32x4x3_t, uint32_t*, u32);
+TEST_STX (vst3q, float32x4x3_t, float32_t*, f32);
+TEST_STX (vst3q, int64x2x3_t, int64_t*, s64);
+TEST_STX (vst3q, uint64x2x3_t, uint64_t*, u64);
+TEST_STX (vst3q, float64x2x3_t, float64_t*, f64);
+TEST_STX (vst3q, poly64x2x3_t, poly64_t*, p64);
+
+TEST_STX (vst4, int8x8x4_t, int8_t*, s8);
+TEST_STX (vst4, uint8x8x4_t, uint8_t*, u8);
+TEST_STX (vst4, poly8x8x4_t, poly8_t*, p8);
+TEST_STX (vst4, int16x4x4_t, int16_t*, s16);
+TEST_STX (vst4, uint16x4x4_t, uint16_t*, u16);
+TEST_STX (vst4, poly16x4x4_t, poly16_t*, p16);
+TEST_STX (vst4, float16x4x4_t, float16_t*, f16);
+TEST_STX (vst4, bfloat16x4x4_t, bfloat16_t*, bf16);
+TEST_STX (vst4, int32x2x4_t, int32_t*, s32);
+TEST_STX (vst4, uint32x2x4_t, uint32_t*, u32);
+TEST_STX (vst4, float32x2x4_t, float32_t*, f32);
+TEST_STX (vst4, int64x1x4_t, int64_t*, s64);
+TEST_STX (vst4, uint64x1x4_t, uint64_t*, u64);
+TEST_STX (vst4, float64x1x4_t, float64_t*, f64);
+TEST_STX (vst4, poly64x1x4_t, poly64_t*, p64);
+
+TEST_STX (vst4q, int8x16x4_t, int8_t*, s8);
+TEST_STX (vst4q, uint8x16x4_t, uint8_t*, u8);
+TEST_STX (vst4q, poly8x16x4_t, poly8_t*, p8);
+TEST_STX (vst4q, int16x8x4_t, int16_t*, s16);
+TEST_STX (vst4q, uint16x8x4_t, uint16_t*, u16);
+TEST_STX (vst4q, poly16x8x4_t, poly16_t*, p16);
+TEST_STX (vst4q, float16x8x4_t, float16_t*, f16);
+TEST_STX (vst4q, bfloat16x8x4_t, bfloat16_t*, bf16);
+TEST_STX (vst4q, int32x4x4_t, int32_t*, s32);
+TEST_STX (vst4q, uint32x4x4_t, uint32_t*, u32);
+TEST_STX (vst4q, float32x4x4_t, float32_t*, f32);
+TEST_STX (vst4q, int64x2x4_t, int64_t*, s64);
+TEST_STX (vst4q, uint64x2x4_t, uint64_t*, u64);
+TEST_STX (vst4q, float64x2x4_t, float64_t*, f64);
+TEST_STX (vst4q, poly64x2x4_t, poly64_t*, p64);
+
+#define TEST_LDX(name, rettype, ptrtype, ts) \
+  rettype test_ ## name ## _ ## ts (ptrtype a) \
 	{ \
-		name ## _ ## ts (a, b); \
+		return name ## _ ## ts (a); \
 	}
 
-TEST_ST3 (vst3q, int8x16x3_t, int8_t*, s8);
-TEST_ST3 (vst3q, uint8x16x3_t, uint8_t*, u8);
-TEST_ST3 (vst3q, poly8x16x3_t, poly8_t*, p8);
-TEST_ST3 (vst3q, int16x8x3_t, int16_t*, s16);
-TEST_ST3 (vst3q, uint16x8x3_t, uint16_t*, u16);
-TEST_ST3 (vst3q, poly16x8x3_t, poly16_t*, p16);
-TEST_ST3 (vst3q, float16x8x3_t, float16_t*, f16);
-TEST_ST3 (vst3q, bfloat16x8x3_t, bfloat16_t*, bf16);
-TEST_ST3 (vst3q, int32x4x3_t, int32_t*, s32);
-TEST_ST3 (vst3q, uint32x4x3_t, uint32_t*, u32);
-TEST_ST3 (vst3q, float32x4x3_t, float32_t*, f32);
-TEST_ST3 (vst3q, int64x2x3_t, int64_t*, s64);
-TEST_ST3 (vst3q, uint64x2x3_t, uint64_t*, u64);
-TEST_ST3 (vst3q, float64x2x3_t, float64_t*, f64);
-TEST_ST3 (vst3q, poly64x2x3_t, poly64_t*, p64);
+TEST_LDX (vld2, int8x8x2_t, int8_t*, s8);
+TEST_LDX (vld2, uint8x8x2_t, uint8_t*, u8);
+TEST_LDX (vld2, poly8x8x2_t, poly8_t*, p8);
+TEST_LDX (vld2, int16x4x2_t, int16_t*, s16);
+TEST_LDX (vld2, uint16x4x2_t, uint16_t*, u16);
+TEST_LDX (vld2, poly16x4x2_t, poly16_t*, p16);
+TEST_LDX (vld2, float16x4x2_t, float16_t*, f16);
+TEST_LDX (vld2, bfloat16x4x2_t, bfloat16_t*, bf16);
+TEST_LDX (vld2, int32x2x2_t, int32_t*, s32);
+TEST_LDX (vld2, uint32x2x2_t, uint32_t*, u32);
+TEST_LDX (vld2, float32x2x2_t, float32_t*, f32);
+TEST_LDX (vld2, int64x1x2_t, int64_t*, s64);
+TEST_LDX (vld2, uint64x1x2_t, uint64_t*, u64);
+TEST_LDX (vld2, float64x1x2_t, float64_t*, f64);
+TEST_LDX (vld2, poly64x1x2_t, poly64_t*, p64);
+
+TEST_LDX (vld2q, int8x16x2_t, int8_t*, s8);
+TEST_LDX (vld2q, uint8x16x2_t, uint8_t*, u8);
+TEST_LDX (vld2q, poly8x16x2_t, poly8_t*, p8);
+TEST_LDX (vld2q, int16x8x2_t, int16_t*, s16);
+TEST_LDX (vld2q, uint16x8x2_t, uint16_t*, u16);
+TEST_LDX (vld2q, poly16x8x2_t, poly16_t*, p16);
+TEST_LDX (vld2q, float16x8x2_t, float16_t*, f16);
+TEST_LDX (vld2q, bfloat16x8x2_t, bfloat16_t*, bf16);
+TEST_LDX (vld2q, int32x4x2_t, int32_t*, s32);
+TEST_LDX (vld2q, uint32x4x2_t, uint32_t*, u32);
+TEST_LDX (vld2q, float32x4x2_t, float32_t*, f32);
+TEST_LDX (vld2q, int64x2x2_t, int64_t*, s64);
+TEST_LDX (vld2q, uint64x2x2_t, uint64_t*, u64);
+TEST_LDX (vld2q, float64x2x2_t, float64_t*, f64);
+TEST_LDX (vld2q, poly64x2x2_t, poly64_t*, p64);
+
+TEST_LDX (vld3, int8x8x3_t, int8_t*, s8);
+TEST_LDX (vld3, uint8x8x3_t, uint8_t*, u8);
+TEST_LDX (vld3, poly8x8x3_t, poly8_t*, p8);
+TEST_LDX (vld3, int16x4x3_t, int16_t*, s16);
+TEST_LDX (vld3, uint16x4x3_t, uint16_t*, u16);
+TEST_LDX (vld3, poly16x4x3_t, poly16_t*, p16);
+TEST_LDX (vld3, float16x4x3_t, float16_t*, f16);
+TEST_LDX (vld3, bfloat16x4x3_t, bfloat16_t*, bf16);
+TEST_LDX (vld3, int32x2x3_t, int32_t*, s32);
+TEST_LDX (vld3, uint32x2x3_t, uint32_t*, u32);
+TEST_LDX (vld3, float32x2x3_t, float32_t*, f32);
+TEST_LDX (vld3, int64x1x3_t, int64_t*, s64);
+TEST_LDX (vld3, uint64x1x3_t, uint64_t*, u64);
+TEST_LDX (vld3, float64x1x3_t, float64_t*, f64);
+TEST_LDX (vld3, poly64x1x3_t, poly64_t*, p64);
+
+TEST_LDX (vld3q, int8x16x3_t, int8_t*, s8);
+TEST_LDX (vld3q, uint8x16x3_t, uint8_t*, u8);
+TEST_LDX (vld3q, poly8x16x3_t, poly8_t*, p8);
+TEST_LDX (vld3q, int16x8x3_t, int16_t*, s16);
+TEST_LDX (vld3q, uint16x8x3_t, uint16_t*, u16);
+TEST_LDX (vld3q, poly16x8x3_t, poly16_t*, p16);
+TEST_LDX (vld3q, float16x8x3_t, float16_t*, f16);
+TEST_LDX (vld3q, bfloat16x8x3_t, bfloat16_t*, bf16);
+TEST_LDX (vld3q, int32x4x3_t, int32_t*, s32);
+TEST_LDX (vld3q, uint32x4x3_t, uint32_t*, u32);
+TEST_LDX (vld3q, float32x4x3_t, float32_t*, f32);
+TEST_LDX (vld3q, int64x2x3_t, int64_t*, s64);
+TEST_LDX (vld3q, uint64x2x3_t, uint64_t*, u64);
+TEST_LDX (vld3q, float64x2x3_t, float64_t*, f64);
+TEST_LDX (vld3q, poly64x2x3_t, poly64_t*, p64);
+
+TEST_LDX (vld4, int8x8x4_t, int8_t*, s8);
+TEST_LDX (vld4, uint8x8x4_t, uint8_t*, u8);
+TEST_LDX (vld4, poly8x8x4_t, poly8_t*, p8);
+TEST_LDX (vld4, int16x4x4_t, int16_t*, s16);
+TEST_LDX (vld4, uint16x4x4_t, uint16_t*, u16);
+TEST_LDX (vld4, poly16x4x4_t, poly16_t*, p16);
+TEST_LDX (vld4, float16x4x4_t, float16_t*, f16);
+TEST_LDX (vld4, bfloat16x4x4_t, bfloat16_t*, bf16);
+TEST_LDX (vld4, int32x2x4_t, int32_t*, s32);
+TEST_LDX (vld4, uint32x2x4_t, uint32_t*, u32);
+TEST_LDX (vld4, float32x2x4_t, float32_t*, f32);
+TEST_LDX (vld4, int64x1x4_t, int64_t*, s64);
+TEST_LDX (vld4, uint64x1x4_t, uint64_t*, u64);
+TEST_LDX (vld4, float64x1x4_t, float64_t*, f64);
+TEST_LDX (vld4, poly64x1x4_t, poly64_t*, p64);
+
+TEST_LDX (vld4q, int8x16x4_t, int8_t*, s8);
+TEST_LDX (vld4q, uint8x16x4_t, uint8_t*, u8);
+TEST_LDX (vld4q, poly8x16x4_t, poly8_t*, p8);
+TEST_LDX (vld4q, int16x8x4_t, int16_t*, s16);
+TEST_LDX (vld4q, uint16x8x4_t, uint16_t*, u16);
+TEST_LDX (vld4q, poly16x8x4_t, poly16_t*, p16);
+TEST_LDX (vld4q, float16x8x4_t, float16_t*, f16);
+TEST_LDX (vld4q, bfloat16x8x4_t, bfloat16_t*, bf16);
+TEST_LDX (vld4q, int32x4x4_t, int32_t*, s32);
+TEST_LDX (vld4q, uint32x4x4_t, uint32_t*, u32);
+TEST_LDX (vld4q, float32x4x4_t, float32_t*, f32);
+TEST_LDX (vld4q, int64x2x4_t, int64_t*, s64);
+TEST_LDX (vld4q, uint64x2x4_t, uint64_t*, u64);
+TEST_LDX (vld4q, float64x2x4_t, float64_t*, f64);
+TEST_LDX (vld4q, poly64x2x4_t, poly64_t*, p64);
 
 #define TEST_STX_LANE(name, tbltype, ptrtype, ts) \
   void test_ ## name ## _ ## ts (ptrtype a, tbltype b) \
 	{ \
-		name ## _ ## ts (a, b, 1); \
+		name ## _ ## ts (a, b, 0); \
 	}
 
-TEST_STX_LANE (vst4q_lane, int8x16x4_t, int8_t*, s8);
-TEST_STX_LANE (vst4q_lane, uint8x16x4_t, uint8_t*, u8);
-TEST_STX_LANE (vst4q_lane, poly8x16x4_t, poly8_t*, p8);
-TEST_STX_LANE (vst4q_lane, int16x8x4_t, int16_t*, s16);
-TEST_STX_LANE (vst4q_lane, uint16x8x4_t, uint16_t*, u16);
-TEST_STX_LANE (vst4q_lane, poly16x8x4_t, poly16_t*, p16);
-TEST_STX_LANE (vst4q_lane, float16x8x4_t, float16_t*, f16);
-TEST_STX_LANE (vst4q_lane, bfloat16x8x4_t, bfloat16_t*, bf16);
-TEST_STX_LANE (vst4q_lane, int32x4x4_t, int32_t*, s32);
-TEST_STX_LANE (vst4q_lane, uint32x4x4_t, uint32_t*, u32);
-TEST_STX_LANE (vst4q_lane, float32x4x4_t, float32_t*, f32);
-TEST_STX_LANE (vst4q_lane, int64x2x4_t, int64_t*, s64);
-TEST_STX_LANE (vst4q_lane, uint64x2x4_t, uint64_t*, u64);
-TEST_STX_LANE (vst4q_lane, float64x2x4_t, float64_t*, f64);
-TEST_STX_LANE (vst4q_lane, poly64x2x4_t, poly64_t*, p64);
+TEST_STX_LANE (vst2_lane, int8x8x2_t, int8_t*, s8);
+TEST_STX_LANE (vst2_lane, uint8x8x2_t, uint8_t*, u8);
+TEST_STX_LANE (vst2_lane, poly8x8x2_t, poly8_t*, p8);
+TEST_STX_LANE (vst2_lane, int16x4x2_t, int16_t*, s16);
+TEST_STX_LANE (vst2_lane, uint16x4x2_t, uint16_t*, u16);
+TEST_STX_LANE (vst2_lane, poly16x4x2_t, poly16_t*, p16);
+TEST_STX_LANE (vst2_lane, float16x4x2_t, float16_t*, f16);
+TEST_STX_LANE (vst2_lane, bfloat16x4x2_t, bfloat16_t*, bf16);
+TEST_STX_LANE (vst2_lane, int32x2x2_t, int32_t*, s32);
+TEST_STX_LANE (vst2_lane, uint32x2x2_t, uint32_t*, u32);
+TEST_STX_LANE (vst2_lane, float32x2x2_t, float32_t*, f32);
+TEST_STX_LANE (vst2_lane, int64x1x2_t, int64_t*, s64);
+TEST_STX_LANE (vst2_lane, uint64x1x2_t, uint64_t*, u64);
+TEST_STX_LANE (vst2_lane, float64x1x2_t, float64_t*, f64);
+TEST_STX_LANE (vst2_lane, poly64x1x2_t, poly64_t*, p64);
 
 TEST_STX_LANE (vst2q_lane, int8x16x2_t, int8_t*, s8);
 TEST_STX_LANE (vst2q_lane, uint8x16x2_t, uint8_t*, u8);
@@ -180,27 +305,171 @@ TEST_STX_LANE (vst2q_lane, uint64x2x2_t, uint64_t*, u64);
 TEST_STX_LANE (vst2q_lane, float64x2x2_t, float64_t*, f64);
 TEST_STX_LANE (vst2q_lane, poly64x2x2_t, poly64_t*, p64);
 
-#define TEST_ST3_LANE(name, tbltype, ptrtype, ts) \
-  void test_ ## name ## _ ## ts (ptrtype a, int8x8_t dummy, tbltype b) \
+TEST_STX_LANE (vst3_lane, int8x8x3_t, int8_t*, s8);
+TEST_STX_LANE (vst3_lane, uint8x8x3_t, uint8_t*, u8);
+TEST_STX_LANE (vst3_lane, poly8x8x3_t, poly8_t*, p8);
+TEST_STX_LANE (vst3_lane, int16x4x3_t, int16_t*, s16);
+TEST_STX_LANE (vst3_lane, uint16x4x3_t, uint16_t*, u16);
+TEST_STX_LANE (vst3_lane, poly16x4x3_t, poly16_t*, p16);
+TEST_STX_LANE (vst3_lane, float16x4x3_t, float16_t*, f16);
+TEST_STX_LANE (vst3_lane, bfloat16x4x3_t, bfloat16_t*, bf16);
+TEST_STX_LANE (vst3_lane, int32x2x3_t, int32_t*, s32);
+TEST_STX_LANE (vst3_lane, uint32x2x3_t, uint32_t*, u32);
+TEST_STX_LANE (vst3_lane, float32x2x3_t, float32_t*, f32);
+TEST_STX_LANE (vst3_lane, int64x1x3_t, int64_t*, s64);
+TEST_STX_LANE (vst3_lane, uint64x1x3_t, uint64_t*, u64);
+TEST_STX_LANE (vst3_lane, float64x1x3_t, float64_t*, f64);
+TEST_STX_LANE (vst3_lane, poly64x1x3_t, poly64_t*, p64);
+
+TEST_STX_LANE (vst3q_lane, int8x16x3_t, int8_t*, s8);
+TEST_STX_LANE (vst3q_lane, uint8x16x3_t, uint8_t*, u8);
+TEST_STX_LANE (vst3q_lane, poly8x16x3_t, poly8_t*, p8);
+TEST_STX_LANE (vst3q_lane, int16x8x3_t, int16_t*, s16);
+TEST_STX_LANE (vst3q_lane, uint16x8x3_t, uint16_t*, u16);
+TEST_STX_LANE (vst3q_lane, poly16x8x3_t, poly16_t*, p16);
+TEST_STX_LANE (vst3q_lane, float16x8x3_t, float16_t*, f16);
+TEST_STX_LANE (vst3q_lane, bfloat16x8x3_t, bfloat16_t*, bf16);
+TEST_STX_LANE (vst3q_lane, int32x4x3_t, int32_t*, s32);
+TEST_STX_LANE (vst3q_lane, uint32x4x3_t, uint32_t*, u32);
+TEST_STX_LANE (vst3q_lane, float32x4x3_t, float32_t*, f32);
+TEST_STX_LANE (vst3q_lane, int64x2x3_t, int64_t*, s64);
+TEST_STX_LANE (vst3q_lane, uint64x2x3_t, uint64_t*, u64);
+TEST_STX_LANE (vst3q_lane, float64x2x3_t, float64_t*, f64);
+TEST_STX_LANE (vst3q_lane, poly64x2x3_t, poly64_t*, p64);
+
+TEST_STX_LANE (vst4_lane, int8x8x4_t, int8_t*, s8);
+TEST_STX_LANE (vst4_lane, uint8x8x4_t, uint8_t*, u8);
+TEST_STX_LANE (vst4_lane, poly8x8x4_t, poly8_t*, p8);
+TEST_STX_LANE (vst4_lane, int16x4x4_t, int16_t*, s16);
+TEST_STX_LANE (vst4_lane, uint16x4x4_t, uint16_t*, u16);
+TEST_STX_LANE (vst4_lane, poly16x4x4_t, poly16_t*, p16);
+TEST_STX_LANE (vst4_lane, float16x4x4_t, float16_t*, f16);
+TEST_STX_LANE (vst4_lane, bfloat16x4x4_t, bfloat16_t*, bf16);
+TEST_STX_LANE (vst4_lane, int32x2x4_t, int32_t*, s32);
+TEST_STX_LANE (vst4_lane, uint32x2x4_t, uint32_t*, u32);
+TEST_STX_LANE (vst4_lane, float32x2x4_t, float32_t*, f32);
+TEST_STX_LANE (vst4_lane, int64x1x4_t, int64_t*, s64);
+TEST_STX_LANE (vst4_lane, uint64x1x4_t, uint64_t*, u64);
+TEST_STX_LANE (vst4_lane, float64x1x4_t, float64_t*, f64);
+TEST_STX_LANE (vst4_lane, poly64x1x4_t, poly64_t*, p64);
+
+TEST_STX_LANE (vst4q_lane, int8x16x4_t, int8_t*, s8);
+TEST_STX_LANE (vst4q_lane, uint8x16x4_t, uint8_t*, u8);
+TEST_STX_LANE (vst4q_lane, poly8x16x4_t, poly8_t*, p8);
+TEST_STX_LANE (vst4q_lane, int16x8x4_t, int16_t*, s16);
+TEST_STX_LANE (vst4q_lane, uint16x8x4_t, uint16_t*, u16);
+TEST_STX_LANE (vst4q_lane, poly16x8x4_t, poly16_t*, p16);
+TEST_STX_LANE (vst4q_lane, float16x8x4_t, float16_t*, f16);
+TEST_STX_LANE (vst4q_lane, bfloat16x8x4_t, bfloat16_t*, bf16);
+TEST_STX_LANE (vst4q_lane, int32x4x4_t, int32_t*, s32);
+TEST_STX_LANE (vst4q_lane, uint32x4x4_t, uint32_t*, u32);
+TEST_STX_LANE (vst4q_lane, float32x4x4_t, float32_t*, f32);
+TEST_STX_LANE (vst4q_lane, int64x2x4_t, int64_t*, s64);
+TEST_STX_LANE (vst4q_lane, uint64x2x4_t, uint64_t*, u64);
+TEST_STX_LANE (vst4q_lane, float64x2x4_t, float64_t*, f64);
+TEST_STX_LANE (vst4q_lane, poly64x2x4_t, poly64_t*, p64);
+
+#define TEST_LDX_LANE(name, rettype, ptrtype, ts) \
+  rettype test_ ## name ## _ ## ts (ptrtype a, rettype b) \
 	{ \
-		name ## _ ## ts (a, b, 1); \
+		return name ## _ ## ts (a, b, 0); \
 	}
 
-TEST_ST3_LANE (vst3q_lane, int8x16x3_t, int8_t*, s8);
-TEST_ST3_LANE (vst3q_lane, uint8x16x3_t, uint8_t*, u8);
-TEST_ST3_LANE (vst3q_lane, poly8x16x3_t, poly8_t*, p8);
-TEST_ST3_LANE (vst3q_lane, int16x8x3_t, int16_t*, s16);
-TEST_ST3_LANE (vst3q_lane, uint16x8x3_t, uint16_t*, u16);
-TEST_ST3_LANE (vst3q_lane, poly16x8x3_t, poly16_t*, p16);
-TEST_ST3_LANE (vst3q_lane, float16x8x3_t, float16_t*, f16);
-TEST_ST3_LANE (vst3q_lane, bfloat16x8x3_t, bfloat16_t*, bf16);
-TEST_ST3_LANE (vst3q_lane, int32x4x3_t, int32_t*, s32);
-TEST_ST3_LANE (vst3q_lane, uint32x4x3_t, uint32_t*, u32);
-TEST_ST3_LANE (vst3q_lane, float32x4x3_t, float32_t*, f32);
-TEST_ST3_LANE (vst3q_lane, int64x2x3_t, int64_t*, s64);
-TEST_ST3_LANE (vst3q_lane, uint64x2x3_t, uint64_t*, u64);
-TEST_ST3_LANE (vst3q_lane, float64x2x3_t, float64_t*, f64);
-TEST_ST3_LANE (vst3q_lane, poly64x2x3_t, poly64_t*, p64);
+TEST_LDX_LANE (vld2_lane, int8x8x2_t, int8_t*, s8);
+TEST_LDX_LANE (vld2_lane, uint8x8x2_t, uint8_t*, u8);
+TEST_LDX_LANE (vld2_lane, poly8x8x2_t, poly8_t*, p8);
+TEST_LDX_LANE (vld2_lane, int16x4x2_t, int16_t*, s16);
+TEST_LDX_LANE (vld2_lane, uint16x4x2_t, uint16_t*, u16);
+TEST_LDX_LANE (vld2_lane, poly16x4x2_t, poly16_t*, p16);
+TEST_LDX_LANE (vld2_lane, float16x4x2_t, float16_t*, f16);
+TEST_LDX_LANE (vld2_lane, bfloat16x4x2_t, bfloat16_t*, bf16);
+TEST_LDX_LANE (vld2_lane, int32x2x2_t, int32_t*, s32);
+TEST_LDX_LANE (vld2_lane, uint32x2x2_t, uint32_t*, u32);
+TEST_LDX_LANE (vld2_lane, float32x2x2_t, float32_t*, f32);
+TEST_LDX_LANE (vld2_lane, int64x1x2_t, int64_t*, s64);
+TEST_LDX_LANE (vld2_lane, uint64x1x2_t, uint64_t*, u64);
+TEST_LDX_LANE (vld2_lane, float64x1x2_t, float64_t*, f64);
+TEST_LDX_LANE (vld2_lane, poly64x1x2_t, poly64_t*, p64);
+
+TEST_LDX_LANE (vld2q_lane, int8x16x2_t, int8_t*, s8);
+TEST_LDX_LANE (vld2q_lane, uint8x16x2_t, uint8_t*, u8);
+TEST_LDX_LANE (vld2q_lane, poly8x16x2_t, poly8_t*, p8);
+TEST_LDX_LANE (vld2q_lane, int16x8x2_t, int16_t*, s16);
+TEST_LDX_LANE (vld2q_lane, uint16x8x2_t, uint16_t*, u16);
+TEST_LDX_LANE (vld2q_lane, poly16x8x2_t, poly16_t*, p16);
+TEST_LDX_LANE (vld2q_lane, float16x8x2_t, float16_t*, f16);
+TEST_LDX_LANE (vld2q_lane, bfloat16x8x2_t, bfloat16_t*, bf16);
+TEST_LDX_LANE (vld2q_lane, int32x4x2_t, int32_t*, s32);
+TEST_LDX_LANE (vld2q_lane, uint32x4x2_t, uint32_t*, u32);
+TEST_LDX_LANE (vld2q_lane, float32x4x2_t, float32_t*, f32);
+TEST_LDX_LANE (vld2q_lane, int64x2x2_t, int64_t*, s64);
+TEST_LDX_LANE (vld2q_lane, uint64x2x2_t, uint64_t*, u64);
+TEST_LDX_LANE (vld2q_lane, float64x2x2_t, float64_t*, f64);
+TEST_LDX_LANE (vld2q_lane, poly64x2x2_t, poly64_t*, p64);
+
+TEST_LDX_LANE (vld3_lane, int8x8x3_t, int8_t*, s8);
+TEST_LDX_LANE (vld3_lane, uint8x8x3_t, uint8_t*, u8);
+TEST_LDX_LANE (vld3_lane, poly8x8x3_t, poly8_t*, p8);
+TEST_LDX_LANE (vld3_lane, int16x4x3_t, int16_t*, s16);
+TEST_LDX_LANE (vld3_lane, uint16x4x3_t, uint16_t*, u16);
+TEST_LDX_LANE (vld3_lane, poly16x4x3_t, poly16_t*, p16);
+TEST_LDX_LANE (vld3_lane, float16x4x3_t, float16_t*, f16);
+TEST_LDX_LANE (vld3_lane, bfloat16x4x3_t, bfloat16_t*, bf16);
+TEST_LDX_LANE (vld3_lane, int32x2x3_t, int32_t*, s32);
+TEST_LDX_LANE (vld3_lane, uint32x2x3_t, uint32_t*, u32);
+TEST_LDX_LANE (vld3_lane, float32x2x3_t, float32_t*, f32);
+TEST_LDX_LANE (vld3_lane, int64x1x3_t, int64_t*, s64);
+TEST_LDX_LANE (vld3_lane, uint64x1x3_t, uint64_t*, u64);
+TEST_LDX_LANE (vld3_lane, float64x1x3_t, float64_t*, f64);
+TEST_LDX_LANE (vld3_lane, poly64x1x3_t, poly64_t*, p64);
+
+TEST_LDX_LANE (vld3q_lane, int8x16x3_t, int8_t*, s8);
+TEST_LDX_LANE (vld3q_lane, uint8x16x3_t, uint8_t*, u8);
+TEST_LDX_LANE (vld3q_lane, poly8x16x3_t, poly8_t*, p8);
+TEST_LDX_LANE (vld3q_lane, int16x8x3_t, int16_t*, s16);
+TEST_LDX_LANE (vld3q_lane, uint16x8x3_t, uint16_t*, u16);
+TEST_LDX_LANE (vld3q_lane, poly16x8x3_t, poly16_t*, p16);
+TEST_LDX_LANE (vld3q_lane, float16x8x3_t, float16_t*, f16);
+TEST_LDX_LANE (vld3q_lane, bfloat16x8x3_t, bfloat16_t*, bf16);
+TEST_LDX_LANE (vld3q_lane, int32x4x3_t, int32_t*, s32);
+TEST_LDX_LANE (vld3q_lane, uint32x4x3_t, uint32_t*, u32);
+TEST_LDX_LANE (vld3q_lane, float32x4x3_t, float32_t*, f32);
+TEST_LDX_LANE (vld3q_lane, int64x2x3_t, int64_t*, s64);
+TEST_LDX_LANE (vld3q_lane, uint64x2x3_t, uint64_t*, u64);
+TEST_LDX_LANE (vld3q_lane, float64x2x3_t, float64_t*, f64);
+TEST_LDX_LANE (vld3q_lane, poly64x2x3_t, poly64_t*, p64);
+
+TEST_LDX_LANE (vld4_lane, int8x8x4_t, int8_t*, s8);
+TEST_LDX_LANE (vld4_lane, uint8x8x4_t, uint8_t*, u8);
+TEST_LDX_LANE (vld4_lane, poly8x8x4_t, poly8_t*, p8);
+TEST_LDX_LANE (vld4_lane, int16x4x4_t, int16_t*, s16);
+TEST_LDX_LANE (vld4_lane, uint16x4x4_t, uint16_t*, u16);
+TEST_LDX_LANE (vld4_lane, poly16x4x4_t, poly16_t*, p16);
+TEST_LDX_LANE (vld4_lane, float16x4x4_t, float16_t*, f16);
+TEST_LDX_LANE (vld4_lane, bfloat16x4x4_t, bfloat16_t*, bf16);
+TEST_LDX_LANE (vld4_lane, int32x2x4_t, int32_t*, s32);
+TEST_LDX_LANE (vld4_lane, uint32x2x4_t, uint32_t*, u32);
+TEST_LDX_LANE (vld4_lane, float32x2x4_t, float32_t*, f32);
+TEST_LDX_LANE (vld4_lane, int64x1x4_t, int64_t*, s64);
+TEST_LDX_LANE (vld4_lane, uint64x1x4_t, uint64_t*, u64);
+TEST_LDX_LANE (vld4_lane, float64x1x4_t, float64_t*, f64);
+TEST_LDX_LANE (vld4_lane, poly64x1x4_t, poly64_t*, p64);
+
+TEST_LDX_LANE (vld4q_lane, int8x16x4_t, int8_t*, s8);
+TEST_LDX_LANE (vld4q_lane, uint8x16x4_t, uint8_t*, u8);
+TEST_LDX_LANE (vld4q_lane, poly8x16x4_t, poly8_t*, p8);
+TEST_LDX_LANE (vld4q_lane, int16x8x4_t, int16_t*, s16);
+TEST_LDX_LANE (vld4q_lane, uint16x8x4_t, uint16_t*, u16);
+TEST_LDX_LANE (vld4q_lane, poly16x8x4_t, poly16_t*, p16);
+TEST_LDX_LANE (vld4q_lane, float16x8x4_t, float16_t*, f16);
+TEST_LDX_LANE (vld4q_lane, bfloat16x8x4_t, bfloat16_t*, bf16);
+TEST_LDX_LANE (vld4q_lane, int32x4x4_t, int32_t*, s32);
+TEST_LDX_LANE (vld4q_lane, uint32x4x4_t, uint32_t*, u32);
+TEST_LDX_LANE (vld4q_lane, float32x4x4_t, float32_t*, f32);
+TEST_LDX_LANE (vld4q_lane, int64x2x4_t, int64_t*, s64);
+TEST_LDX_LANE (vld4q_lane, uint64x2x4_t, uint64_t*, u64);
+TEST_LDX_LANE (vld4q_lane, float64x2x4_t, float64_t*, f64);
+TEST_LDX_LANE (vld4q_lane, poly64x2x4_t, poly64_t*, p64);
 
 #define TEST_ST1xN(name, tbltype, ptrtype, ts, xn) \
   void test_ ## name ## _ ## ts ## _ ## xn (ptrtype a, tbltype b) \
@@ -208,21 +477,21 @@ TEST_ST3_LANE (vst3q_lane, poly64x2x3_t, poly64_t*, p64);
 		name ## _ ## ts ## _ ## xn (a, b); \
 	}
 
-TEST_ST1xN (vst1q, int8x16x4_t, int8_t*, s8, x4);
-TEST_ST1xN (vst1q, uint8x16x4_t, uint8_t*, u8, x4);
-TEST_ST1xN (vst1q, poly8x16x4_t, poly8_t*, p8, x4);
-TEST_ST1xN (vst1q, int16x8x4_t, int16_t*, s16, x4);
-TEST_ST1xN (vst1q, uint16x8x4_t, uint16_t*, u16, x4);
-TEST_ST1xN (vst1q, poly16x8x4_t, poly16_t*, p16, x4);
-TEST_ST1xN (vst1q, float16x8x4_t, float16_t*, f16, x4);
-TEST_ST1xN (vst1q, bfloat16x8x4_t, bfloat16_t*, bf16, x4);
-TEST_ST1xN (vst1q, int32x4x4_t, int32_t*, s32, x4);
-TEST_ST1xN (vst1q, uint32x4x4_t, uint32_t*, u32, x4);
-TEST_ST1xN (vst1q, float32x4x4_t, float32_t*, f32, x4);
-TEST_ST1xN (vst1q, int64x2x4_t, int64_t*, s64, x4);
-TEST_ST1xN (vst1q, uint64x2x4_t, uint64_t*, u64, x4);
-TEST_ST1xN (vst1q, poly64x2x4_t, poly64_t*, p64, x4);
-TEST_ST1xN (vst1q, float64x2x4_t, float64_t*, f64, x4);
+TEST_ST1xN (vst1, int8x8x2_t, int8_t*, s8, x2);
+TEST_ST1xN (vst1, uint8x8x2_t, uint8_t*, u8, x2);
+TEST_ST1xN (vst1, poly8x8x2_t, poly8_t*, p8, x2);
+TEST_ST1xN (vst1, int16x4x2_t, int16_t*, s16, x2);
+TEST_ST1xN (vst1, uint16x4x2_t, uint16_t*, u16, x2);
+TEST_ST1xN (vst1, poly16x4x2_t, poly16_t*, p16, x2);
+TEST_ST1xN (vst1, float16x4x2_t, float16_t*, f16, x2);
+TEST_ST1xN (vst1, bfloat16x4x2_t, bfloat16_t*, bf16, x2);
+TEST_ST1xN (vst1, int32x2x2_t, int32_t*, s32, x2);
+TEST_ST1xN (vst1, uint32x2x2_t, uint32_t*, u32, x2);
+TEST_ST1xN (vst1, float32x2x2_t, float32_t*, f32, x2);
+TEST_ST1xN (vst1, int64x1x2_t, int64_t*, s64, x2);
+TEST_ST1xN (vst1, uint64x1x2_t, uint64_t*, u64, x2);
+TEST_ST1xN (vst1, poly64x1x2_t, poly64_t*, p64, x2);
+TEST_ST1xN (vst1, float64x1x2_t, float64_t*, f64, x2);
 
 TEST_ST1xN (vst1q, int8x16x2_t, int8_t*, s8, x2);
 TEST_ST1xN (vst1q, uint8x16x2_t, uint8_t*, u8, x2);
@@ -240,34 +509,184 @@ TEST_ST1xN (vst1q, uint64x2x2_t, uint64_t*, u64, x2);
 TEST_ST1xN (vst1q, poly64x2x2_t, poly64_t*, p64, x2);
 TEST_ST1xN (vst1q, float64x2x2_t, float64_t*, f64, x2);
 
-#define TEST_ST1x3(name, tbltype, ptrtype, ts, xn) \
-  void test_ ## name ## _ ## ts ## _ ## xn (ptrtype a, int8x8_t dummy, \
-					    tbltype b) \
+TEST_ST1xN (vst1, int8x8x3_t, int8_t*, s8, x3);
+TEST_ST1xN (vst1, uint8x8x3_t, uint8_t*, u8, x3);
+TEST_ST1xN (vst1, poly8x8x3_t, poly8_t*, p8, x3);
+TEST_ST1xN (vst1, int16x4x3_t, int16_t*, s16, x3);
+TEST_ST1xN (vst1, uint16x4x3_t, uint16_t*, u16, x3);
+TEST_ST1xN (vst1, poly16x4x3_t, poly16_t*, p16, x3);
+TEST_ST1xN (vst1, float16x4x3_t, float16_t*, f16, x3);
+TEST_ST1xN (vst1, bfloat16x4x3_t, bfloat16_t*, bf16, x3);
+TEST_ST1xN (vst1, int32x2x3_t, int32_t*, s32, x3);
+TEST_ST1xN (vst1, uint32x2x3_t, uint32_t*, u32, x3);
+TEST_ST1xN (vst1, float32x2x3_t, float32_t*, f32, x3);
+TEST_ST1xN (vst1, int64x1x3_t, int64_t*, s64, x3);
+TEST_ST1xN (vst1, uint64x1x3_t, uint64_t*, u64, x3);
+TEST_ST1xN (vst1, poly64x1x3_t, poly64_t*, p64, x3);
+TEST_ST1xN (vst1, float64x1x3_t, float64_t*, f64, x3);
+
+TEST_ST1xN (vst1q, int8x16x3_t, int8_t*, s8, x3);
+TEST_ST1xN (vst1q, uint8x16x3_t, uint8_t*, u8, x3);
+TEST_ST1xN (vst1q, poly8x16x3_t, poly8_t*, p8, x3);
+TEST_ST1xN (vst1q, int16x8x3_t, int16_t*, s16, x3);
+TEST_ST1xN (vst1q, uint16x8x3_t, uint16_t*, u16, x3);
+TEST_ST1xN (vst1q, poly16x8x3_t, poly16_t*, p16, x3);
+TEST_ST1xN (vst1q, float16x8x3_t, float16_t*, f16, x3);
+TEST_ST1xN (vst1q, bfloat16x8x3_t, bfloat16_t*, bf16, x3);
+TEST_ST1xN (vst1q, int32x4x3_t, int32_t*, s32, x3);
+TEST_ST1xN (vst1q, uint32x4x3_t, uint32_t*, u32, x3);
+TEST_ST1xN (vst1q, float32x4x3_t, float32_t*, f32, x3);
+TEST_ST1xN (vst1q, int64x2x3_t, int64_t*, s64, x3);
+TEST_ST1xN (vst1q, uint64x2x3_t, uint64_t*, u64, x3);
+TEST_ST1xN (vst1q, poly64x2x3_t, poly64_t*, p64, x3);
+TEST_ST1xN (vst1q, float64x2x3_t, float64_t*, f64, x3);
+
+TEST_ST1xN (vst1, int8x8x4_t, int8_t*, s8, x4);
+TEST_ST1xN (vst1, uint8x8x4_t, uint8_t*, u8, x4);
+TEST_ST1xN (vst1, poly8x8x4_t, poly8_t*, p8, x4);
+TEST_ST1xN (vst1, int16x4x4_t, int16_t*, s16, x4);
+TEST_ST1xN (vst1, uint16x4x4_t, uint16_t*, u16, x4);
+TEST_ST1xN (vst1, poly16x4x4_t, poly16_t*, p16, x4);
+TEST_ST1xN (vst1, float16x4x4_t, float16_t*, f16, x4);
+TEST_ST1xN (vst1, bfloat16x4x4_t, bfloat16_t*, bf16, x4);
+TEST_ST1xN (vst1, int32x2x4_t, int32_t*, s32, x4);
+TEST_ST1xN (vst1, uint32x2x4_t, uint32_t*, u32, x4);
+TEST_ST1xN (vst1, float32x2x4_t, float32_t*, f32, x4);
+TEST_ST1xN (vst1, int64x1x4_t, int64_t*, s64, x4);
+TEST_ST1xN (vst1, uint64x1x4_t, uint64_t*, u64, x4);
+TEST_ST1xN (vst1, poly64x1x4_t, poly64_t*, p64, x4);
+TEST_ST1xN (vst1, float64x1x4_t, float64_t*, f64, x4);
+
+TEST_ST1xN (vst1q, int8x16x4_t, int8_t*, s8, x4);
+TEST_ST1xN (vst1q, uint8x16x4_t, uint8_t*, u8, x4);
+TEST_ST1xN (vst1q, poly8x16x4_t, poly8_t*, p8, x4);
+TEST_ST1xN (vst1q, int16x8x4_t, int16_t*, s16, x4);
+TEST_ST1xN (vst1q, uint16x8x4_t, uint16_t*, u16, x4);
+TEST_ST1xN (vst1q, poly16x8x4_t, poly16_t*, p16, x4);
+TEST_ST1xN (vst1q, float16x8x4_t, float16_t*, f16, x4);
+TEST_ST1xN (vst1q, bfloat16x8x4_t, bfloat16_t*, bf16, x4);
+TEST_ST1xN (vst1q, int32x4x4_t, int32_t*, s32, x4);
+TEST_ST1xN (vst1q, uint32x4x4_t, uint32_t*, u32, x4);
+TEST_ST1xN (vst1q, float32x4x4_t, float32_t*, f32, x4);
+TEST_ST1xN (vst1q, int64x2x4_t, int64_t*, s64, x4);
+TEST_ST1xN (vst1q, uint64x2x4_t, uint64_t*, u64, x4);
+TEST_ST1xN (vst1q, poly64x2x4_t, poly64_t*, p64, x4);
+TEST_ST1xN (vst1q, float64x2x4_t, float64_t*, f64, x4);
+
+#define TEST_LD1xN(name, rettype, ptrtype, ts, xn) \
+  rettype test_ ## name ## _ ## ts ## _ ## xn (ptrtype a) \
 	{ \
-		name ## _ ## ts ## _ ## xn (a, b); \
+		return name ## _ ## ts ## _ ## xn (a); \
 	}
 
-TEST_ST1x3 (vst1q, int8x16x3_t, int8_t*, s8, x3);
-TEST_ST1x3 (vst1q, uint8x16x3_t, uint8_t*, u8, x3);
-TEST_ST1x3 (vst1q, poly8x16x3_t, poly8_t*, p8, x3);
-TEST_ST1x3 (vst1q, int16x8x3_t, int16_t*, s16, x3);
-TEST_ST1x3 (vst1q, uint16x8x3_t, uint16_t*, u16, x3);
-TEST_ST1x3 (vst1q, poly16x8x3_t, poly16_t*, p16, x3);
-TEST_ST1x3 (vst1q, float16x8x3_t, float16_t*, f16, x3);
-TEST_ST1x3 (vst1q, bfloat16x8x3_t, bfloat16_t*, bf16, x3);
-TEST_ST1x3 (vst1q, int32x4x3_t, int32_t*, s32, x3);
-TEST_ST1x3 (vst1q, uint32x4x3_t, uint32_t*, u32, x3);
-TEST_ST1x3 (vst1q, float32x4x3_t, float32_t*, f32, x3);
-TEST_ST1x3 (vst1q, int64x2x3_t, int64_t*, s64, x3);
-TEST_ST1x3 (vst1q, uint64x2x3_t, uint64_t*, u64, x3);
-TEST_ST1x3 (vst1q, poly64x2x3_t, poly64_t*, p64, x3);
-TEST_ST1x3 (vst1q, float64x2x3_t, float64_t*, f64, x3);
+TEST_LD1xN (vld1, int8x8x2_t, int8_t*, s8, x2);
+TEST_LD1xN (vld1, uint8x8x2_t, uint8_t*, u8, x2);
+TEST_LD1xN (vld1, poly8x8x2_t, poly8_t*, p8, x2);
+TEST_LD1xN (vld1, int16x4x2_t, int16_t*, s16, x2);
+TEST_LD1xN (vld1, uint16x4x2_t, uint16_t*, u16, x2);
+TEST_LD1xN (vld1, poly16x4x2_t, poly16_t*, p16, x2);
+TEST_LD1xN (vld1, float16x4x2_t, float16_t*, f16, x2);
+TEST_LD1xN (vld1, bfloat16x4x2_t, bfloat16_t*, bf16, x2);
+TEST_LD1xN (vld1, int32x2x2_t, int32_t*, s32, x2);
+TEST_LD1xN (vld1, uint32x2x2_t, uint32_t*, u32, x2);
+TEST_LD1xN (vld1, float32x2x2_t, float32_t*, f32, x2);
+TEST_LD1xN (vld1, int64x1x2_t, int64_t*, s64, x2);
+TEST_LD1xN (vld1, uint64x1x2_t, uint64_t*, u64, x2);
+TEST_LD1xN (vld1, poly64x1x2_t, poly64_t*, p64, x2);
+TEST_LD1xN (vld1, float64x1x2_t, float64_t*, f64, x2);
+
+TEST_LD1xN (vld1q, int8x16x2_t, int8_t*, s8, x2);
+TEST_LD1xN (vld1q, uint8x16x2_t, uint8_t*, u8, x2);
+TEST_LD1xN (vld1q, poly8x16x2_t, poly8_t*, p8, x2);
+TEST_LD1xN (vld1q, int16x8x2_t, int16_t*, s16, x2);
+TEST_LD1xN (vld1q, uint16x8x2_t, uint16_t*, u16, x2);
+TEST_LD1xN (vld1q, poly16x8x2_t, poly16_t*, p16, x2);
+TEST_LD1xN (vld1q, float16x8x2_t, float16_t*, f16, x2);
+TEST_LD1xN (vld1q, bfloat16x8x2_t, bfloat16_t*, bf16, x2);
+TEST_LD1xN (vld1q, int32x4x2_t, int32_t*, s32, x2);
+TEST_LD1xN (vld1q, uint32x4x2_t, uint32_t*, u32, x2);
+TEST_LD1xN (vld1q, float32x4x2_t, float32_t*, f32, x2);
+TEST_LD1xN (vld1q, int64x2x2_t, int64_t*, s64, x2);
+TEST_LD1xN (vld1q, uint64x2x2_t, uint64_t*, u64, x2);
+TEST_LD1xN (vld1q, poly64x2x2_t, poly64_t*, p64, x2);
+TEST_LD1xN (vld1q, float64x2x2_t, float64_t*, f64, x2);
+
+TEST_LD1xN (vld1, int8x8x3_t, int8_t*, s8, x3);
+TEST_LD1xN (vld1, uint8x8x3_t, uint8_t*, u8, x3);
+TEST_LD1xN (vld1, poly8x8x3_t, poly8_t*, p8, x3);
+TEST_LD1xN (vld1, int16x4x3_t, int16_t*, s16, x3);
+TEST_LD1xN (vld1, uint16x4x3_t, uint16_t*, u16, x3);
+TEST_LD1xN (vld1, poly16x4x3_t, poly16_t*, p16, x3);
+TEST_LD1xN (vld1, float16x4x3_t, float16_t*, f16, x3);
+TEST_LD1xN (vld1, bfloat16x4x3_t, bfloat16_t*, bf16, x3);
+TEST_LD1xN (vld1, int32x2x3_t, int32_t*, s32, x3);
+TEST_LD1xN (vld1, uint32x2x3_t, uint32_t*, u32, x3);
+TEST_LD1xN (vld1, float32x2x3_t, float32_t*, f32, x3);
+TEST_LD1xN (vld1, int64x1x3_t, int64_t*, s64, x3);
+TEST_LD1xN (vld1, uint64x1x3_t, uint64_t*, u64, x3);
+TEST_LD1xN (vld1, poly64x1x3_t, poly64_t*, p64, x3);
+TEST_LD1xN (vld1, float64x1x3_t, float64_t*, f64, x3);
+
+TEST_LD1xN (vld1q, int8x16x3_t, int8_t*, s8, x3);
+TEST_LD1xN (vld1q, uint8x16x3_t, uint8_t*, u8, x3);
+TEST_LD1xN (vld1q, poly8x16x3_t, poly8_t*, p8, x3);
+TEST_LD1xN (vld1q, int16x8x3_t, int16_t*, s16, x3);
+TEST_LD1xN (vld1q, uint16x8x3_t, uint16_t*, u16, x3);
+TEST_LD1xN (vld1q, poly16x8x3_t, poly16_t*, p16, x3);
+TEST_LD1xN (vld1q, float16x8x3_t, float16_t*, f16, x3);
+TEST_LD1xN (vld1q, bfloat16x8x3_t, bfloat16_t*, bf16, x3);
+TEST_LD1xN (vld1q, int32x4x3_t, int32_t*, s32, x3);
+TEST_LD1xN (vld1q, uint32x4x3_t, uint32_t*, u32, x3);
+TEST_LD1xN (vld1q, float32x4x3_t, float32_t*, f32, x3);
+TEST_LD1xN (vld1q, int64x2x3_t, int64_t*, s64, x3);
+TEST_LD1xN (vld1q, uint64x2x3_t, uint64_t*, u64, x3);
+TEST_LD1xN (vld1q, poly64x2x3_t, poly64_t*, p64, x3);
+TEST_LD1xN (vld1q, float64x2x3_t, float64_t*, f64, x3);
+
+TEST_LD1xN (vld1, int8x8x4_t, int8_t*, s8, x4);
+TEST_LD1xN (vld1, uint8x8x4_t, uint8_t*, u8, x4);
+TEST_LD1xN (vld1, poly8x8x4_t, poly8_t*, p8, x4);
+TEST_LD1xN (vld1, int16x4x4_t, int16_t*, s16, x4);
+TEST_LD1xN (vld1, uint16x4x4_t, uint16_t*, u16, x4);
+TEST_LD1xN (vld1, poly16x4x4_t, poly16_t*, p16, x4);
+TEST_LD1xN (vld1, float16x4x4_t, float16_t*, f16, x4);
+TEST_LD1xN (vld1, bfloat16x4x4_t, bfloat16_t*, bf16, x4);
+TEST_LD1xN (vld1, int32x2x4_t, int32_t*, s32, x4);
+TEST_LD1xN (vld1, uint32x2x4_t, uint32_t*, u32, x4);
+TEST_LD1xN (vld1, float32x2x4_t, float32_t*, f32, x4);
+TEST_LD1xN (vld1, int64x1x4_t, int64_t*, s64, x4);
+TEST_LD1xN (vld1, uint64x1x4_t, uint64_t*, u64, x4);
+TEST_LD1xN (vld1, poly64x1x4_t, poly64_t*, p64, x4);
+TEST_LD1xN (vld1, float64x1x4_t, float64_t*, f64, x4);
+
+TEST_LD1xN (vld1q, int8x16x4_t, int8_t*, s8, x4);
+TEST_LD1xN (vld1q, uint8x16x4_t, uint8_t*, u8, x4);
+TEST_LD1xN (vld1q, poly8x16x4_t, poly8_t*, p8, x4);
+TEST_LD1xN (vld1q, int16x8x4_t, int16_t*, s16, x4);
+TEST_LD1xN (vld1q, uint16x8x4_t, uint16_t*, u16, x4);
+TEST_LD1xN (vld1q, poly16x8x4_t, poly16_t*, p16, x4);
+TEST_LD1xN (vld1q, float16x8x4_t, float16_t*, f16, x4);
+TEST_LD1xN (vld1q, bfloat16x8x4_t, bfloat16_t*, bf16, x4);
+TEST_LD1xN (vld1q, int32x4x4_t, int32_t*, s32, x4);
+TEST_LD1xN (vld1q, uint32x4x4_t, uint32_t*, u32, x4);
+TEST_LD1xN (vld1q, float32x4x4_t, float32_t*, f32, x4);
+TEST_LD1xN (vld1q, int64x2x4_t, int64_t*, s64, x4);
+TEST_LD1xN (vld1q, uint64x2x4_t, uint64_t*, u64, x4);
+TEST_LD1xN (vld1q, poly64x2x4_t, poly64_t*, p64, x4);
+TEST_LD1xN (vld1q, float64x2x4_t, float64_t*, f64, x4);
 
 /* { dg-final { scan-assembler-not "mov\\t" { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-not "ldr\\t" { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-not "str\\t" { target aarch64_little_endian } } } */
+/* { dg-final { scan-assembler-not "sp" { target aarch64_little_endian } } } */
 
 /* { dg-final { scan-assembler-times "tbl\\t" 18} }  */
 /* { dg-final { scan-assembler-times "tbx\\t" 18} }  */
-/* { dg-final { scan-assembler-times "st4\\t" 30} }  */
-/* { dg-final { scan-assembler-times "st3\\t" 30} }  */
-/* { dg-final { scan-assembler-times "st2\\t" 30} }  */
-/* { dg-final { scan-assembler-times "st1\\t" 45} }  */
+/* { dg-final { scan-assembler-times "st4\\t" 56} }  */
+/* { dg-final { scan-assembler-times "st3\\t" 56} }  */
+/* { dg-final { scan-assembler-times "st2\\t" 56} }  */
+/* { dg-final { scan-assembler-times "st1\\t" 102} }  */
+/* { dg-final { scan-assembler-times "ld4\\t" 56} }  */
+/* { dg-final { scan-assembler-times "ld3\\t" 56} }  */
+/* { dg-final { scan-assembler-times "ld2\\t" 56} }  */
+/* { dg-final { scan-assembler-times "ld1\\t" 102} }  */
diff --git a/gcc/testsuite/gcc.target/arc/add_n-combine.c b/gcc/testsuite/gcc.target/arc/add_n-combine.c
index bc400df..84e261e 100644
--- a/gcc/testsuite/gcc.target/arc/add_n-combine.c
+++ b/gcc/testsuite/gcc.target/arc/add_n-combine.c
@@ -45,6 +45,6 @@ void f() {
   a(at3.bn[bu]);
 }
 
-/* { dg-final { scan-assembler "add1" } } */
-/* { dg-final { scan-assembler "add2" } } */
+/* { dg-final { scan-assembler "@at1\\+1" } } */
+/* { dg-final { scan-assembler "@at2\\+2" } } */
 /* { dg-final { scan-assembler "add3" } } */
diff --git a/gcc/testsuite/gcc.target/arc/builtin_eh.c b/gcc/testsuite/gcc.target/arc/builtin_eh.c
index 717a54b..83f4f1d 100644
--- a/gcc/testsuite/gcc.target/arc/builtin_eh.c
+++ b/gcc/testsuite/gcc.target/arc/builtin_eh.c
@@ -19,4 +19,5 @@ foo (int x)
 /* { dg-final { scan-assembler "r13" } } */
 /* { dg-final { scan-assembler "r0" } } */
 /* { dg-final { scan-assembler "fp" } } */
-/* { dg-final { scan-assembler "fp,64" } } */
+/* { dg-final { scan-assembler "fp,64" { target { *-elf32-* } } } } */
+/* { dg-final { scan-assembler "fp,60" { target { *-linux-* } } } } */
diff --git a/gcc/testsuite/gcc.target/arc/mul64-1.c b/gcc/testsuite/gcc.target/arc/mul64-1.c
index 2543fc3..1a351fe 100644
--- a/gcc/testsuite/gcc.target/arc/mul64-1.c
+++ b/gcc/testsuite/gcc.target/arc/mul64-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-skip-if "MUL64 is ARC600 extension." { ! { clmcpu } } } */
+/* { dg-skip-if "MUL64 is ARC600 extension." { { ! { clmcpu } } || *-linux-* } } */
 /* { dg-options "-O2 -mmul64 -mbig-endian -mcpu=arc600" } */
 
 /* Check if mlo/mhi registers are correctly layout when we compile for
diff --git a/gcc/testsuite/gcc.target/arc/tls-gd.c b/gcc/testsuite/gcc.target/arc/tls-gd.c
index aa1b542..d02af95 100644
--- a/gcc/testsuite/gcc.target/arc/tls-gd.c
+++ b/gcc/testsuite/gcc.target/arc/tls-gd.c
@@ -13,5 +13,5 @@ int *ae2 (void)
   return &e2;
 }
 
-/* { dg-final { scan-assembler "add r0,pcl,@e2@tlsgd" } } */
-/* { dg-final { scan-assembler "bl @__tls_get_addr@plt" } } */
+/* { dg-final { scan-assembler "add\\s+r0,pcl,@e2@tlsgd" } } */
+/* { dg-final { scan-assembler "bl\\s+@__tls_get_addr@plt" } } */
diff --git a/gcc/testsuite/gcc.target/arc/tls-ie.c b/gcc/testsuite/gcc.target/arc/tls-ie.c
index 0c981cf..f4ad635 100644
--- a/gcc/testsuite/gcc.target/arc/tls-ie.c
+++ b/gcc/testsuite/gcc.target/arc/tls-ie.c
@@ -13,5 +13,5 @@ int *ae2 (void)
   return &e2;
 }
 
-/* { dg-final { scan-assembler "ld r0,\\\[pcl,@e2@tlsie\\\]" } } */
-/* { dg-final { scan-assembler "add_s r0,r0,r25" } } */
+/* { dg-final { scan-assembler "ld\\s+r0,\\\[pcl,@e2@tlsie\\\]" } } */
+/* { dg-final { scan-assembler "add_s\\s+r0,r0,r25" } } */
diff --git a/gcc/testsuite/gcc.target/arc/tls-ld.c b/gcc/testsuite/gcc.target/arc/tls-ld.c
index 351c3f0..68ab9bf 100644
--- a/gcc/testsuite/gcc.target/arc/tls-ld.c
+++ b/gcc/testsuite/gcc.target/arc/tls-ld.c
@@ -13,6 +13,6 @@ int *ae2 (void)
   return &e2;
 }
 
-/* { dg-final { scan-assembler "add r0,pcl,@.tbss@tlsgd" } } */
-/* { dg-final { scan-assembler "bl @__tls_get_addr@plt" } } */
-/* { dg-final { scan-assembler "add_s r0,r0,@e2@dtpoff" } } */
+/* { dg-final { scan-assembler "add\\s+r0,pcl,@.tbss@tlsgd" } } */
+/* { dg-final { scan-assembler "bl\\s+@__tls_get_addr@plt" } } */
+/* { dg-final { scan-assembler "add_s\\s+r0,r0,@e2@dtpoff" } } */
diff --git a/gcc/testsuite/gcc.target/arc/tmac-4.c b/gcc/testsuite/gcc.target/arc/tmac-4.c
new file mode 100644
index 0000000..3c6b993
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arc/tmac-4.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+/* { dg-skip-if "" { ! { clmcpu } } } */
+/* { dg-options "-O3 -mbig-endian -mcpu=hs38" } */
+
+struct a {};
+struct b {
+  int c;
+  int d;
+};
+
+struct {
+  struct a e;
+  struct b f[];
+} g;
+short h;
+
+extern void bar (int *);
+
+int foo(void)
+{
+  struct b *a;
+  for (;;)
+    {
+      a = &g.f[h];
+      bar(&a->d);
+    }
+}
+
+/* { dg-final { scan-assembler "dmach" } } */
diff --git a/gcc/testsuite/gcc.target/arc/uncached-8.c b/gcc/testsuite/gcc.target/arc/uncached-8.c
index 060229b..b5ea235 100644
--- a/gcc/testsuite/gcc.target/arc/uncached-8.c
+++ b/gcc/testsuite/gcc.target/arc/uncached-8.c
@@ -29,5 +29,6 @@ void bar (void)
   x.c.b.a = 10;
 }
 
-/* { dg-final { scan-assembler-times "st\.di" 1 } } */
-/* { dg-final { scan-assembler-times "st\.as\.di" 1 } } */
+/* { dg-final { scan-assembler-times "st\.di" 2 { target { *-linux-* } } } } */
+/* { dg-final { scan-assembler-times "st\.di" 1 { target { *-elf32-* } } } } */
+/* { dg-final { scan-assembler-times "st\.as\.di" 1 { target { *-elf32-* } } } } */
diff --git a/gcc/testsuite/gcc.target/arm/multilib.exp b/gcc/testsuite/gcc.target/arm/multilib.exp
index 4b30025..e3f06c3 100644
--- a/gcc/testsuite/gcc.target/arm/multilib.exp
+++ b/gcc/testsuite/gcc.target/arm/multilib.exp
@@ -135,6 +135,14 @@ if {[multilib_config "aprofile"] } {
 	{-march=armv8.6-a+simd+fp16 -mfloat-abi=softfp} "thumb/v8-a+simd/softfp"
 	{-march=armv8.6-a+simd+fp16+nofp -mfloat-abi=softfp} "thumb/v8-a/nofp"
 	{-march=armv8.6-a+simd+nofp+fp16 -mfloat-abi=softfp} "thumb/v8-a+simd/softfp"
+	{-march=armv9-a+crypto -mfloat-abi=soft} "thumb/v9-a/nofp"
+	{-march=armv9-a+simd+crypto -mfloat-abi=softfp} "thumb/v9-a+simd/softfp"
+	{-march=armv9-a+simd+crypto+nofp -mfloat-abi=softfp} "thumb/v9-a/nofp"
+	{-march=armv9-a+simd+nofp+crypto -mfloat-abi=softfp} "thumb/v9-a+simd/softfp"
+	{-march=armv9-a+fp16 -mfloat-abi=soft} "thumb/v9-a/nofp"
+	{-march=armv9-a+simd+fp16 -mfloat-abi=softfp} "thumb/v9-a+simd/softfp"
+	{-march=armv9-a+simd+fp16+nofp -mfloat-abi=softfp} "thumb/v9-a/nofp"
+	{-march=armv9-a+simd+nofp+fp16 -mfloat-abi=softfp} "thumb/v9-a+simd/softfp"
 	{-mcpu=cortex-a53+crypto -mfloat-abi=hard} "thumb/v8-a+simd/hard"
 	{-mcpu=cortex-a53+nofp -mfloat-abi=softfp} "thumb/v8-a/nofp"
 	{-march=armv8-a+crc -mfloat-abi=hard -mfpu=vfp} "thumb/v8-a+simd/hard"
diff --git a/gcc/testsuite/gcc.target/bfin/20090914-3.c b/gcc/testsuite/gcc.target/bfin/20090914-3.c
index fb0a9e1..6be5528 100644
--- a/gcc/testsuite/gcc.target/bfin/20090914-3.c
+++ b/gcc/testsuite/gcc.target/bfin/20090914-3.c
@@ -1,10 +1,11 @@
 /* { dg-do compile { target bfin-*-* } } */
 typedef long fract32;
-main() {
+fract32 foo() {
   fract32 val_tmp;
   fract32 val1 = 0x7FFFFFFF;
   fract32 val2 = 0x40000000;
   val_tmp = __builtin_bfin_mult_fr1x32x32 (0x06666667, val1);
   val2 = __builtin_bfin_mult_fr1x32x32 (0x79999999, val2);
   val2 = __builtin_bfin_add_fr1x32 (val_tmp, val2);
+  return val2;
 }
diff --git a/gcc/testsuite/gcc.target/bfin/ssashift-1.c b/gcc/testsuite/gcc.target/bfin/ssashift-1.c
new file mode 100644
index 0000000..aba90a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/bfin/ssashift-1.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+int test_ok_pos()
+{
+  int x = 100;
+  return __builtin_bfin_shl_fr1x32(x,24);
+}
+
+int test_ok_neg()
+{
+  int x = -100;
+  return __builtin_bfin_shl_fr1x32(x,24);
+}
+
+int test_sat_max()
+{
+  int x = 10000;
+  return __builtin_bfin_shl_fr1x32(x,24);
+}
+
+int test_sat_min()
+{
+  int x = -10000;
+  return __builtin_bfin_shl_fr1x32(x,24);
+}
+
+short stest_ok_pos()
+{
+  short x = 100;
+  return __builtin_bfin_shl_fr1x16(x,8);
+}
+
+short stest_ok_neg()
+{
+  short x = -100;
+  return __builtin_bfin_shl_fr1x16(x,8);
+}
+
+short stest_sat_max()
+{
+  short x = 10000;
+  return __builtin_bfin_shl_fr1x16(x,8);
+}
+
+short stest_sat_min()
+{
+  short x = -10000;
+  return __builtin_bfin_shl_fr1x16(x,8);
+}
+/* { dg-final { scan-assembler-not "\\(S\\)" } } */
+/* { dg-final { scan-assembler-not "\\(V,S\\)" } } */
diff --git a/gcc/testsuite/gcc.target/i386/amxtile-3.c b/gcc/testsuite/gcc.target/i386/amxtile-3.c
new file mode 100644
index 0000000..26204e3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/amxtile-3.c
@@ -0,0 +1,34 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O2 -mamx-tile " } */
+/* { dg-final { scan-assembler "tileloadd\[ \\t]+\[^\n\]*\\(%\[a-z0-9]*\,%\[a-z0-9\]*\,\[124\]\\)+\[^\n\]*%tmm\[0-9\]"  } } */
+/* { dg-final { scan-assembler "tileloaddt1\[ \\t]+\[^\n\]*\\(%\[a-z0-9]*\,%\[a-z0-9\]*\,\[124\]\\)+\[^\n\]*%tmm\[0-9\]"  } } */
+/* { dg-final { scan-assembler "tilestored\[ \\t]+\[^\n\]*%tmm\[0-9\]+\[^\n\]*\\(%\[a-z0-9]*\,%\[a-z0-9\]*\,\[124\]\\)"  } } */
+/* { dg-final { scan-assembler "leaq\[ \\t]+4" { target lp64 } } } */
+/* { dg-final { scan-assembler "leaq\[ \\t]+8" { target lp64 } } } */
+/* { dg-final { scan-assembler "addq\[ \\t]+\\\$12" { target lp64 } } } */
+/* { dg-final { scan-assembler "leal\[ \\t]+4" { target x32 } } } */
+/* { dg-final { scan-assembler "leal\[ \\t]+8" { target x32 } } } */
+/* { dg-final { scan-assembler "addl\[ \\t]+\\\$12" { target x32 } } } */
+/* { dg-final { scan-assembler-not "leaq\[ \\t]+1" { target lp64 } } } */
+/* { dg-final { scan-assembler-not "leaq\[ \\t]+2" { target lp64 } } } */
+/* { dg-final { scan-assembler-not "addq\[ \\t]+\\\$3" { target lp64 } } } */
+/* { dg-final { scan-assembler-not "leal\[ \\t]+1" { target x32 } } } */
+/* { dg-final { scan-assembler-not "leal\[ \\t]+2" { target x32 } } } */
+/* { dg-final { scan-assembler-not "addl\[ \\t]+\\\$3" { target x32 } } } */
+#include <immintrin.h>
+
+extern int a[];
+extern const float* base;
+extern const int stride;
+
+#define TMM0 0
+#define TMM1 1
+#define TMM2 2
+#define TMM3 3
+
+void TEST ()
+{
+  _tile_loadd (TMM3, base + 1, stride);
+  _tile_stream_loadd (TMM2, base + 2, stride);
+  _tile_stored (TMM2, base + 3, stride);
+}
diff --git a/gcc/testsuite/gcc.target/i386/auto-init-2.c b/gcc/testsuite/gcc.target/i386/auto-init-2.c
index e22930a..0c59c62 100644
--- a/gcc/testsuite/gcc.target/i386/auto-init-2.c
+++ b/gcc/testsuite/gcc.target/i386/auto-init-2.c
@@ -29,7 +29,7 @@ void foo()
   return;
 }
 
-/* { dg-final { scan-rtl-dump-times "0xfffffffffffffffe" 2 "expand" } } */
+/* { dg-final { scan-rtl-dump-times "0xfffffffffffffffe" 1 "expand" } } */
 /* { dg-final { scan-rtl-dump-times "0xfffffffffffffefe" 1 "expand" } } */
 /* { dg-final { scan-rtl-dump-times "0xfffffffffefefefe" 2 "expand" { target lp64 } } } */
 /* { dg-final { scan-rtl-dump-times "0xfefefefefefefefe" 3 "expand" { target lp64 } } } */
diff --git a/gcc/testsuite/gcc.target/i386/auto-init-4.c b/gcc/testsuite/gcc.target/i386/auto-init-4.c
index 7b46c74..1803dd4 100644
--- a/gcc/testsuite/gcc.target/i386/auto-init-4.c
+++ b/gcc/testsuite/gcc.target/i386/auto-init-4.c
@@ -1,6 +1,6 @@
 /* Verify pattern initialization for floating point type automatic variables.  */
 /* { dg-do compile } */
-/* { dg-options "-ftrivial-auto-var-init=pattern -fdump-rtl-expand -march=x86-64 -mtune=generic -msse" } */
+/* { dg-options "-ftrivial-auto-var-init=pattern -march=x86-64 -mtune=generic -msse" } */
 
 long double result;
 
@@ -14,8 +14,6 @@ long double foo()
   return result;
 }
 
-/* { dg-final { scan-rtl-dump-times "0xfffffffffefefefe" 1 "expand" { target { ! ia32 } } } } */
-/* { dg-final { scan-rtl-dump-times "\\\[0xfefefefefefefefe\\\]" 1 "expand" { target { ! ia32 } } } } */
-/* { dg-final { scan-rtl-dump-times "0xfffffffffffffffe\\\]\\\) repeated x16" 1 "expand" { target { ! ia32 } } } } */
-/* { dg-final { scan-rtl-dump-times "0xfffffffffefefefe" 2 "expand" { target ia32 } } } */
-/* { dg-final { scan-rtl-dump-times "\\\[0xfefefefefefefefe\\\]" 2 "expand" { target ia32 } } } */
+
+/* { dg-final { scan-assembler-times "long\t-16843010" 5  { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "long\t-16843010" 3  { target { ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/auto-init-6.c b/gcc/testsuite/gcc.target/i386/auto-init-6.c
index f75081e..e53385f 100644
--- a/gcc/testsuite/gcc.target/i386/auto-init-6.c
+++ b/gcc/testsuite/gcc.target/i386/auto-init-6.c
@@ -1,6 +1,8 @@
 /* Verify pattern initialization for complex type automatic variables.  */
+/* Note, _Complex long double is initialized to zeroes due to the current
+   implemenation limitation.  */
 /* { dg-do compile } */
-/* { dg-options "-ftrivial-auto-var-init=pattern -fdump-rtl-expand -march=x86-64 -mtune=generic -msse" } */
+/* { dg-options "-ftrivial-auto-var-init=pattern -march=x86-64 -mtune=generic -msse" } */
 
 
 _Complex long double result;
@@ -15,5 +17,6 @@ _Complex long double foo()
   return result;
 }
 
-/* { dg-final { scan-rtl-dump-times "\\\[0xfefefefefefefefe\\\]" 1 "expand" } } */
-/* { dg-final { scan-rtl-dump-times "0xfffffffffffffffe\\\]\\\) repeated x16" 2 "expand" } } */
+/* { dg-final { scan-assembler-times "long\t0" 8  { target { ! ia32 } } } } */
+/* { dg-final { scan-assembler-times "long\t-16843010" 6  } } */
+
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-pr96891-3.c b/gcc/testsuite/gcc.target/i386/avx512f-pr96891-3.c
index 1cf18f2..06db752 100644
--- a/gcc/testsuite/gcc.target/i386/avx512f-pr96891-3.c
+++ b/gcc/testsuite/gcc.target/i386/avx512f-pr96891-3.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-mavx512vl -mavx512bw -mavx512dq -O2 -masm=att" } */
+/* { dg-options "-mavx512vl -mavx512bw -mavx512dq -O2 -masm=att -mstv -mno-stackrealign" } */
 /* { dg-final { scan-assembler-not {not[bwlqd]\]} } } */
 /* { dg-final { scan-assembler-times {(?n)vpcmp[bwdq][ \t]*\$5} 4} } */
 /* { dg-final { scan-assembler-times {(?n)vpcmp[bwdq][ \t]*\$6} 4} } */
diff --git a/gcc/testsuite/gcc.target/i386/avx512fp16-13.c b/gcc/testsuite/gcc.target/i386/avx512fp16-13.c
index c3bae65..b73a8f4 100644
--- a/gcc/testsuite/gcc.target/i386/avx512fp16-13.c
+++ b/gcc/testsuite/gcc.target/i386/avx512fp16-13.c
@@ -18,7 +18,7 @@ store256_ph (void *p, __m256h a)
   _mm256_store_ph (p, a);
 }
 
-/* { dg-final { scan-assembler-times "vmovdqa64\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\[^\n\]*\\)" 1 } } */
+/* { dg-final { scan-assembler-times "vmovdqa\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\[^\n\]*\\)" 1 } } */
 
 void
 __attribute__ ((noinline, noclone))
@@ -27,7 +27,7 @@ store_ph (void *p, __m128h a)
   _mm_store_ph (p, a);
 }
 
-/* { dg-final { scan-assembler-times "vmovdqa64\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*\\)" 1 } } */
+/* { dg-final { scan-assembler-times "vmovdqa\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*\\)" 1 } } */
 
 __m512h
 __attribute__ ((noinline, noclone))
@@ -45,7 +45,7 @@ load256_ph (void const *p)
   return _mm256_load_ph (p);
 }
 
-/* { dg-final { scan-assembler-times "vmovdqa64\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\[^\n\]*\\)" 1 } } */
+/* { dg-final { scan-assembler-times "vmovdqa\[ \\t\]+\[^\{\n\]*%ymm\[0-9\]+\[^\n\]*\\)" 1 } } */
 
 __m128h
 __attribute__ ((noinline, noclone))
@@ -53,7 +53,7 @@ load_ph (void const *p)
 {
   return _mm_load_ph (p);
 }
-/* { dg-final { scan-assembler-times "vmovdqa64\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*\\)" 1 } } */
+/* { dg-final { scan-assembler-times "vmovdqa\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+\[^\n\]*\\)" 1 } } */
 
 __m512h
 __attribute__ ((noinline, noclone))
diff --git a/gcc/testsuite/gcc.target/i386/avx512fp16-complex-fma.c b/gcc/testsuite/gcc.target/i386/avx512fp16-complex-fma.c
new file mode 100644
index 0000000..2dfd369
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512fp16-complex-fma.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-mavx512fp16 -O2 -Ofast" } */
+/* { dg-final { scan-assembler-times "vfmaddcph\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\[^\n\r]*%zmm\[0-9\]+\[^\n\r]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)" 2 } } */
+/* { dg-final { scan-assembler-not "vaddph\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\[^\n\r]*%zmm\[0-9\]+\[^\n\r]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"} } */
+/* { dg-final { scan-assembler-not "vfmulcph\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\[^\n\r]*%zmm\[0-9\]+\[^\n\r]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)"} } */
+/* { dg-final { scan-assembler-times "vfcmaddcph\[ \\t\]+\[^\{\n\]*%zmm\[0-9\]+\[^\n\r]*%zmm\[0-9\]+\[^\n\r]*%zmm\[0-9\]+(?:\n|\[ \\t\]+#)" 2 } } */
+
+#include <immintrin.h>
+volatile __m512h x1, x2, res, a, b;
+void extern
+avx512f_test (void)
+{
+  res = _mm512_add_ph (x1, _mm512_fmadd_pch (a, b, _mm512_setzero_ph()));
+  res = _mm512_add_ph (x1, _mm512_fcmadd_pch (a, b, _mm512_setzero_ph()));
+
+  res = _mm512_add_ph (x1, _mm512_fmul_pch (a, b));
+  res = _mm512_add_ph (x1, _mm512_fcmul_pch (a, b));
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512fp16-recip-1.c b/gcc/testsuite/gcc.target/i386/avx512fp16-recip-1.c
new file mode 100644
index 0000000..bc7cbbc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512fp16-recip-1.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx512fp16 -mavx512vl -ffast-math" } */
+/* { dg-final { scan-assembler "vrcpsh.*\n.*vmulsh" } } */
+/* { dg-final { scan-assembler "vrcpph.*\n.*vmulph" } } */
+/* { dg-final { scan-assembler "vrsqrtsh.*\n.*vmulsh" } } */
+/* { dg-final { scan-assembler "vrsqrtph.*\n.*vmulph" } } */
+/* { dg-final { scan-assembler-not "vsqrtsh" } } */
+/* { dg-final { scan-assembler-not "vsqrtph" } } */
+/* { dg-final { scan-assembler-not "vdivsh" } } */
+/* { dg-final { scan-assembler-not "vdivph" } } */
+
+#define FAST_ATTR \
+  __attribute__((noinline, noclone, optimize("fast-math"), target("recip")))
+
+_Float16 FAST_ATTR
+scalar_hf_rcp_fast (_Float16 a, _Float16 b)
+{
+  return a / b;
+}
+
+_Float16 FAST_ATTR
+scalar_hf_rsqrt_fast (_Float16 a, _Float16 b)
+{
+  return a / __builtin_sqrtf16 (b);
+}
+
+void FAST_ATTR
+vector_hf_rcp_fast (_Float16 * restrict a, _Float16 * restrict b,
+		    _Float16 * restrict c, int n)
+{
+  int i;
+  for (i = 0; i < n; i++)
+    c[i] = a[i] / b[i];
+}
+
+void FAST_ATTR
+vector_hf_rsqrt_fast (_Float16 * restrict a, _Float16 * restrict b,
+		    _Float16 * restrict c, int n)
+{
+  int i;
+  for (i = 0; i < n; i++)
+    c[i] = a[i] / __builtin_sqrtf16(b[i]);
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512fp16-recip-2.c b/gcc/testsuite/gcc.target/i386/avx512fp16-recip-2.c
new file mode 100644
index 0000000..ed7e0a2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512fp16-recip-2.c
@@ -0,0 +1,97 @@
+/* { dg-do run { target avx512fp16 } } */
+/* { dg-options "-O3 -mavx512fp16 -mavx512vl -ffast-math" } */
+
+static void recip_op_test (void);
+#define DO_TEST recip_op_test
+#define AVX512FP16
+#define AVX512VL
+#include "avx512f-check.h"
+#include "avx512fp16-recip-1.c"
+
+_Float16 a[32], b[32], vexp[32], vref[32], sa, sb, sexp, sref;
+
+#define NO_FAST_ATTR  \
+  __attribute__((noinline, noclone, \
+		 optimize("fast-math,trapping-math")))
+
+_Float16 NO_FAST_ATTR
+scalar_hf_rcp_no_fast (_Float16 a, _Float16 b)
+{
+  return a / b;
+}
+
+_Float16 NO_FAST_ATTR
+scalar_hf_rsqrt_no_fast (_Float16 a, _Float16 b)
+{
+  return a / __builtin_sqrtf16 (b);
+}
+
+void NO_FAST_ATTR
+vector_hf_rcp_no_fast (_Float16 * restrict a, _Float16 * restrict b,
+		    _Float16 * restrict c, int n)
+{
+  int i;
+  for (i = 0; i < n; i++)
+    c[i] = a[i] / b[i];
+}
+
+void NO_FAST_ATTR
+vector_hf_rsqrt_no_fast (_Float16 * restrict a, _Float16 * restrict b,
+		    _Float16 * restrict c, int n)
+{
+  int i;
+  for (i = 0; i < n; i++)
+    c[i] = a[i] / __builtin_sqrtf16 (b[i]);
+}
+
+void init()
+{
+  int i;
+  sa = 3.75;
+  sb = 6.25;
+  sexp = sref = 2.75;
+  for (i = 0; i < 32; i++)
+    {
+      a[i] = i + 0.5; 
+      b[i] = i * 1.5;
+      vexp[i] = vref[i] = 2.75 * i;
+    }
+}
+
+int check_cond(void *a, void *b, int size)
+{
+  int i;
+  unsigned short *pa = (unsigned short *)a,
+		 *pb = (unsigned short *)b;
+  for (i = 0; i < size; i++)
+    if (pa[i] != pb[i])
+      return 0;
+  return 1;
+}
+
+static void recip_op_test()
+{
+  init ();
+  sexp = scalar_hf_rcp_fast (sa, sb);
+  sref = scalar_hf_rcp_no_fast (sa, sb);
+  if (!check_cond (&sexp, &sref, 1))
+    abort ();
+
+  init ();
+  sexp = scalar_hf_rsqrt_fast (sa, sb);
+  sref = scalar_hf_rsqrt_no_fast (sa, sb);
+  if (!check_cond (&sexp, &sref, 1))
+    abort ();
+
+  init ();
+  vector_hf_rcp_fast (a, b, vexp, 32);
+  vector_hf_rcp_no_fast (a, b, vref, 32);
+  if (!check_cond (vexp, vref, 1))
+    abort ();
+
+  init ();
+  vector_hf_rsqrt_fast (a, b, vexp, 32);
+  vector_hf_rsqrt_no_fast (a, b, vref, 32);
+  if (!check_cond (vexp, vref, 1))
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512fp16-trunchf.c b/gcc/testsuite/gcc.target/i386/avx512fp16-trunchf.c
index 2c025b7..337a7b0 100644
--- a/gcc/testsuite/gcc.target/i386/avx512fp16-trunchf.c
+++ b/gcc/testsuite/gcc.target/i386/avx512fp16-trunchf.c
@@ -1,7 +1,7 @@
 /* { dg-do compile } */
 /* { dg-options "-O2 -mavx512fp16" } */
-/* { dg-final { scan-assembler-times "vcvttsh2si\[ \\t\]+\[^\{\n\]*(?:%xmm\[0-9\]|\\(%esp\\))+, %eax(?:\n|\[ \\t\]+#)" 3 } } */
-/* { dg-final { scan-assembler-times "vcvttsh2usi\[ \\t\]+\[^\{\n\]*(?:%xmm\[0-9\]|\\(%esp\\))+, %eax(?:\n|\[ \\t\]+#)" 2 } } */
+/* { dg-final { scan-assembler-times "vcvttsh2si\[ \\t\]+\[^\{\n\]*(?:%xmm\[0-9\]|\\(%e\[bs\]p\\))+, %eax(?:\n|\[ \\t\]+#)" 3 } } */
+/* { dg-final { scan-assembler-times "vcvttsh2usi\[ \\t\]+\[^\{\n\]*(?:%xmm\[0-9\]|\\(%e\[bs\]p\\))+, %eax(?:\n|\[ \\t\]+#)" 2 } } */
 /* { dg-final { scan-assembler-times "vcvttsh2si\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+, %rax(?:\n|\[ \\t\]+#)" 1 { target { ! ia32 } } } } */
 /* { dg-final { scan-assembler-times "vcvttsh2usi\[ \\t\]+\[^\{\n\]*%xmm\[0-9\]+, %rax(?:\n|\[ \\t\]+#)" 1 { target { ! ia32 } } } } */
 /* { dg-final { scan-assembler "xorl\[ \\t\]+%edx, %edx" { target ia32 } } } */
diff --git a/gcc/testsuite/gcc.target/i386/avx512fp16-vector-complex-float.c b/gcc/testsuite/gcc.target/i386/avx512fp16-vector-complex-float.c
new file mode 100644
index 0000000..7c579cb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512fp16-vector-complex-float.c
@@ -0,0 +1,40 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ffast-math -mavx512fp16 -mavx512vl -mprefer-vector-width=512" } */
+/* { dg-final { scan-assembler-times "vfmaddcph\[ \\t\]" 1 } } */
+/* { dg-final { scan-assembler-not "vfmadd\[123]*ph\[ \\t\]"} } */
+/* { dg-final { scan-assembler-not "vfmadd\[123]*sh\[ \\t\]"} } */
+/* { dg-final { scan-assembler-times "vfcmaddcph\[ \\t\]" 1 } } */
+/* { dg-final { scan-assembler-times "vfmulcph\[ \\t\]" 1 } } */
+/* { dg-final { scan-assembler-times "vfcmulcph\[ \\t\]" 1 } } */
+
+#include<complex.h>
+#define TYPE _Float16
+#define N 16
+
+void fma0 (_Complex TYPE *a, _Complex TYPE *b,
+           _Complex TYPE * __restrict c)
+{
+  for (int i = 0; i < N; i++)
+    c[i] += a[i] * b[i];
+}
+
+void fmaconj (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
+	      _Complex TYPE c[restrict N])
+{
+  for (int i = 0; i < N; i++)
+    c[i] += a[i] * ~b[i];
+}
+
+void fmul (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
+	   _Complex TYPE c[restrict N])
+{
+  for (int i = 0; i < N; i++)
+    c[i] = a[i] * b[i];
+}
+
+void fmulconj (_Complex TYPE a[restrict N], _Complex TYPE b[restrict N],
+	       _Complex TYPE c[restrict N])
+{
+  for (int i = 0; i < N; i++)
+    c[i] = a[i] * ~b[i];
+}
diff --git a/gcc/testsuite/gcc.target/i386/avx512fp16vl-complex-broadcast-1.c b/gcc/testsuite/gcc.target/i386/avx512fp16vl-complex-broadcast-1.c
new file mode 100644
index 0000000..3c8e8423
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512fp16vl-complex-broadcast-1.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512fp16 -mavx512vl" } */
+/* { dg-final { scan-assembler-times "\[^\n\]*\\\{1to4\\\}" 2 } }  */
+/* { dg-final { scan-assembler-times "\[^\n\]*\\\{1to8\\\}" 2 } }  */
+/* { dg-final { scan-assembler-times "\[^\n\]*\\\{1to16\\\}" 2 } }  */
+
+#include <immintrin.h>
+
+volatile __m512h res0, a0, c0;
+volatile __m256h res1, a1, c1;
+volatile __m128h res2, a2, c2;
+volatile _Float16 *b;
+
+void extern
+avx_test(void)
+{
+  res0 = _mm512_fmadd_pch (a0, _mm512_set1_pch(*(b + 2 * 6)), c0);
+  res0 = _mm512_fcmadd_pch (a0, _mm512_set1_pch(*(b + 2 * 6)), c0);
+
+  res1 = _mm256_fmadd_pch (a1, _mm256_set1_pch(*(b + 2 * 6)), c1);
+  res1 = _mm256_fcmadd_pch (a1, _mm256_set1_pch(*(b + 2 * 6)), c1);
+
+  res2 =  _mm_fmadd_pch (a2, _mm_set1_pch(*(b + 2 * 6)), c2);
+  res2 =  _mm_fcmadd_pch (a2, _mm_set1_pch(*(b + 2 * 6)), c2);
+}
diff --git a/gcc/testsuite/gcc.target/i386/ifcvt-reduction-logic-op.c b/gcc/testsuite/gcc.target/i386/ifcvt-reduction-logic-op.c
new file mode 100644
index 0000000..eeb822d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/ifcvt-reduction-logic-op.c
@@ -0,0 +1,80 @@
+/* PR tree-optimization/103126.  */
+/* { dg-do compile } */
+/* { dg-options "-Ofast -mavx2 -ftree-vectorize -fdump-tree-vect-details" } */
+/* { dg-final { scan-tree-dump-times "vectorized \[1-3] loops" 3 "vect" } } */
+#include<stdint.h>
+
+void xor_bit_arr_nolcd (uint64_t *__restrict mat, uint64_t* a,uint64_t* b, uint64_t *__restrict ans,
+    int64_t n)
+{
+  int64_t i;
+  uint64_t vec1, sum1;
+  uint64_t vec2, sum2;
+
+  while (n > 0) {
+    sum1 = 0;
+    vec1 = a[n];
+    sum2 = 0;
+    vec2 = b[n];
+
+    for (i = 0; i < 64; i++) {
+      uint64_t tmp = mat[i];
+      uint64_t vec1_i = (vec1 >> i);
+      uint64_t vec2_i = (vec2 >> i);
+      sum1 ^= (vec1_i & 1) ? tmp : 0;
+      if (vec2_i&1) sum2 ^= tmp;
+    }
+    *ans++ ^= sum1;  n--;
+    *ans++ ^= sum2;  n--;
+  }
+}
+
+void ior_bit_arr_nolcd (uint64_t *__restrict mat, uint64_t* a,uint64_t* b, uint64_t *__restrict ans,
+    int64_t n)
+{
+  int64_t i;
+  uint64_t vec1, sum1;
+  uint64_t vec2, sum2;
+
+  while (n > 0) {
+    sum1 = 0;
+    vec1 = a[n];
+    sum2 = 0;
+    vec2 = b[n];
+
+    for (i = 0; i < 64; i++) {
+      uint64_t tmp = mat[i];
+      uint64_t vec1_i = (vec1 >> i);
+      uint64_t vec2_i = (vec2 >> i);
+      sum1 |= (vec1_i & 1) ? tmp : 0;
+      if (vec2_i&1) sum2 |= tmp;
+    }
+    *ans++ |= sum1;  n--;
+    *ans++ |= sum2;  n--;
+  }
+}
+
+void and_bit_arr_nolcd (uint64_t *__restrict mat, uint64_t* a,uint64_t* b, uint64_t *__restrict ans,
+    int64_t n)
+{
+  int64_t i;
+  uint64_t vec1, sum1;
+  uint64_t vec2, sum2;
+
+  while (n > 0) {
+    sum1 = -1;
+    vec1 = a[n];
+    sum2 = 0;
+    vec2 = b[n];
+
+    for (i = 0; i < 64; i++) {
+      uint64_t tmp = mat[i];
+      uint64_t vec1_i = (vec1 >> i);
+      uint64_t vec2_i = (vec2 >> i);
+      sum1 &= (vec1_i & 1) ? tmp : -1;
+      if (vec2_i&1) sum2 &= tmp;
+    }
+    *ans++ &= sum1;  n--;
+    *ans++ &= sum2;  n--;
+  }
+}
diff --git a/gcc/testsuite/gcc.target/i386/pieces-memset-1.c b/gcc/testsuite/gcc.target/i386/pieces-memset-1.c
index 2b80326..f7487ba 100644
--- a/gcc/testsuite/gcc.target/i386/pieces-memset-1.c
+++ b/gcc/testsuite/gcc.target/i386/pieces-memset-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -mno-avx -msse2 -mtune=generic" } */
+/* { dg-options "-O2 -mno-avx -msse2 -mtune=generic -mno-stackrealign" } */
 
 extern char *dst;
 
diff --git a/gcc/testsuite/gcc.target/i386/pieces-memset-4.c b/gcc/testsuite/gcc.target/i386/pieces-memset-4.c
index 9256919..a12b9dd 100644
--- a/gcc/testsuite/gcc.target/i386/pieces-memset-4.c
+++ b/gcc/testsuite/gcc.target/i386/pieces-memset-4.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -mno-avx -msse2 -mtune=generic" } */
+/* { dg-options "-O2 -mno-avx -msse2 -mtune=generic -mno-stackrealign" } */
 
 extern char *dst;
 
diff --git a/gcc/testsuite/gcc.target/i386/pieces-memset-41.c b/gcc/testsuite/gcc.target/i386/pieces-memset-41.c
index f86b698..93df810 100644
--- a/gcc/testsuite/gcc.target/i386/pieces-memset-41.c
+++ b/gcc/testsuite/gcc.target/i386/pieces-memset-41.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -mno-avx2 -mavx -mtune=sandybridge" } */
+/* { dg-options "-O2 -mno-avx2 -mavx -mtune=sandybridge -mno-stackrealign" } */
 
 extern char *dst;
 
diff --git a/gcc/testsuite/gcc.target/i386/pieces-memset-7.c b/gcc/testsuite/gcc.target/i386/pieces-memset-7.c
index fd15986..0d02877 100644
--- a/gcc/testsuite/gcc.target/i386/pieces-memset-7.c
+++ b/gcc/testsuite/gcc.target/i386/pieces-memset-7.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -mno-avx -msse2 -mtune=generic" } */
+/* { dg-options "-O2 -mno-avx -msse2 -mtune=generic -mno-stackrealign" } */
 
 extern char *dst;
 
diff --git a/gcc/testsuite/gcc.target/i386/pieces-memset-8.c b/gcc/testsuite/gcc.target/i386/pieces-memset-8.c
index 7df0019..816d83e 100644
--- a/gcc/testsuite/gcc.target/i386/pieces-memset-8.c
+++ b/gcc/testsuite/gcc.target/i386/pieces-memset-8.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -mno-avx2 -mavx -mtune=generic" } */
+/* { dg-options "-O2 -mno-avx2 -mavx -mtune=generic -mno-stackrealign" } */
 
 extern char *dst;
 
diff --git a/gcc/testsuite/gcc.target/i386/pr100704-1.c b/gcc/testsuite/gcc.target/i386/pr100704-1.c
index 02461db..b292dc2 100644
--- a/gcc/testsuite/gcc.target/i386/pr100704-1.c
+++ b/gcc/testsuite/gcc.target/i386/pr100704-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile { target { ! ia32 } } } */
-/* { dg-options "-O2 -march=x86-64" } */
+/* { dg-options "-O2 -fomit-frame-pointer -march=x86-64" } */
 
 struct S
 {
diff --git a/gcc/testsuite/gcc.target/i386/pr100704-2.c b/gcc/testsuite/gcc.target/i386/pr100704-2.c
index 07b9bd1..d010658 100644
--- a/gcc/testsuite/gcc.target/i386/pr100704-2.c
+++ b/gcc/testsuite/gcc.target/i386/pr100704-2.c
@@ -1,5 +1,5 @@
 /* { dg-do compile { target { ! ia32 } } } */
-/* { dg-options "-O2 -march=x86-64" } */
+/* { dg-options "-O2 -fomit-frame-pointer -march=x86-64" } */
 
 struct S
 {
diff --git a/gcc/testsuite/gcc.target/i386/pr101346.c b/gcc/testsuite/gcc.target/i386/pr101346.c
index fefabaf..313753f 100644
--- a/gcc/testsuite/gcc.target/i386/pr101346.c
+++ b/gcc/testsuite/gcc.target/i386/pr101346.c
@@ -2,6 +2,7 @@
 /* { dg-do compile } */
 /* { dg-options "-O0 -fprofile-generate -msse" } */
 /* { dg-require-profiling "-fprofile-generate" } */
+/* { dg-require-effective-target dfp } */
 
 _Decimal128
 foo (_Decimal128 x)
diff --git a/gcc/testsuite/gcc.target/i386/pr101989-3.c b/gcc/testsuite/gcc.target/i386/pr101989-3.c
new file mode 100644
index 0000000..dfd8991
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr101989-3.c
@@ -0,0 +1,40 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512fp16 -mavx512vl" } */
+/* { dg-final { scan-assembler-times "vpternlog" 5 } } */
+/* { dg-final { scan-assembler-not "vpxor" } } */
+/* { dg-final { scan-assembler-not "vpor" } } */
+/* { dg-final { scan-assembler-not "vpand" } } */
+
+#include<immintrin.h>
+
+extern __m256i src1, src2, src3;
+
+__m256i
+foo (void)
+{
+  return (src2 & ~src1) | (src3 & src1);
+}
+
+__m256i
+foo1 (void)
+{
+  return (src2 & src1) | (src3 & ~src1);
+}
+
+__m256i
+foo2 (void)
+{
+  return (src2 & src1) | (~src3 & src1);
+}
+
+__m256i
+foo3 (void)
+{
+  return (~src2 & src1) | (src3 & src1);
+}
+
+__m256i
+foo4 (void)
+{
+  return src3 & src2 ^ src1;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr102464-copysign-1.c b/gcc/testsuite/gcc.target/i386/pr102464-copysign-1.c
new file mode 100644
index 0000000..95a3950
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102464-copysign-1.c
@@ -0,0 +1,80 @@
+/* PR target/102464.  */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512fp16 -mavx512vl -ftree-vectorize -mfpmath=sse -fdump-tree-optimized" } */
+
+#include<math.h>
+void foo1 (_Float16* __restrict a, _Float16* b, _Float16* c)
+{
+  for (int i = 0; i != 8; i++)
+    a[i] = copysignf (b[i], c[i]);
+}
+
+void foo2 (_Float16* __restrict a, _Float16* b, _Float16* c)
+{
+  for (int i = 0; i != 8; i++)
+    a[i] = copysign (b[i], c[i]);
+}
+
+void foo3 (_Float16* __restrict a, _Float16* b, _Float16* c)
+{
+  for (int i = 0; i != 8; i++)
+    a[i] = copysignl (b[i], c[i]);
+}
+
+void foo4 (float* __restrict a, float* b, float* c)
+{
+  for (int i = 0; i != 4; i++)
+    a[i] = copysign (b[i], c[i]);
+}
+
+void foo5 (float* __restrict a, float* b, float* c)
+{
+  for (int i = 0; i != 4; i++)
+    a[i] = copysignl (b[i], c[i]);
+}
+
+void foo6 (double* __restrict a, double* b, double* c)
+{
+  for (int i = 0; i != 4; i++)
+    a[i] = copysignl (b[i], c[i]);
+}
+
+void foo7 (_Float16* __restrict a, _Float16* b, _Float16* c)
+{
+  a[0] = copysignf (b[0], c[0]);
+}
+
+void foo8 (_Float16* __restrict a, _Float16* b, _Float16* c)
+{
+  a[0] = copysign (b[0], c[0]);
+}
+
+void foo9 (_Float16* __restrict a, _Float16* b, _Float16* c)
+{
+  a[0] = copysignl (b[0], c[0]);
+}
+
+void foo10 (float* __restrict a, float* b, float* c)
+{
+  a[0] = copysign (b[0], c[0]);
+}
+
+void foo11 (float* __restrict a, float* b, float* c)
+{
+  a[0] = copysignl (b[0], c[0]);
+}
+
+void foo12 (double* __restrict a, double* b, double* c)
+{
+  a[0] = copysignl (b[0], c[0]);
+}
+
+/* { dg-final { scan-assembler-not "vcvtsh2s\[sd\]" } } */
+/* { dg-final { scan-assembler-not "vcvtss2sd" } } */
+/* { dg-final { scan-assembler-not "fld" } } */
+/* { dg-final { scan-assembler-not "vcvtph2p\[sd\]" } } */
+/* { dg-final { scan-assembler-not "vcvtps2pd" } } */
+/* { dg-final { scan-assembler-not "extendhfxf" } } */
+/* { dg-final { scan-assembler-not "\\\{1to8\\\}" } } */
+/* { dg-final { scan-tree-dump-times "\.COPYSIGN" 12 "optimized" } } */
+/* { dg-final { scan-assembler-times "vpternlog" 12 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102464-fma.c b/gcc/testsuite/gcc.target/i386/pr102464-fma.c
new file mode 100644
index 0000000..9c70d93
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102464-fma.c
@@ -0,0 +1,32 @@
+/* PR target/102464.  */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512fp16 -mavx512vl -ffast-math -ftree-vectorize -mtune=generic -mfpmath=sse" } */
+/* { dg-final { scan-assembler-times "vfmadd...ph" 3 } }  */
+/* { dg-final { scan-assembler-times "vfmadd...sh" 3 } }  */
+/* { dg-final { scan-assembler-times "vfmadd...ps" 2 } }  */
+/* { dg-final { scan-assembler-times "vfmadd...ss" 2 } }  */
+/* { dg-final { scan-assembler-times "vfmadd...pd" 1 } }  */
+/* { dg-final { scan-assembler-times "vfmadd...sd" 1 } }  */
+
+#include<math.h>
+#define FOO(TYPE,SUFFIX)						\
+  void									\
+  foo_vect_##TYPE##SUFFIX (TYPE* __restrict a, TYPE* b, TYPE* c, TYPE* d) \
+  {									\
+    for (int i = 0; i != 8; i++)					\
+      a[i] = fma##SUFFIX (b[i], c[i], d[i]);				\
+  }									\
+  TYPE									\
+  foo_##TYPE##SUFFIX (TYPE b, TYPE c, TYPE d)				\
+  {									\
+    return fma##l (b, c, d);						\
+  }
+
+FOO (_Float16, f);
+FOO (_Float16,);
+FOO (_Float16, l);
+
+FOO (float,);
+FOO (float, l);
+
+FOO (double, l);
diff --git a/gcc/testsuite/gcc.target/i386/pr102464-maxmin.c b/gcc/testsuite/gcc.target/i386/pr102464-maxmin.c
new file mode 100644
index 0000000..3786723
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102464-maxmin.c
@@ -0,0 +1,44 @@
+/* PR target/102464.  */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512fp16 -mavx512vl -ffast-math -ftree-vectorize -mtune=generic -mfpmath=sse" } */
+/* { dg-final { scan-assembler-times "vmaxph" 3 } }  */
+/* { dg-final { scan-assembler-times "vminph" 3 } }  */
+/* { dg-final { scan-assembler-times "vmaxsh" 3 } }  */
+/* { dg-final { scan-assembler-times "vminsh" 3 } }  */
+/* { dg-final { scan-assembler-times "vmaxps" 2 } }  */
+/* { dg-final { scan-assembler-times "vminps" 2 } }  */
+/* { dg-final { scan-assembler-times "vmaxss" 2 } }  */
+/* { dg-final { scan-assembler-times "vminss" 2 } }  */
+/* { dg-final { scan-assembler-times "vmaxpd" 1 } }  */
+/* { dg-final { scan-assembler-times "vminpd" 1 } }  */
+/* { dg-final { scan-assembler-times "vmaxsd" 1 } }  */
+/* { dg-final { scan-assembler-times "vminsd" 1 } }  */
+
+#include<math.h>
+#define FOO(CODE,TYPE,SUFFIX)						\
+  void									\
+  foo_vect_##CODE##TYPE##SUFFIX (TYPE* __restrict a, TYPE* b, TYPE* c)	\
+  {									\
+    for (int i = 0; i != 8; i++)					\
+      a[i] = CODE##SUFFIX (b[i], c[i]);					\
+  }									\
+  TYPE									\
+  foo_##CODE##TYPE##SUFFIX (TYPE b, TYPE c)				\
+  {									\
+    return CODE##l (b, c);						\
+  }
+
+FOO (fmax, _Float16, f);
+FOO (fmax, _Float16,);
+FOO (fmax, _Float16, l);
+FOO (fmin, _Float16, f);
+FOO (fmin, _Float16,);
+FOO (fmin, _Float16, l);
+
+FOO (fmax, float,);
+FOO (fmax, float, l);
+FOO (fmin, float,);
+FOO (fmin, float, l);
+
+FOO (fmax, double, l);
+FOO (fmin, double, l);
diff --git a/gcc/testsuite/gcc.target/i386/pr102464-sqrtph.c b/gcc/testsuite/gcc.target/i386/pr102464-sqrtph.c
new file mode 100644
index 0000000..8bd19c6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102464-sqrtph.c
@@ -0,0 +1,27 @@
+/* PR target/102464.  */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512fp16 -mavx512vl -ffast-math -ftree-vectorize" } */
+
+#include<math.h>
+void foo1 (_Float16* __restrict a, _Float16* b)
+{
+  for (int i = 0; i != 8; i++)
+    a[i] =  sqrtf (b[i]);
+}
+
+void foo2 (_Float16* __restrict a, _Float16* b)
+{
+  for (int i = 0; i != 8; i++)
+    a[i] =  sqrt (b[i]);
+}
+
+void foo3 (_Float16* __restrict a, _Float16* b)
+{
+  for (int i = 0; i != 8; i++)
+    a[i] =  sqrtl (b[i]);
+}
+
+/* { dg-final { scan-assembler-not "vcvtsh2s\[sd\]" } } */
+/* { dg-final { scan-assembler-not "vcvtph2p\[sd\]" } } */
+/* { dg-final { scan-assembler-not "extendhfxf" } } */
+/* { dg-final { scan-assembler-times "vsqrtph\[^\n\r\]*xmm\[0-9\]" 3 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102464-sqrtsh.c b/gcc/testsuite/gcc.target/i386/pr102464-sqrtsh.c
new file mode 100644
index 0000000..4cf0089
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102464-sqrtsh.c
@@ -0,0 +1,23 @@
+/* PR target/102464.  */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx512fp16 -ffast-math" } */
+
+#include<math.h>
+_Float16 foo1 (_Float16 a)
+{
+  return sqrtf (a);
+}
+
+_Float16 foo2 (_Float16 a)
+{
+  return sqrt (a);
+}
+
+_Float16 foo3 (_Float16 a)
+{
+  return sqrtl (a);
+}
+
+/* { dg-final { scan-assembler-not "vcvtsh2s\[sd\]" } } */
+/* { dg-final { scan-assembler-not "extendhfxf" } } */
+/* { dg-final { scan-assembler-times "vsqrtsh\[^\n\r\]*xmm\[0-9\]" 3 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102464-vrndscaleph.c b/gcc/testsuite/gcc.target/i386/pr102464-vrndscaleph.c
new file mode 100644
index 0000000..a76d9e7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102464-vrndscaleph.c
@@ -0,0 +1,115 @@
+/* PR target/102464.  */
+/* { dg-do compile } */
+/* { dg-options "-Ofast -mavx512fp16 -mavx512vl -mprefer-vector-width=512" } */
+#include<math.h>
+void
+foo (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 8; i++)
+      a[i] = floor (b[i]);
+}
+
+void
+foo1 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 8; i++)
+      a[i] = ceil (b[i]);
+}
+
+void
+foo2 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 8; i++)
+      a[i] = trunc (b[i]);
+}
+
+void
+foo3 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 8; i++)
+      a[i] = nearbyint (b[i]);
+}
+
+void
+foo4 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 8; i++)
+      a[i] = rint (b[i]);
+}
+
+void
+foo5 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 16; i++)
+      a[i] = floor (b[i]);
+}
+
+void
+foo6 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 16; i++)
+      a[i] = ceil (b[i]);
+}
+
+void
+foo7 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 16; i++)
+      a[i] = trunc (b[i]);
+}
+
+void
+foo8 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 16; i++)
+      a[i] = nearbyint (b[i]);
+}
+
+void
+foo9 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 16; i++)
+      a[i] = rint (b[i]);
+}
+
+void
+foo10 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 32; i++)
+      a[i] = floor (b[i]);
+}
+
+void
+foo11 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 32; i++)
+      a[i] = ceil (b[i]);
+}
+
+void
+foo12 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 32; i++)
+      a[i] = trunc (b[i]);
+}
+
+void
+foo13 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 32; i++)
+      a[i] = nearbyint (b[i]);
+}
+
+void
+foo14 (_Float16* __restrict a, _Float16* b)
+{
+    for (int i = 0; i != 32; i++)
+      a[i] = rint (b[i]);
+}
+
+/* { dg-final { scan-assembler-not "vcvtsh2s\[sd\]" } } */
+/* { dg-final { scan-assembler-not "vcvtph2p\[sd\]" } } */
+/* { dg-final { scan-assembler-not "extendhfxf" } } */
+/* { dg-final { scan-assembler-times "vrndscaleph\[^\n\r\]*xmm\[0-9\]" 5 } } */
+/* { dg-final { scan-assembler-times "vrndscaleph\[^\n\r\]*ymm\[0-9\]" 5 } } */
+/* { dg-final { scan-assembler-times "vrndscaleph\[^\n\r\]*zmm\[0-9\]" 5 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102464.c b/gcc/testsuite/gcc.target/i386/pr102464.c
index e3e060e..7e1fbdc 100644
--- a/gcc/testsuite/gcc.target/i386/pr102464.c
+++ b/gcc/testsuite/gcc.target/i386/pr102464.c
@@ -1,6 +1,6 @@
 /* PR target/102464.  */
 /* { dg-do compile } */
-/* { dg-options "-O2 -mavx512fp16" } */
+/* { dg-options "-O2 -mavx512fp16 -fno-trapping-math" } */
 
 #define FOO(FUNC,SUFFIX)                       \
   _Float16                                     \
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-10a.c b/gcc/testsuite/gcc.target/i386/pr102566-10a.c
new file mode 100644
index 0000000..1c1f86a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-10a.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo (_Atomic int *v, int bit)
+{
+  int mask = 1 << bit;
+  return atomic_fetch_and_explicit (v, ~mask, memory_order_relaxed) & mask;
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btrl" 1 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-10b.c b/gcc/testsuite/gcc.target/i386/pr102566-10b.c
new file mode 100644
index 0000000..0bf3982
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-10b.c
@@ -0,0 +1,15 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo (_Atomic long long int *v, int bit)
+{
+  long long int mask = 1ll << bit;
+  return atomic_fetch_and_explicit (v, ~mask, memory_order_relaxed) & mask;
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btrq" 1 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-11.c b/gcc/testsuite/gcc.target/i386/pr102566-11.c
new file mode 100644
index 0000000..2c8f8c4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-11.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+#define MASK 0x1234
+
+bool
+foo1 (_Atomic int *v)
+{
+  return atomic_fetch_or_explicit (v, MASK, memory_order_relaxed) & MASK;
+}
+
+bool
+foo2 (_Atomic unsigned int *v, int mask)
+{
+  return atomic_fetch_or_explicit (v, mask, memory_order_relaxed) & mask;
+}
+
+bool
+foo3 (_Atomic unsigned int *v, int mask)
+{
+  return !(atomic_fetch_or_explicit (v, mask, memory_order_relaxed) & mask);
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*cmpxchg" 3 } } */
+/* { dg-final { scan-assembler-not "bts" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-12.c b/gcc/testsuite/gcc.target/i386/pr102566-12.c
new file mode 100644
index 0000000..4603a77
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-12.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+#define MASK 0x1234
+
+bool
+foo1 (_Atomic long *v)
+{
+  return atomic_fetch_and_explicit (v, ~MASK, memory_order_relaxed) & MASK;
+}
+
+bool
+foo2 (_Atomic long *v, long mask)
+{
+  return atomic_fetch_and_explicit (v, ~mask, memory_order_relaxed) & mask;
+}
+
+bool
+foo3 (_Atomic long *v, long mask)
+{
+  return !(atomic_fetch_and_explicit (v, ~mask, memory_order_relaxed) & mask);
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*cmpxchg" 3 } } */
+/* { dg-final { scan-assembler-not "btr" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-13.c b/gcc/testsuite/gcc.target/i386/pr102566-13.c
new file mode 100644
index 0000000..2657a2f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-13.c
@@ -0,0 +1,66 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+#include <stdatomic.h>
+#include <stdbool.h>
+
+#define FOO(TYPE,MASK)							\
+  __attribute__((noinline,noclone)) TYPE				\
+  atomic_fetch_or_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1 << MASK;						\
+    return __atomic_fetch_or (a, mask, __ATOMIC_RELAXED) & mask;	\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  atomic_fetch_xor_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1 << MASK;						\
+    return __atomic_fetch_xor (a, mask, __ATOMIC_RELAXED) & mask;	\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  atomic_xor_fetch_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1 << MASK;						\
+    return __atomic_xor_fetch (a, mask, __ATOMIC_RELAXED) & mask;	\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  atomic_fetch_and_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1 << MASK;						\
+    return __atomic_fetch_and (a, ~mask, __ATOMIC_RELAXED) & mask;	\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  sync_fetch_and_or_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1 << MASK;						\
+    return __sync_fetch_and_or (a, mask) & mask;			\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  sync_fetch_and_xor_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1 << MASK;						\
+    return __sync_fetch_and_xor (a, mask) & mask;			\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  sync_xor_and_fetch_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1 << MASK;						\
+    return __sync_xor_and_fetch (a, mask) & mask;			\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  sync_fetch_and_and_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1 << MASK;						\
+    return __sync_fetch_and_and (a, ~mask) & mask;			\
+  }									\
+
+FOO(short, 0);
+FOO(short, 7);
+FOO(short, 15);
+FOO(int, 0);
+FOO(int, 15);
+FOO(int, 31);
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*bts" 12 } } */
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btc" 24 } } */
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btr" 12 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-14.c b/gcc/testsuite/gcc.target/i386/pr102566-14.c
new file mode 100644
index 0000000..24681c1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-14.c
@@ -0,0 +1,65 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O2" } */
+#include <stdatomic.h>
+#include <stdbool.h>
+typedef long long int64;
+
+#define FOO(TYPE,MASK)							\
+  __attribute__((noinline,noclone)) TYPE				\
+  atomic_fetch_or_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1ll << MASK;						\
+    return __atomic_fetch_or (a, mask, __ATOMIC_RELAXED) & mask;	\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  atomic_fetch_xor_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1ll << MASK;						\
+    return __atomic_fetch_xor (a, mask, __ATOMIC_RELAXED) & mask;	\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  atomic_xor_fetch_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1ll << MASK;						\
+    return __atomic_xor_fetch (a, mask, __ATOMIC_RELAXED) & mask;	\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  atomic_fetch_and_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1ll << MASK;						\
+    return __atomic_fetch_and (a, ~mask, __ATOMIC_RELAXED) & mask;	\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  sync_fetch_and_or_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1ll << MASK;						\
+    return __sync_fetch_and_or (a, mask) & mask;			\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  sync_fetch_and_xor_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1ll << MASK;						\
+    return __sync_fetch_and_xor (a, mask) & mask;			\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  sync_xor_and_fetch_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1ll << MASK;						\
+    return __sync_xor_and_fetch (a, mask) & mask;			\
+  }									\
+  __attribute__((noinline,noclone)) TYPE				\
+  sync_fetch_and_and_##TYPE##_##MASK (_Atomic TYPE* a)			\
+  {									\
+    TYPE mask = 1ll << MASK;						\
+    return __sync_fetch_and_and (a, ~mask) & mask;			\
+  }									\
+
+
+FOO(int64, 0);
+FOO(int64, 32);
+FOO(int64, 63);
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*bts" 6 } } */
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btc" 12 } } */
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btr" 6 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-1a.c b/gcc/testsuite/gcc.target/i386/pr102566-1a.c
new file mode 100644
index 0000000..a915de3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-1a.c
@@ -0,0 +1,188 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+void bar (void);
+
+__attribute__((noinline, noclone)) int
+f1 (int *a, int bit)
+{
+  int mask = 1 << bit;
+  return (__sync_fetch_and_or (a, mask) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f2 (int *a, int bit)
+{
+  int mask = 1 << bit;
+  int t1 = __atomic_fetch_or (a, mask, __ATOMIC_RELAXED);
+  int t2 = t1 & mask;
+  return t2 != 0;
+}
+
+__attribute__((noinline, noclone)) long int
+f3 (long int *a, int bit)
+{
+  long int mask = 1l << bit;
+  return (__atomic_fetch_or (a, mask, __ATOMIC_SEQ_CST) & mask) == 0;
+}
+
+__attribute__((noinline, noclone)) int
+f4 (int *a)
+{
+  int mask = 1 << 7;
+  return (__sync_fetch_and_or (a, mask) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f5 (int *a)
+{
+  int mask = 1 << 13;
+  return (__atomic_fetch_or (a, mask, __ATOMIC_RELAXED) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f6 (int *a)
+{
+  int mask = 1 << 0;
+  return (__atomic_fetch_or (a, mask, __ATOMIC_SEQ_CST) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) void
+f7 (int *a, int bit)
+{
+  int mask = 1 << bit;
+  if ((__sync_fetch_and_xor (a, mask) & mask) != 0)
+    bar ();
+}
+
+__attribute__((noinline, noclone)) void
+f8 (int *a, int bit)
+{
+  int mask = 1 << bit;
+  if ((__atomic_fetch_xor (a, mask, __ATOMIC_RELAXED) & mask) == 0)
+    bar ();
+}
+
+__attribute__((noinline, noclone)) int
+f9 (int *a, int bit)
+{
+  int mask = 1 << bit;
+  return (__atomic_fetch_xor (a, mask, __ATOMIC_SEQ_CST) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f10 (int *a)
+{
+  int mask = 1 << 7;
+  return (__sync_fetch_and_xor (a, mask) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f11 (int *a)
+{
+  int mask = 1 << 13;
+  return (__atomic_fetch_xor (a, mask, __ATOMIC_RELAXED) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f12 (int *a)
+{
+  int mask = 1 << 0;
+  return (__atomic_fetch_xor (a, mask, __ATOMIC_SEQ_CST) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f13 (int *a, int bit)
+{
+  int mask = 1 << bit;
+  return (__sync_fetch_and_and (a, ~mask) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f14 (int *a, int bit)
+{
+  int mask = 1 << bit;
+  return (__atomic_fetch_and (a, ~mask, __ATOMIC_RELAXED) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f15 (int *a, int bit)
+{
+  int mask = 1 << bit;
+  return (__atomic_fetch_and (a, ~mask, __ATOMIC_SEQ_CST) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f16 (int *a)
+{
+  int mask = 1 << 7;
+  return (__sync_fetch_and_and (a, ~mask) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f17 (int *a)
+{
+  int mask = 1 << 13;
+  return (__atomic_fetch_and (a, ~mask, __ATOMIC_RELAXED) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) int
+f18 (int *a)
+{
+  int mask = 1 << 0;
+  return (__atomic_fetch_and (a, ~mask, __ATOMIC_SEQ_CST) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) long int
+f19 (long int *a, int bit)
+{
+  long int mask = 1l << bit;
+  return (__atomic_xor_fetch (a, mask, __ATOMIC_SEQ_CST) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) long int
+f20 (long int *a)
+{
+  long int mask = 1l << 7;
+  return (__atomic_xor_fetch (a, mask, __ATOMIC_SEQ_CST) & mask) == 0;
+}
+
+__attribute__((noinline, noclone)) int
+f21 (int *a, int bit)
+{
+  int mask = 1 << bit;
+  return (__sync_fetch_and_or (a, mask) & mask);
+}
+
+__attribute__((noinline, noclone)) long int
+f22 (long int *a)
+{
+  long int mask = 1l << 7;
+  return (__atomic_xor_fetch (a, mask, __ATOMIC_SEQ_CST) & mask);
+}
+
+__attribute__((noinline, noclone)) long int
+f23 (long int *a)
+{
+  long int mask = 1l << 7;
+  return (__atomic_fetch_xor (a, mask, __ATOMIC_SEQ_CST) & mask);
+}
+
+__attribute__((noinline, noclone)) short int
+f24 (short int *a)
+{
+  short int mask = 1 << 7;
+  return (__sync_fetch_and_or (a, mask) & mask) != 0;
+}
+
+__attribute__((noinline, noclone)) short int
+f25 (short int *a)
+{
+  short int mask = 1 << 7;
+  return (__atomic_fetch_or (a, mask, __ATOMIC_SEQ_CST) & mask) != 0;
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*bts" 9 } } */
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btc" 10 } } */
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btr" 6 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-1b.c b/gcc/testsuite/gcc.target/i386/pr102566-1b.c
new file mode 100644
index 0000000..c4dab81
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-1b.c
@@ -0,0 +1,107 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -g" } */
+
+int cnt;
+
+__attribute__((noinline, noclone)) void
+bar (void)
+{
+  cnt++;
+}
+
+#include "pr102566-1a.c"
+
+int a;
+long int b;
+unsigned long int c;
+unsigned short int d;
+
+int
+main ()
+{
+  __atomic_store_n (&a, 15, __ATOMIC_RELAXED);
+  if (f1 (&a, 2) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 15
+      || f1 (&a, 4) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 31)
+    __builtin_abort ();
+  if (f2 (&a, 1) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 31
+      || f2 (&a, 5) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 63)
+    __builtin_abort ();
+  __atomic_store_n (&b, 24, __ATOMIC_RELAXED);
+  if (f3 (&b, 2) != 1 || __atomic_load_n (&b, __ATOMIC_RELAXED) != 28
+      || f3 (&b, 3) != 0 || __atomic_load_n (&b, __ATOMIC_RELAXED) != 28)
+    __builtin_abort ();
+  __atomic_store_n (&a, 0, __ATOMIC_RELAXED);
+  if (f4 (&a) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 128
+      || f4 (&a) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 128)
+    __builtin_abort ();
+  if (f5 (&a) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8320
+      || f5 (&a) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8320)
+    __builtin_abort ();
+  if (f6 (&a) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8321
+      || f6 (&a) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8321)
+    __builtin_abort ();
+  if (cnt != 0
+      || (f7 (&a, 7), cnt) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8193
+      || (f7 (&a, 7), cnt) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8321)
+    __builtin_abort ();
+  if ((f8 (&a, 7), cnt) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8193
+      || (f8 (&a, 7), cnt) != 2 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8321)
+    __builtin_abort ();
+  if (f9 (&a, 13) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 129
+      || f9 (&a, 13) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8321)
+    __builtin_abort ();
+  if (f10 (&a) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8193
+      || f10 (&a) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8321)
+    __builtin_abort ();
+  if (f11 (&a) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 129
+      || f11 (&a) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8321)
+    __builtin_abort ();
+  if (f12 (&a) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8320
+      || f12 (&a) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8321)
+    __builtin_abort ();
+  if (f13 (&a, 7) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8193
+      || f13 (&a, 7) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8193)
+    __builtin_abort ();
+  if (f14 (&a, 13) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 1
+      || f14 (&a, 13) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 1)
+    __builtin_abort ();
+  if (f15 (&a, 0) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 0
+      || f15 (&a, 0) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 0)
+    __builtin_abort ();
+  __atomic_store_n (&a, 8321, __ATOMIC_RELAXED);
+  if (f16 (&a) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8193
+      || f16 (&a) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 8193)
+    __builtin_abort ();
+  if (f17 (&a) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 1
+      || f17 (&a) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 1)
+    __builtin_abort ();
+  if (f18 (&a) != 1 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 0
+      || f18 (&a) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 0)
+    __builtin_abort ();
+  if (f19 (&c, 7) != 1 || __atomic_load_n (&c, __ATOMIC_RELAXED) != 128
+      || f19 (&c, 7) != 0 || __atomic_load_n (&c, __ATOMIC_RELAXED) != 0)
+    __builtin_abort ();
+  if (f20 (&c) != 0 || __atomic_load_n (&c, __ATOMIC_RELAXED) != 128
+      || f20 (&c) != 1 || __atomic_load_n (&c, __ATOMIC_RELAXED) != 0)
+    __builtin_abort ();
+  __atomic_store_n (&a, 128, __ATOMIC_RELAXED);
+  if (f21 (&a, 4) != 0 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 144
+      || f21 (&a, 4) != 16 || __atomic_load_n (&a, __ATOMIC_RELAXED) != 144)
+    __builtin_abort ();
+  __atomic_store_n (&c, 1, __ATOMIC_RELAXED);
+  if (f22 (&c) != 128 || __atomic_load_n (&c, __ATOMIC_RELAXED) != 129
+      || f22 (&c) != 0 || __atomic_load_n (&c, __ATOMIC_RELAXED) != 1)
+    __builtin_abort ();
+  if (f23 (&c) != 0 || __atomic_load_n (&c, __ATOMIC_RELAXED) != 129
+      || f23 (&c) != 128 || __atomic_load_n (&c, __ATOMIC_RELAXED) != 1)
+    __builtin_abort ();
+  if (f24 (&d) != 0 || __atomic_load_n (&d, __ATOMIC_RELAXED) != 128
+      || f24 (&d) != 1 || __atomic_load_n (&d, __ATOMIC_RELAXED) != 128)
+    __builtin_abort ();
+  __atomic_store_n (&d, 1, __ATOMIC_RELAXED);
+  if (f25 (&d) != 0 || __atomic_load_n (&d, __ATOMIC_RELAXED) != 129
+      || f25 (&d) != 1 || __atomic_load_n (&d, __ATOMIC_RELAXED) != 129
+      || cnt != 2)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-2.c b/gcc/testsuite/gcc.target/i386/pr102566-2.c
new file mode 100644
index 0000000..00a7c34
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-2.c
@@ -0,0 +1,32 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo0 (_Atomic int *v)
+{
+#define BIT (1 << 0)
+  return atomic_fetch_or_explicit (v, BIT, memory_order_relaxed) & BIT;
+#undef BIT
+}
+
+bool
+foo30 (_Atomic int *v)
+{
+#define BIT (1 << 30)
+  return atomic_fetch_or_explicit (v, BIT, memory_order_relaxed) & BIT;
+#undef BIT
+}
+
+bool
+foo31 (_Atomic int *v)
+{
+#define BIT (1 << 31)
+  return atomic_fetch_or_explicit (v, BIT, memory_order_relaxed) & BIT;
+#undef BIT
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btsl" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-3a.c b/gcc/testsuite/gcc.target/i386/pr102566-3a.c
new file mode 100644
index 0000000..8bf1cd6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-3a.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo (_Atomic int *v, int bit)
+{
+  int mask = 1 << bit;
+  return atomic_fetch_or_explicit (v, mask, memory_order_relaxed) & mask;
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btsl" 1 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-3b.c b/gcc/testsuite/gcc.target/i386/pr102566-3b.c
new file mode 100644
index 0000000..d155ed3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-3b.c
@@ -0,0 +1,15 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo (_Atomic long long int *v, int bit)
+{
+  long long int mask = 1ll << bit;
+  return atomic_fetch_or_explicit (v, mask, memory_order_relaxed) & mask;
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btsq" 1 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-4.c b/gcc/testsuite/gcc.target/i386/pr102566-4.c
new file mode 100644
index 0000000..2668ccf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-4.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo (_Atomic int *v, int bit)
+{
+  unsigned int mask = 1 << bit;
+  return atomic_fetch_or_explicit (v, mask, memory_order_relaxed) & mask;
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btsl" 1 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-5.c b/gcc/testsuite/gcc.target/i386/pr102566-5.c
new file mode 100644
index 0000000..8bf1cd6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-5.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo (_Atomic int *v, int bit)
+{
+  int mask = 1 << bit;
+  return atomic_fetch_or_explicit (v, mask, memory_order_relaxed) & mask;
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btsl" 1 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-6.c b/gcc/testsuite/gcc.target/i386/pr102566-6.c
new file mode 100644
index 0000000..3dfe55a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-6.c
@@ -0,0 +1,32 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo0 (_Atomic int *v)
+{
+#define BIT (1 << 0)
+  return !(atomic_fetch_or_explicit (v, BIT, memory_order_relaxed) & BIT);
+#undef BIT
+}
+
+bool
+foo30 (_Atomic int *v)
+{
+#define BIT (1 << 30)
+  return !(atomic_fetch_or_explicit (v, BIT, memory_order_relaxed) & BIT);
+#undef BIT
+}
+
+bool
+foo31 (_Atomic int *v)
+{
+#define BIT (1 << 31)
+  return !(atomic_fetch_or_explicit (v, BIT, memory_order_relaxed) & BIT);
+#undef BIT
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btsl" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-7.c b/gcc/testsuite/gcc.target/i386/pr102566-7.c
new file mode 100644
index 0000000..6bc0ae0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-7.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+typedef int __attribute__ ((mode (__word__))) int_type;
+
+#define BIT (1 << 0)
+
+bool
+foo0 (_Atomic int_type *v)
+{
+  return atomic_fetch_or_explicit (v, BIT, memory_order_relaxed) & ~1;
+}
+
+bool
+foo1 (_Atomic int_type *v)
+{
+  return atomic_fetch_or_explicit (v, BIT, memory_order_relaxed) & ~2;
+}
+
+bool
+foo2 (_Atomic int_type *v)
+{
+  return atomic_fetch_or_explicit (v, BIT, memory_order_relaxed) & ~3;
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*cmpxchg" 3 } } */
+/* { dg-final { scan-assembler-not "bts" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-8a.c b/gcc/testsuite/gcc.target/i386/pr102566-8a.c
new file mode 100644
index 0000000..168e3db
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-8a.c
@@ -0,0 +1,32 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo0 (_Atomic int *v)
+{
+#define BIT (1 << 0)
+  return atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT;
+#undef BIT
+}
+
+bool
+foo30 (_Atomic int *v)
+{
+#define BIT (1 << 30)
+  return atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT;
+#undef BIT
+}
+
+bool
+foo31 (_Atomic int *v)
+{
+#define BIT (1 << 31)
+  return atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT;
+#undef BIT
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btrl" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-8b.c b/gcc/testsuite/gcc.target/i386/pr102566-8b.c
new file mode 100644
index 0000000..392da30
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-8b.c
@@ -0,0 +1,32 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo0 (_Atomic long long *v)
+{
+#define BIT (1ll << 0)
+  return atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT;
+#undef BIT
+}
+
+bool
+foo30 (_Atomic long long *v)
+{
+#define BIT (1ll << 62)
+  return atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT;
+#undef BIT
+}
+
+bool
+foo31 (_Atomic long long *v)
+{
+#define BIT (1ll << 63)
+  return atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT;
+#undef BIT
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btrq" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-9a.c b/gcc/testsuite/gcc.target/i386/pr102566-9a.c
new file mode 100644
index 0000000..3fa2a3e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-9a.c
@@ -0,0 +1,32 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo0 (_Atomic int *v)
+{
+#define BIT (1 << 0)
+  return !(atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT);
+#undef BIT
+}
+
+bool
+foo30 (_Atomic int *v)
+{
+#define BIT (1 << 30)
+  return !(atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT);
+#undef BIT
+}
+
+bool
+foo31 (_Atomic int *v)
+{
+#define BIT (1 << 31)
+  return !(atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT);
+#undef BIT
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btrl" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr102566-9b.c b/gcc/testsuite/gcc.target/i386/pr102566-9b.c
new file mode 100644
index 0000000..38ddbdc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr102566-9b.c
@@ -0,0 +1,32 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O2" } */
+
+#include <stdatomic.h>
+#include <stdbool.h>
+
+bool
+foo0 (_Atomic long long *v)
+{
+#define BIT (1ll << 0)
+  return !(atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT);
+#undef BIT
+}
+
+bool
+foo30 (_Atomic long long *v)
+{
+#define BIT (1ll << 62)
+  return !(atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT);
+#undef BIT
+}
+
+bool
+foo31 (_Atomic long long *v)
+{
+#define BIT (1ll << 63)
+  return !(atomic_fetch_and_explicit (v, ~BIT, memory_order_relaxed) & BIT);
+#undef BIT
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btrq" 3 } } */
+/* { dg-final { scan-assembler-not "cmpxchg" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr103020.c b/gcc/testsuite/gcc.target/i386/pr103020.c
new file mode 100644
index 0000000..75253c7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr103020.c
@@ -0,0 +1,11 @@
+/* PR target/103020 */
+/* { dg-do compile { target { ! int128 } } } */
+/* { dg-additional-options "-mavx512f" } */
+
+typedef int TI __attribute__((mode (TI)));	/* { dg-error "unable to emulate" } */
+typedef int V1TI __attribute__((mode (V1TI)));	/* { dg-error "unable to emulate" } */
+typedef int V2TI __attribute__((mode (V2TI)));	/* { dg-error "unable to emulate" } */
+typedef int V4TI __attribute__((mode (V4TI)));	/* { dg-error "unable to emulate" } */
+/* { dg-warning "is deprecated" "V1TI" { target *-*-* } .-3 } */
+/* { dg-warning "is deprecated" "V2TI" { target *-*-* } .-3 } */
+/* { dg-warning "is deprecated" "V4TI" { target *-*-* } .-3 } */
diff --git a/gcc/testsuite/gcc.target/i386/pr103069-1.c b/gcc/testsuite/gcc.target/i386/pr103069-1.c
new file mode 100644
index 0000000..f819af4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr103069-1.c
@@ -0,0 +1,35 @@
+/* PR target/103068 */
+/* { dg-do compile } */
+/* { dg-additional-options "-O2 -march=x86-64 -mtune=generic -mrelax-cmpxchg-loop" } */ 
+/* { dg-final { scan-assembler-times "rep;?\[ \\t\]+nop" 32 } } */
+
+#include <stdint.h>
+
+#define FUNC_ATOMIC(TYPE, OP) \
+__attribute__ ((noinline, noclone))	\
+TYPE f_##TYPE##_##OP##_fetch (TYPE *a, TYPE b)	\
+{ \
+  return __atomic_##OP##_fetch (a, b, __ATOMIC_RELAXED);  \
+} \
+__attribute__ ((noinline, noclone))	\
+TYPE f_##TYPE##_fetch_##OP (TYPE *a, TYPE b)	\
+{ \
+  return __atomic_fetch_##OP (a, b, __ATOMIC_RELAXED);  \
+}
+
+FUNC_ATOMIC (int64_t, and)
+FUNC_ATOMIC (int64_t, nand)
+FUNC_ATOMIC (int64_t, or)
+FUNC_ATOMIC (int64_t, xor)
+FUNC_ATOMIC (int, and)
+FUNC_ATOMIC (int, nand)
+FUNC_ATOMIC (int, or)
+FUNC_ATOMIC (int, xor)
+FUNC_ATOMIC (short, and)
+FUNC_ATOMIC (short, nand)
+FUNC_ATOMIC (short, or)
+FUNC_ATOMIC (short, xor)
+FUNC_ATOMIC (char, and)
+FUNC_ATOMIC (char, nand)
+FUNC_ATOMIC (char, or)
+FUNC_ATOMIC (char, xor)
diff --git a/gcc/testsuite/gcc.target/i386/pr103069-2.c b/gcc/testsuite/gcc.target/i386/pr103069-2.c
new file mode 100644
index 0000000..8ac824c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr103069-2.c
@@ -0,0 +1,70 @@
+/* PR target/103068 */
+/* { dg-do compile } */
+/* { dg-additional-options "-O2 -march=x86-64 -mtune=generic" } */ 
+
+#include <stdlib.h>
+#include "pr103069-1.c"
+
+#define FUNC_ATOMIC_RELAX(TYPE, OP) \
+__attribute__ ((noinline, noclone, target ("relax-cmpxchg-loop")))	\
+TYPE relax_##TYPE##_##OP##_fetch (TYPE *a, TYPE b)	\
+{ \
+  return __atomic_##OP##_fetch (a, b, __ATOMIC_RELAXED);  \
+} \
+__attribute__ ((noinline, noclone, target ("relax-cmpxchg-loop")))	\
+TYPE relax_##TYPE##_fetch_##OP (TYPE *a, TYPE b)	\
+{ \
+  return __atomic_fetch_##OP (a, b, __ATOMIC_RELAXED);  \
+}
+
+FUNC_ATOMIC_RELAX (int64_t, and)
+FUNC_ATOMIC_RELAX (int64_t, nand)
+FUNC_ATOMIC_RELAX (int64_t, or)
+FUNC_ATOMIC_RELAX (int64_t, xor)
+FUNC_ATOMIC_RELAX (int, and)
+FUNC_ATOMIC_RELAX (int, nand)
+FUNC_ATOMIC_RELAX (int, or)
+FUNC_ATOMIC_RELAX (int, xor)
+FUNC_ATOMIC_RELAX (short, and)
+FUNC_ATOMIC_RELAX (short, nand)
+FUNC_ATOMIC_RELAX (short, or)
+FUNC_ATOMIC_RELAX (short, xor)
+FUNC_ATOMIC_RELAX (char, and)
+FUNC_ATOMIC_RELAX (char, nand)
+FUNC_ATOMIC_RELAX (char, or)
+FUNC_ATOMIC_RELAX (char, xor)
+
+#define TEST_ATOMIC_FETCH_LOGIC(TYPE, OP) \
+{ \
+  TYPE a = 11, b = 101, res, exp; \
+  res = relax_##TYPE##_##OP##_fetch (&a, b); \
+  exp = f_##TYPE##_##OP##_fetch (&a, b);  \
+  if (res != exp) \
+    abort (); \
+  a = 21, b = 92; \
+  res = relax_##TYPE##_fetch_##OP (&a, b); \
+  exp = f_##TYPE##_fetch_##OP (&a, b);  \
+  if (res != exp) \
+    abort (); \
+}
+
+int main (void)
+{
+  TEST_ATOMIC_FETCH_LOGIC (int64_t, and)
+  TEST_ATOMIC_FETCH_LOGIC (int64_t, nand)
+  TEST_ATOMIC_FETCH_LOGIC (int64_t, or)
+  TEST_ATOMIC_FETCH_LOGIC (int64_t, xor)
+  TEST_ATOMIC_FETCH_LOGIC (int, and)
+  TEST_ATOMIC_FETCH_LOGIC (int, nand)
+  TEST_ATOMIC_FETCH_LOGIC (int, or)
+  TEST_ATOMIC_FETCH_LOGIC (int, xor)
+  TEST_ATOMIC_FETCH_LOGIC (short, and)
+  TEST_ATOMIC_FETCH_LOGIC (short, nand)
+  TEST_ATOMIC_FETCH_LOGIC (short, or)
+  TEST_ATOMIC_FETCH_LOGIC (short, xor)
+  TEST_ATOMIC_FETCH_LOGIC (char, and)
+  TEST_ATOMIC_FETCH_LOGIC (char, nand)
+  TEST_ATOMIC_FETCH_LOGIC (char, or)
+  TEST_ATOMIC_FETCH_LOGIC (char, xor)
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr103151.c b/gcc/testsuite/gcc.target/i386/pr103151.c
new file mode 100644
index 0000000..d7db255
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr103151.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-mavx512fp16 -O1 -mxop" } */
+
+typedef _Float16 v16hf __attribute__((vector_size(32)));
+typedef _Float16 v8hf __attribute__((vector_size(16)));
+v8hf f1() {
+  int i;
+  v8hf z;
+  z[i] = i;
+  return z;
+}
+
+v16hf f2() {
+  int i;
+  v16hf z;
+  z[i] = i;
+  return z;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr103205-2.c b/gcc/testsuite/gcc.target/i386/pr103205-2.c
new file mode 100644
index 0000000..705081e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr103205-2.c
@@ -0,0 +1,46 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mtune-ctrl=^himode_math" } */
+
+extern short foo;
+extern unsigned short bar;
+
+int
+foo1 (void)
+{
+  return __sync_fetch_and_and (&foo, ~1) & 1;
+}
+
+int
+foo2 (void)
+{
+  return __sync_fetch_and_or (&foo, 1) & 1;
+}
+
+int
+foo3 (void)
+{
+  return __sync_fetch_and_xor (&foo, 1) & 1;
+}
+
+unsigned short
+bar1 (void)
+{
+  return __sync_fetch_and_and (&bar, ~1) & 1;
+}
+
+unsigned short
+bar2 (void)
+{
+  return __sync_fetch_and_or (&bar, 1) & 1;
+}
+
+unsigned short
+bar3 (void)
+{
+  return __sync_fetch_and_xor (&bar, 1) & 1;
+}
+
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btrw" 2 } } */
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btsw" 2 } } */
+/* { dg-final { scan-assembler-times "lock;?\[ \t\]*btcw" 2 } } */
+/* { dg-final { scan-assembler-not "cmpxchgw" } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr103205.c b/gcc/testsuite/gcc.target/i386/pr103205.c
new file mode 100644
index 0000000..eaeea82
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr103205.c
@@ -0,0 +1,11 @@
+/* PR target/103205 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -mtune-ctrl=^himode_math" } */
+
+unsigned short a;
+
+unsigned short
+foo (void)
+{
+  return __sync_fetch_and_and (&a, ~1) & 1;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr22076.c b/gcc/testsuite/gcc.target/i386/pr22076.c
index 427ffcd..766b732 100644
--- a/gcc/testsuite/gcc.target/i386/pr22076.c
+++ b/gcc/testsuite/gcc.target/i386/pr22076.c
@@ -1,9 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -fomit-frame-pointer -mmmx -mno-sse2" } */
+/* { dg-options "-O2 -fomit-frame-pointer -mmmx -msse -mno-sse2" } */
 /* { dg-additional-options "-fno-common" { target *-*-darwin* } } */
 /* { dg-additional-options "-mdynamic-no-pic" { target { ia32 && *-*-darwin* } } } */
 
-#include <mmintrin.h>
+#include <xmmintrin.h>
 
 __m64 x;
 
@@ -12,7 +12,7 @@ void test ()
   __m64 mm0 = (__m64)(__v8qi) {1,2,3,4,5,6,7,8};
   __m64 mm1 = (__m64)(__v8qi) {11,22,33,44,55,66,77,88};
 
-  x = _mm_add_pi8 (mm0, mm1);
+  x = _mm_sad_pu8 (mm0, mm1);
 }
 
 /* { dg-final { scan-assembler-times "movq" 2 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr90773-1.c b/gcc/testsuite/gcc.target/i386/pr90773-1.c
index 4fd5a40..d51e287 100644
--- a/gcc/testsuite/gcc.target/i386/pr90773-1.c
+++ b/gcc/testsuite/gcc.target/i386/pr90773-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -msse2 -mtune=generic" } */
+/* { dg-options "-O2 -msse2 -mtune=generic -mno-stackrealign" } */
 
 extern char *dst, *src;
 
diff --git a/gcc/testsuite/gcc.target/i386/sse2-v1ti-ashiftrt-1.c b/gcc/testsuite/gcc.target/i386/sse2-v1ti-ashiftrt-1.c
new file mode 100644
index 0000000..05869bf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/sse2-v1ti-ashiftrt-1.c
@@ -0,0 +1,167 @@
+/* { dg-do run { target int128 } } */
+/* { dg-options "-O2 -msse2" } */
+/* { dg-require-effective-target sse2 } */
+
+typedef __int128 v1ti __attribute__ ((__vector_size__ (16)));
+typedef __int128 ti;
+
+ti ashr(ti x, unsigned int i) { return x >> i; }
+
+v1ti ashr_1(v1ti x) { return x >> 1; }
+v1ti ashr_2(v1ti x) { return x >> 2; }
+v1ti ashr_7(v1ti x) { return x >> 7; }
+v1ti ashr_8(v1ti x) { return x >> 8; }
+v1ti ashr_9(v1ti x) { return x >> 9; }
+v1ti ashr_15(v1ti x) { return x >> 15; }
+v1ti ashr_16(v1ti x) { return x >> 16; }
+v1ti ashr_17(v1ti x) { return x >> 17; }
+v1ti ashr_23(v1ti x) { return x >> 23; }
+v1ti ashr_24(v1ti x) { return x >> 24; }
+v1ti ashr_25(v1ti x) { return x >> 25; }
+v1ti ashr_31(v1ti x) { return x >> 31; }
+v1ti ashr_32(v1ti x) { return x >> 32; }
+v1ti ashr_33(v1ti x) { return x >> 33; }
+v1ti ashr_47(v1ti x) { return x >> 47; }
+v1ti ashr_48(v1ti x) { return x >> 48; }
+v1ti ashr_49(v1ti x) { return x >> 49; }
+v1ti ashr_63(v1ti x) { return x >> 63; }
+v1ti ashr_64(v1ti x) { return x >> 64; }
+v1ti ashr_65(v1ti x) { return x >> 65; }
+v1ti ashr_72(v1ti x) { return x >> 72; }
+v1ti ashr_79(v1ti x) { return x >> 79; }
+v1ti ashr_80(v1ti x) { return x >> 80; }
+v1ti ashr_81(v1ti x) { return x >> 81; }
+v1ti ashr_95(v1ti x) { return x >> 95; }
+v1ti ashr_96(v1ti x) { return x >> 96; }
+v1ti ashr_97(v1ti x) { return x >> 97; }
+v1ti ashr_111(v1ti x) { return x >> 111; }
+v1ti ashr_112(v1ti x) { return x >> 112; }
+v1ti ashr_113(v1ti x) { return x >> 113; }
+v1ti ashr_119(v1ti x) { return x >> 119; }
+v1ti ashr_120(v1ti x) { return x >> 120; }
+v1ti ashr_121(v1ti x) { return x >> 121; }
+v1ti ashr_126(v1ti x) { return x >> 126; }
+v1ti ashr_127(v1ti x) { return x >> 127; }
+
+typedef v1ti (*fun)(v1ti);
+
+struct {
+  unsigned int i;
+  fun ashr;
+} table[35] = {
+  {   1, ashr_1   },
+  {   2, ashr_2   },
+  {   7, ashr_7   },
+  {   8, ashr_8   },
+  {   9, ashr_9   },
+  {  15, ashr_15  },
+  {  16, ashr_16  },
+  {  17, ashr_17  },
+  {  23, ashr_23  },
+  {  24, ashr_24  },
+  {  25, ashr_25  },
+  {  31, ashr_31  },
+  {  32, ashr_32  },
+  {  33, ashr_33  },
+  {  47, ashr_47  },
+  {  48, ashr_48  },
+  {  49, ashr_49  },
+  {  63, ashr_63  },
+  {  64, ashr_64  },
+  {  65, ashr_65  },
+  {  72, ashr_72  },
+  {  79, ashr_79  },
+  {  80, ashr_80  },
+  {  81, ashr_81  },
+  {  95, ashr_95  },
+  {  96, ashr_96  },
+  {  97, ashr_97  },
+  { 111, ashr_111 },
+  { 112, ashr_112 },
+  { 113, ashr_113 },
+  { 119, ashr_119 },
+  { 120, ashr_120 },
+  { 121, ashr_121 },
+  { 126, ashr_126 },
+  { 127, ashr_127 }
+};
+
+void test(ti x)
+{
+  unsigned int i;
+  v1ti t = (v1ti)x;
+
+  for (i=0; i<(sizeof(table)/sizeof(table[0])); i++) {
+    if ((ti)(*table[i].ashr)(t) != ashr(x,table[i].i))
+      __builtin_abort();
+  }
+}
+
+int main()
+{
+  ti x;
+
+  x = ((ti)0x0011223344556677ull)<<64 | 0x8899aabbccddeeffull;
+  test(x);
+  x = ((ti)0xffeeddccbbaa9988ull)<<64 | 0x7766554433221100ull;
+  test(x);
+  x = ((ti)0x0123456789abcdefull)<<64 | 0x0123456789abcdefull;
+  test(x);
+  x = ((ti)0xfedcba9876543210ull)<<64 | 0xfedcba9876543210ull;
+  test(x);
+  x = ((ti)0x0123456789abcdefull)<<64 | 0xfedcba9876543210ull;
+  test(x);
+  x = ((ti)0xfedcba9876543210ull)<<64 | 0x0123456789abcdefull;
+  test(x);
+  x = 0;
+  test(x);
+  x = 0xffffffffffffffffull;
+  test(x);
+  x = ((ti)0xffffffffffffffffull)<<64;
+  test(x);
+  x = ((ti)0xffffffffffffffffull)<<64 | 0xffffffffffffffffull;
+  test(x);
+  x = ((ti)0x5a5a5a5a5a5a5a5aull)<<64 | 0x5a5a5a5a5a5a5a5aull;
+  test(x);
+  x = ((ti)0xa5a5a5a5a5a5a5a5ull)<<64 | 0xa5a5a5a5a5a5a5a5ull;
+  test(x);
+  x = 0xffull;
+  test(x);
+  x = 0xff00ull;
+  test(x);
+  x = 0xff0000ull;
+  test(x);
+  x = 0xff000000ull;
+  test(x);
+  x = 0xff00000000ull;
+  test(x);
+  x = 0xff0000000000ull;
+  test(x);
+  x = 0xff000000000000ull;
+  test(x);
+  x = 0xff00000000000000ull;
+  test(x);
+  x = ((ti)0xffull)<<64;
+  test(x);
+  x = ((ti)0xff00ull)<<64;
+  test(x);
+  x = ((ti)0xff0000ull)<<64;
+  test(x);
+  x = ((ti)0xff000000ull)<<64;
+  test(x);
+  x = ((ti)0xff00000000ull)<<64;
+  test(x);
+  x = ((ti)0xff0000000000ull)<<64;
+  test(x);
+  x = ((ti)0xff000000000000ull)<<64;
+  test(x);
+  x = ((ti)0xff00000000000000ull)<<64;
+  test(x);
+  x = 0xdeadbeefcafebabeull;
+  test(x);
+  x = ((ti)0xdeadbeefcafebabeull)<<64;
+  test(x);
+
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.target/i386/sse2-v1ti-ashiftrt-2.c b/gcc/testsuite/gcc.target/i386/sse2-v1ti-ashiftrt-2.c
new file mode 100644
index 0000000..b3d0aa3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/sse2-v1ti-ashiftrt-2.c
@@ -0,0 +1,166 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O2 -msse2 -mavx2 " } */
+
+typedef __int128 v1ti __attribute__ ((__vector_size__ (16)));
+typedef __int128 ti;
+
+ti ashr(ti x, unsigned int i) { return x >> i; }
+
+v1ti ashr_1(v1ti x) { return x >> 1; }
+v1ti ashr_2(v1ti x) { return x >> 2; }
+v1ti ashr_7(v1ti x) { return x >> 7; }
+v1ti ashr_8(v1ti x) { return x >> 8; }
+v1ti ashr_9(v1ti x) { return x >> 9; }
+v1ti ashr_15(v1ti x) { return x >> 15; }
+v1ti ashr_16(v1ti x) { return x >> 16; }
+v1ti ashr_17(v1ti x) { return x >> 17; }
+v1ti ashr_23(v1ti x) { return x >> 23; }
+v1ti ashr_24(v1ti x) { return x >> 24; }
+v1ti ashr_25(v1ti x) { return x >> 25; }
+v1ti ashr_31(v1ti x) { return x >> 31; }
+v1ti ashr_32(v1ti x) { return x >> 32; }
+v1ti ashr_33(v1ti x) { return x >> 33; }
+v1ti ashr_47(v1ti x) { return x >> 47; }
+v1ti ashr_48(v1ti x) { return x >> 48; }
+v1ti ashr_49(v1ti x) { return x >> 49; }
+v1ti ashr_63(v1ti x) { return x >> 63; }
+v1ti ashr_64(v1ti x) { return x >> 64; }
+v1ti ashr_65(v1ti x) { return x >> 65; }
+v1ti ashr_72(v1ti x) { return x >> 72; }
+v1ti ashr_79(v1ti x) { return x >> 79; }
+v1ti ashr_80(v1ti x) { return x >> 80; }
+v1ti ashr_81(v1ti x) { return x >> 81; }
+v1ti ashr_95(v1ti x) { return x >> 95; }
+v1ti ashr_96(v1ti x) { return x >> 96; }
+v1ti ashr_97(v1ti x) { return x >> 97; }
+v1ti ashr_111(v1ti x) { return x >> 111; }
+v1ti ashr_112(v1ti x) { return x >> 112; }
+v1ti ashr_113(v1ti x) { return x >> 113; }
+v1ti ashr_119(v1ti x) { return x >> 119; }
+v1ti ashr_120(v1ti x) { return x >> 120; }
+v1ti ashr_121(v1ti x) { return x >> 121; }
+v1ti ashr_126(v1ti x) { return x >> 126; }
+v1ti ashr_127(v1ti x) { return x >> 127; }
+
+typedef v1ti (*fun)(v1ti);
+
+struct {
+  unsigned int i;
+  fun ashr;
+} table[35] = {
+  {   1, ashr_1   },
+  {   2, ashr_2   },
+  {   7, ashr_7   },
+  {   8, ashr_8   },
+  {   9, ashr_9   },
+  {  15, ashr_15  },
+  {  16, ashr_16  },
+  {  17, ashr_17  },
+  {  23, ashr_23  },
+  {  24, ashr_24  },
+  {  25, ashr_25  },
+  {  31, ashr_31  },
+  {  32, ashr_32  },
+  {  33, ashr_33  },
+  {  47, ashr_47  },
+  {  48, ashr_48  },
+  {  49, ashr_49  },
+  {  63, ashr_63  },
+  {  64, ashr_64  },
+  {  65, ashr_65  },
+  {  72, ashr_72  },
+  {  79, ashr_79  },
+  {  80, ashr_80  },
+  {  81, ashr_81  },
+  {  95, ashr_95  },
+  {  96, ashr_96  },
+  {  97, ashr_97  },
+  { 111, ashr_111 },
+  { 112, ashr_112 },
+  { 113, ashr_113 },
+  { 119, ashr_119 },
+  { 120, ashr_120 },
+  { 121, ashr_121 },
+  { 126, ashr_126 },
+  { 127, ashr_127 }
+};
+
+void test(ti x)
+{
+  unsigned int i;
+  v1ti t = (v1ti)x;
+
+  for (i=0; i<(sizeof(table)/sizeof(table[0])); i++) {
+    if ((ti)(*table[i].ashr)(t) != ashr(x,table[i].i))
+      __builtin_abort();
+  }
+}
+
+int main()
+{
+  ti x;
+
+  x = ((ti)0x0011223344556677ull)<<64 | 0x8899aabbccddeeffull;
+  test(x);
+  x = ((ti)0xffeeddccbbaa9988ull)<<64 | 0x7766554433221100ull;
+  test(x);
+  x = ((ti)0x0123456789abcdefull)<<64 | 0x0123456789abcdefull;
+  test(x);
+  x = ((ti)0xfedcba9876543210ull)<<64 | 0xfedcba9876543210ull;
+  test(x);
+  x = ((ti)0x0123456789abcdefull)<<64 | 0xfedcba9876543210ull;
+  test(x);
+  x = ((ti)0xfedcba9876543210ull)<<64 | 0x0123456789abcdefull;
+  test(x);
+  x = 0;
+  test(x);
+  x = 0xffffffffffffffffull;
+  test(x);
+  x = ((ti)0xffffffffffffffffull)<<64;
+  test(x);
+  x = ((ti)0xffffffffffffffffull)<<64 | 0xffffffffffffffffull;
+  test(x);
+  x = ((ti)0x5a5a5a5a5a5a5a5aull)<<64 | 0x5a5a5a5a5a5a5a5aull;
+  test(x);
+  x = ((ti)0xa5a5a5a5a5a5a5a5ull)<<64 | 0xa5a5a5a5a5a5a5a5ull;
+  test(x);
+  x = 0xffull;
+  test(x);
+  x = 0xff00ull;
+  test(x);
+  x = 0xff0000ull;
+  test(x);
+  x = 0xff000000ull;
+  test(x);
+  x = 0xff00000000ull;
+  test(x);
+  x = 0xff0000000000ull;
+  test(x);
+  x = 0xff000000000000ull;
+  test(x);
+  x = 0xff00000000000000ull;
+  test(x);
+  x = ((ti)0xffull)<<64;
+  test(x);
+  x = ((ti)0xff00ull)<<64;
+  test(x);
+  x = ((ti)0xff0000ull)<<64;
+  test(x);
+  x = ((ti)0xff000000ull)<<64;
+  test(x);
+  x = ((ti)0xff00000000ull)<<64;
+  test(x);
+  x = ((ti)0xff0000000000ull)<<64;
+  test(x);
+  x = ((ti)0xff000000000000ull)<<64;
+  test(x);
+  x = ((ti)0xff00000000000000ull)<<64;
+  test(x);
+  x = 0xdeadbeefcafebabeull;
+  test(x);
+  x = ((ti)0xdeadbeefcafebabeull)<<64;
+  test(x);
+
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.target/i386/sse2-v1ti-ashiftrt-3.c b/gcc/testsuite/gcc.target/i386/sse2-v1ti-ashiftrt-3.c
new file mode 100644
index 0000000..61d4f4c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/sse2-v1ti-ashiftrt-3.c
@@ -0,0 +1,166 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O2 -msse2 -msse4.1" } */
+
+typedef __int128 v1ti __attribute__ ((__vector_size__ (16)));
+typedef __int128 ti;
+
+ti ashr(ti x, unsigned int i) { return x >> i; }
+
+v1ti ashr_1(v1ti x) { return x >> 1; }
+v1ti ashr_2(v1ti x) { return x >> 2; }
+v1ti ashr_7(v1ti x) { return x >> 7; }
+v1ti ashr_8(v1ti x) { return x >> 8; }
+v1ti ashr_9(v1ti x) { return x >> 9; }
+v1ti ashr_15(v1ti x) { return x >> 15; }
+v1ti ashr_16(v1ti x) { return x >> 16; }
+v1ti ashr_17(v1ti x) { return x >> 17; }
+v1ti ashr_23(v1ti x) { return x >> 23; }
+v1ti ashr_24(v1ti x) { return x >> 24; }
+v1ti ashr_25(v1ti x) { return x >> 25; }
+v1ti ashr_31(v1ti x) { return x >> 31; }
+v1ti ashr_32(v1ti x) { return x >> 32; }
+v1ti ashr_33(v1ti x) { return x >> 33; }
+v1ti ashr_47(v1ti x) { return x >> 47; }
+v1ti ashr_48(v1ti x) { return x >> 48; }
+v1ti ashr_49(v1ti x) { return x >> 49; }
+v1ti ashr_63(v1ti x) { return x >> 63; }
+v1ti ashr_64(v1ti x) { return x >> 64; }
+v1ti ashr_65(v1ti x) { return x >> 65; }
+v1ti ashr_72(v1ti x) { return x >> 72; }
+v1ti ashr_79(v1ti x) { return x >> 79; }
+v1ti ashr_80(v1ti x) { return x >> 80; }
+v1ti ashr_81(v1ti x) { return x >> 81; }
+v1ti ashr_95(v1ti x) { return x >> 95; }
+v1ti ashr_96(v1ti x) { return x >> 96; }
+v1ti ashr_97(v1ti x) { return x >> 97; }
+v1ti ashr_111(v1ti x) { return x >> 111; }
+v1ti ashr_112(v1ti x) { return x >> 112; }
+v1ti ashr_113(v1ti x) { return x >> 113; }
+v1ti ashr_119(v1ti x) { return x >> 119; }
+v1ti ashr_120(v1ti x) { return x >> 120; }
+v1ti ashr_121(v1ti x) { return x >> 121; }
+v1ti ashr_126(v1ti x) { return x >> 126; }
+v1ti ashr_127(v1ti x) { return x >> 127; }
+
+typedef v1ti (*fun)(v1ti);
+
+struct {
+  unsigned int i;
+  fun ashr;
+} table[35] = {
+  {   1, ashr_1   },
+  {   2, ashr_2   },
+  {   7, ashr_7   },
+  {   8, ashr_8   },
+  {   9, ashr_9   },
+  {  15, ashr_15  },
+  {  16, ashr_16  },
+  {  17, ashr_17  },
+  {  23, ashr_23  },
+  {  24, ashr_24  },
+  {  25, ashr_25  },
+  {  31, ashr_31  },
+  {  32, ashr_32  },
+  {  33, ashr_33  },
+  {  47, ashr_47  },
+  {  48, ashr_48  },
+  {  49, ashr_49  },
+  {  63, ashr_63  },
+  {  64, ashr_64  },
+  {  65, ashr_65  },
+  {  72, ashr_72  },
+  {  79, ashr_79  },
+  {  80, ashr_80  },
+  {  81, ashr_81  },
+  {  95, ashr_95  },
+  {  96, ashr_96  },
+  {  97, ashr_97  },
+  { 111, ashr_111 },
+  { 112, ashr_112 },
+  { 113, ashr_113 },
+  { 119, ashr_119 },
+  { 120, ashr_120 },
+  { 121, ashr_121 },
+  { 126, ashr_126 },
+  { 127, ashr_127 }
+};
+
+void test(ti x)
+{
+  unsigned int i;
+  v1ti t = (v1ti)x;
+
+  for (i=0; i<(sizeof(table)/sizeof(table[0])); i++) {
+    if ((ti)(*table[i].ashr)(t) != ashr(x,table[i].i))
+      __builtin_abort();
+  }
+}
+
+int main()
+{
+  ti x;
+
+  x = ((ti)0x0011223344556677ull)<<64 | 0x8899aabbccddeeffull;
+  test(x);
+  x = ((ti)0xffeeddccbbaa9988ull)<<64 | 0x7766554433221100ull;
+  test(x);
+  x = ((ti)0x0123456789abcdefull)<<64 | 0x0123456789abcdefull;
+  test(x);
+  x = ((ti)0xfedcba9876543210ull)<<64 | 0xfedcba9876543210ull;
+  test(x);
+  x = ((ti)0x0123456789abcdefull)<<64 | 0xfedcba9876543210ull;
+  test(x);
+  x = ((ti)0xfedcba9876543210ull)<<64 | 0x0123456789abcdefull;
+  test(x);
+  x = 0;
+  test(x);
+  x = 0xffffffffffffffffull;
+  test(x);
+  x = ((ti)0xffffffffffffffffull)<<64;
+  test(x);
+  x = ((ti)0xffffffffffffffffull)<<64 | 0xffffffffffffffffull;
+  test(x);
+  x = ((ti)0x5a5a5a5a5a5a5a5aull)<<64 | 0x5a5a5a5a5a5a5a5aull;
+  test(x);
+  x = ((ti)0xa5a5a5a5a5a5a5a5ull)<<64 | 0xa5a5a5a5a5a5a5a5ull;
+  test(x);
+  x = 0xffull;
+  test(x);
+  x = 0xff00ull;
+  test(x);
+  x = 0xff0000ull;
+  test(x);
+  x = 0xff000000ull;
+  test(x);
+  x = 0xff00000000ull;
+  test(x);
+  x = 0xff0000000000ull;
+  test(x);
+  x = 0xff000000000000ull;
+  test(x);
+  x = 0xff00000000000000ull;
+  test(x);
+  x = ((ti)0xffull)<<64;
+  test(x);
+  x = ((ti)0xff00ull)<<64;
+  test(x);
+  x = ((ti)0xff0000ull)<<64;
+  test(x);
+  x = ((ti)0xff000000ull)<<64;
+  test(x);
+  x = ((ti)0xff00000000ull)<<64;
+  test(x);
+  x = ((ti)0xff0000000000ull)<<64;
+  test(x);
+  x = ((ti)0xff000000000000ull)<<64;
+  test(x);
+  x = ((ti)0xff00000000000000ull)<<64;
+  test(x);
+  x = 0xdeadbeefcafebabeull;
+  test(x);
+  x = ((ti)0xdeadbeefcafebabeull)<<64;
+  test(x);
+
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.target/i386/sse2-v1ti-logic-2.c b/gcc/testsuite/gcc.target/i386/sse2-v1ti-logic-2.c
new file mode 100644
index 0000000..3ec6455
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/sse2-v1ti-logic-2.c
@@ -0,0 +1,53 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O2 -msse2" } */
+
+typedef unsigned __int128 v1ti __attribute__ ((__vector_size__ (16)));
+
+v1ti x;
+v1ti y;
+v1ti z;
+
+void and2()
+{
+  x &= y;
+}
+
+void and3()
+{
+  x = y & z;
+}
+
+void ior2()
+{
+  x |= y;
+}
+
+void ior3()
+{
+  x = y | z;
+}
+
+
+void xor2()
+{
+  x ^= y;
+}
+
+void xor3()
+{
+  x =  y ^ z;
+}
+
+void not1()
+{
+  x = ~x;
+}
+
+void not2()
+{
+  x = ~y;
+}
+
+/* { dg-final { scan-assembler-times "pand" 2 } } */
+/* { dg-final { scan-assembler-times "por" 2 } } */
+/* { dg-final { scan-assembler-times "pxor" 4 } } */
diff --git a/gcc/testsuite/gcc.target/i386/sse2-v1ti-logic.c b/gcc/testsuite/gcc.target/i386/sse2-v1ti-logic.c
new file mode 100644
index 0000000..130a89b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/sse2-v1ti-logic.c
@@ -0,0 +1,28 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O2 -msse2" } */
+
+typedef unsigned __int128 v1ti __attribute__ ((__vector_size__ (16)));
+
+v1ti and(v1ti x, v1ti y)
+{
+  return x & y;
+}
+
+v1ti ior(v1ti x, v1ti y)
+{
+  return x | y;
+}
+
+v1ti xor(v1ti x, v1ti y)
+{
+  return x ^ y;
+}
+
+v1ti not(v1ti x)
+{
+  return ~x;
+}
+
+/* { dg-final { scan-assembler "pand" } } */
+/* { dg-final { scan-assembler "por" } } */
+/* { dg-final { scan-assembler-times "pxor" 2 } } */
diff --git a/gcc/testsuite/gcc.target/i386/sse2-v1ti-shift-2.c b/gcc/testsuite/gcc.target/i386/sse2-v1ti-shift-2.c
new file mode 100644
index 0000000..18da2ef
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/sse2-v1ti-shift-2.c
@@ -0,0 +1,13 @@
+/* PR target/102986 */
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O2 -msse2" } */
+
+typedef unsigned __int128 uv1ti __attribute__ ((__vector_size__ (16)));
+typedef __int128 sv1ti __attribute__ ((__vector_size__ (16)));
+
+uv1ti ashl(uv1ti x, unsigned int i) { return x << i; }
+uv1ti lshr(uv1ti x, unsigned int i) { return x >> i; }
+sv1ti ashr(sv1ti x, unsigned int i) { return x >> i; }
+uv1ti rotr(uv1ti x, unsigned int i) { return (x >> i) | (x << (128-i)); }
+uv1ti rotl(uv1ti x, unsigned int i) { return (x << i) | (x >> (128-i)); }
+
diff --git a/gcc/testsuite/gcc.target/i386/sse2-v1ti-shift-3.c b/gcc/testsuite/gcc.target/i386/sse2-v1ti-shift-3.c
new file mode 100644
index 0000000..8d5c122
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/sse2-v1ti-shift-3.c
@@ -0,0 +1,113 @@
+/* PR target/102986 */
+/* { dg-do run { target int128 } } */
+/* { dg-options "-O2 -msse2" } */
+/* { dg-require-effective-target sse2 } */
+
+typedef unsigned __int128 uv1ti __attribute__ ((__vector_size__ (16)));
+typedef __int128 sv1ti __attribute__ ((__vector_size__ (16)));
+typedef __int128 v1ti __attribute__ ((__vector_size__ (16)));
+
+typedef unsigned __int128 uti;
+typedef __int128 sti;
+typedef __int128 ti;
+
+uv1ti ashl_v1ti(uv1ti x, unsigned int i) { return x << i; }
+uv1ti lshr_v1ti(uv1ti x, unsigned int i) { return x >> i; }
+sv1ti ashr_v1ti(sv1ti x, unsigned int i) { return x >> i; }
+uv1ti rotr_v1ti(uv1ti x, unsigned int i) { return (x >> i) | (x << (128-i)); }
+uv1ti rotl_v1ti(uv1ti x, unsigned int i) { return (x << i) | (x >> (128-i)); }
+
+uti ashl_ti(uti x, unsigned int i) { return x << i; }
+uti lshr_ti(uti x, unsigned int i) { return x >> i; }
+sti ashr_ti(sti x, unsigned int i) { return x >> i; }
+uti rotr_ti(uti x, unsigned int i) { return (x >> i) | (x << (128-i)); }
+uti rotl_ti(uti x, unsigned int i) { return (x << i) | (x >> (128-i)); }
+
+void test(ti x)
+{
+  unsigned int i;
+  uv1ti ut = (uv1ti)x;
+  sv1ti st = (sv1ti)x;
+
+  for (i=0; i<128; i++) {
+    if ((ti)ashl_v1ti(ut,i) != (ti)ashl_ti(x,i))
+      __builtin_abort();
+    if ((ti)lshr_v1ti(ut,i) != (ti)lshr_ti(x,i))
+      __builtin_abort();
+    if ((ti)ashr_v1ti(st,i) != (ti)ashr_ti(x,i))
+      __builtin_abort();
+    if ((ti)rotr_v1ti(ut,i) != (ti)rotr_ti(x,i))
+      __builtin_abort();
+    if ((ti)rotl_v1ti(ut,i) != (ti)rotl_ti(x,i))
+      __builtin_abort();
+  }
+}
+
+int main()
+{
+  ti x;
+
+  x = ((ti)0x0011223344556677ull)<<64 | 0x8899aabbccddeeffull;
+  test(x);
+  x = ((ti)0xffeeddccbbaa9988ull)<<64 | 0x7766554433221100ull;
+  test(x);
+  x = ((ti)0x0123456789abcdefull)<<64 | 0x0123456789abcdefull;
+  test(x);
+  x = ((ti)0xfedcba9876543210ull)<<64 | 0xfedcba9876543210ull;
+  test(x);
+  x = ((ti)0x0123456789abcdefull)<<64 | 0xfedcba9876543210ull;
+  test(x);
+  x = ((ti)0xfedcba9876543210ull)<<64 | 0x0123456789abcdefull;
+  test(x);
+  x = 0;
+  test(x);
+  x = 0xffffffffffffffffull;
+  test(x);
+  x = ((ti)0xffffffffffffffffull)<<64;
+  test(x);
+  x = ((ti)0xffffffffffffffffull)<<64 | 0xffffffffffffffffull;
+  test(x);
+  x = ((ti)0x5a5a5a5a5a5a5a5aull)<<64 | 0x5a5a5a5a5a5a5a5aull;
+  test(x);
+  x = ((ti)0xa5a5a5a5a5a5a5a5ull)<<64 | 0xa5a5a5a5a5a5a5a5ull;
+  test(x);
+  x = 0xffull;
+  test(x);
+  x = 0xff00ull;
+  test(x);
+  x = 0xff0000ull;
+  test(x);
+  x = 0xff000000ull;
+  test(x);
+  x = 0xff00000000ull;
+  test(x);
+  x = 0xff0000000000ull;
+  test(x);
+  x = 0xff000000000000ull;
+  test(x);
+  x = 0xff00000000000000ull;
+  test(x);
+  x = ((ti)0xffull)<<64;
+  test(x);
+  x = ((ti)0xff00ull)<<64;
+  test(x);
+  x = ((ti)0xff0000ull)<<64;
+  test(x);
+  x = ((ti)0xff000000ull)<<64;
+  test(x);
+  x = ((ti)0xff00000000ull)<<64;
+  test(x);
+  x = ((ti)0xff0000000000ull)<<64;
+  test(x);
+  x = ((ti)0xff000000000000ull)<<64;
+  test(x);
+  x = ((ti)0xff00000000000000ull)<<64;
+  test(x);
+  x = 0xdeadbeefcafebabeull;
+  test(x);
+  x = ((ti)0xdeadbeefcafebabeull)<<64;
+  test(x);
+
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.target/i386/sse2-v1ti-shift.c b/gcc/testsuite/gcc.target/i386/sse2-v1ti-shift.c
new file mode 100644
index 0000000..dbae418
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/sse2-v1ti-shift.c
@@ -0,0 +1,212 @@
+/* { dg-do run { target int128 } } */
+/* { dg-options "-O2 -msse2" } */
+/* { dg-require-effective-target sse2 } */
+
+typedef unsigned __int128 v1ti __attribute__ ((__vector_size__ (16)));
+typedef unsigned __int128 ti;
+
+ti ashl(ti x, unsigned int i) { return x << i; }
+ti lshr(ti x, unsigned int i) { return x >> i; }
+ti rotr(ti x, unsigned int i) { return (x >> i) | (x << (128-i)); }
+
+v1ti ashl_1(v1ti x) { return x << 1; }
+v1ti ashl_2(v1ti x) { return x << 2; }
+v1ti ashl_7(v1ti x) { return x << 7; }
+v1ti ashl_8(v1ti x) { return x << 8; }
+v1ti ashl_9(v1ti x) { return x << 9; }
+v1ti ashl_15(v1ti x) { return x << 15; }
+v1ti ashl_16(v1ti x) { return x << 16; }
+v1ti ashl_17(v1ti x) { return x << 17; }
+v1ti ashl_31(v1ti x) { return x << 31; }
+v1ti ashl_32(v1ti x) { return x << 32; }
+v1ti ashl_33(v1ti x) { return x << 33; }
+v1ti ashl_63(v1ti x) { return x << 63; }
+v1ti ashl_64(v1ti x) { return x << 64; }
+v1ti ashl_65(v1ti x) { return x << 65; }
+v1ti ashl_72(v1ti x) { return x << 72; }
+v1ti ashl_95(v1ti x) { return x << 95; }
+v1ti ashl_96(v1ti x) { return x << 96; }
+v1ti ashl_97(v1ti x) { return x << 97; }
+v1ti ashl_111(v1ti x) { return x << 111; }
+v1ti ashl_112(v1ti x) { return x << 112; }
+v1ti ashl_113(v1ti x) { return x << 113; }
+v1ti ashl_119(v1ti x) { return x << 119; }
+v1ti ashl_120(v1ti x) { return x << 120; }
+v1ti ashl_121(v1ti x) { return x << 121; }
+v1ti ashl_126(v1ti x) { return x << 126; }
+v1ti ashl_127(v1ti x) { return x << 127; }
+
+v1ti lshr_1(v1ti x) { return x >> 1; }
+v1ti lshr_2(v1ti x) { return x >> 2; }
+v1ti lshr_7(v1ti x) { return x >> 7; }
+v1ti lshr_8(v1ti x) { return x >> 8; }
+v1ti lshr_9(v1ti x) { return x >> 9; }
+v1ti lshr_15(v1ti x) { return x >> 15; }
+v1ti lshr_16(v1ti x) { return x >> 16; }
+v1ti lshr_17(v1ti x) { return x >> 17; }
+v1ti lshr_31(v1ti x) { return x >> 31; }
+v1ti lshr_32(v1ti x) { return x >> 32; }
+v1ti lshr_33(v1ti x) { return x >> 33; }
+v1ti lshr_63(v1ti x) { return x >> 63; }
+v1ti lshr_64(v1ti x) { return x >> 64; }
+v1ti lshr_65(v1ti x) { return x >> 65; }
+v1ti lshr_72(v1ti x) { return x >> 72; }
+v1ti lshr_95(v1ti x) { return x >> 95; }
+v1ti lshr_96(v1ti x) { return x >> 96; }
+v1ti lshr_97(v1ti x) { return x >> 97; }
+v1ti lshr_111(v1ti x) { return x >> 111; }
+v1ti lshr_112(v1ti x) { return x >> 112; }
+v1ti lshr_113(v1ti x) { return x >> 113; }
+v1ti lshr_119(v1ti x) { return x >> 119; }
+v1ti lshr_120(v1ti x) { return x >> 120; }
+v1ti lshr_121(v1ti x) { return x >> 121; }
+v1ti lshr_126(v1ti x) { return x >> 126; }
+v1ti lshr_127(v1ti x) { return x >> 127; }
+
+v1ti rotr_1(v1ti x) { return (x >> 1) | (x << 127); }
+v1ti rotr_2(v1ti x) { return (x >> 2) | (x << 126); }
+v1ti rotr_7(v1ti x) { return (x >> 7) | (x << 121); }
+v1ti rotr_8(v1ti x) { return (x >> 8) | (x << 120); }
+v1ti rotr_9(v1ti x) { return (x >> 9) | (x << 119); }
+v1ti rotr_15(v1ti x) { return (x >> 15) | (x << 113); }
+v1ti rotr_16(v1ti x) { return (x >> 16) | (x << 112); }
+v1ti rotr_17(v1ti x) { return (x >> 17) | (x << 111); }
+v1ti rotr_31(v1ti x) { return (x >> 31) | (x << 97); }
+v1ti rotr_32(v1ti x) { return (x >> 32) | (x << 96); }
+v1ti rotr_33(v1ti x) { return (x >> 33) | (x << 95); }
+v1ti rotr_63(v1ti x) { return (x >> 63) | (x << 65); }
+v1ti rotr_64(v1ti x) { return (x >> 64) | (x << 64); }
+v1ti rotr_65(v1ti x) { return (x >> 65) | (x << 63); }
+v1ti rotr_72(v1ti x) { return (x >> 72) | (x << 56); }
+v1ti rotr_95(v1ti x) { return (x >> 95) | (x << 33); }
+v1ti rotr_96(v1ti x) { return (x >> 96) | (x << 32); }
+v1ti rotr_97(v1ti x) { return (x >> 97) | (x << 31); }
+v1ti rotr_111(v1ti x) { return (x >> 111) | (x << 17); }
+v1ti rotr_112(v1ti x) { return (x >> 112) | (x << 16); }
+v1ti rotr_113(v1ti x) { return (x >> 113) | (x << 15); }
+v1ti rotr_119(v1ti x) { return (x >> 119) | (x << 9); }
+v1ti rotr_120(v1ti x) { return (x >> 120) | (x << 8); }
+v1ti rotr_121(v1ti x) { return (x >> 121) | (x << 7); }
+v1ti rotr_126(v1ti x) { return (x >> 126) | (x << 2); }
+v1ti rotr_127(v1ti x) { return (x >> 127) | (x << 1); }
+
+
+typedef v1ti (*fun)(v1ti);
+
+struct {
+  unsigned int i;
+  fun ashl;
+  fun lshr;
+  fun rotr;
+} table[26] = {
+  {   1, ashl_1,   lshr_1,   rotr_1   },
+  {   2, ashl_2,   lshr_2,   rotr_2   },
+  {   7, ashl_7,   lshr_7,   rotr_7   },
+  {   8, ashl_8,   lshr_8,   rotr_8   },
+  {   9, ashl_9,   lshr_9,   rotr_9   },
+  {  15, ashl_15,  lshr_15,  rotr_15  },
+  {  16, ashl_16,  lshr_16,  rotr_16  },
+  {  17, ashl_17,  lshr_17,  rotr_17  },
+  {  31, ashl_31,  lshr_31,  rotr_31  },
+  {  32, ashl_32,  lshr_32,  rotr_32  },
+  {  33, ashl_33,  lshr_33,  rotr_33  },
+  {  63, ashl_63,  lshr_63,  rotr_63  },
+  {  64, ashl_64,  lshr_64,  rotr_64  },
+  {  65, ashl_65,  lshr_65,  rotr_65  },
+  {  72, ashl_72,  lshr_72,  rotr_72  },
+  {  95, ashl_95,  lshr_95,  rotr_95  },
+  {  96, ashl_96,  lshr_96,  rotr_96  },
+  {  97, ashl_97,  lshr_97,  rotr_97  },
+  { 111, ashl_111, lshr_111, rotr_111 },
+  { 112, ashl_112, lshr_112, rotr_112 },
+  { 113, ashl_113, lshr_113, rotr_113 },
+  { 119, ashl_119, lshr_119, rotr_119 },
+  { 120, ashl_120, lshr_120, rotr_120 },
+  { 121, ashl_121, lshr_121, rotr_121 },
+  { 126, ashl_126, lshr_126, rotr_126 },
+  { 127, ashl_127, lshr_127, rotr_127 }
+};
+
+void test(ti x)
+{
+  unsigned int i;
+  v1ti t = (v1ti)x;
+
+  for (i=0; i<(sizeof(table)/sizeof(table[0])); i++) {
+    if ((ti)(*table[i].ashl)(t) != ashl(x,table[i].i))
+      __builtin_abort();
+    if ((ti)(*table[i].lshr)(t) != lshr(x,table[i].i))
+      __builtin_abort();
+    if ((ti)(*table[i].rotr)(t) != rotr(x,table[i].i))
+      __builtin_abort();
+  }
+}
+
+int main()
+{
+  ti x;
+
+  x = ((ti)0x0011223344556677ull)<<64 | 0x8899aabbccddeeffull;
+  test(x);
+  x = ((ti)0xffeeddccbbaa9988ull)<<64 | 0x7766554433221100ull;
+  test(x);
+  x = ((ti)0x0123456789abcdefull)<<64 | 0x0123456789abcdefull;
+  test(x);
+  x = ((ti)0xfedcba9876543210ull)<<64 | 0xfedcba9876543210ull;
+  test(x);
+  x = ((ti)0x0123456789abcdefull)<<64 | 0xfedcba9876543210ull;
+  test(x);
+  x = ((ti)0xfedcba9876543210ull)<<64 | 0x0123456789abcdefull;
+  test(x);
+  x = 0;
+  test(x);
+  x = 0xffffffffffffffffull;
+  test(x);
+  x = ((ti)0xffffffffffffffffull)<<64;
+  test(x);
+  x = ((ti)0xffffffffffffffffull)<<64 | 0xffffffffffffffffull;
+  test(x);
+  x = ((ti)0x5a5a5a5a5a5a5a5aull)<<64 | 0x5a5a5a5a5a5a5a5aull;
+  test(x);
+  x = ((ti)0xa5a5a5a5a5a5a5a5ull)<<64 | 0xa5a5a5a5a5a5a5a5ull;
+  test(x);
+  x = 0xffull;
+  test(x);
+  x = 0xff00ull;
+  test(x);
+  x = 0xff0000ull;
+  test(x);
+  x = 0xff000000ull;
+  test(x);
+  x = 0xff00000000ull;
+  test(x);
+  x = 0xff0000000000ull;
+  test(x);
+  x = 0xff000000000000ull;
+  test(x);
+  x = 0xff00000000000000ull;
+  test(x);
+  x = ((ti)0xffull)<<64;
+  test(x);
+  x = ((ti)0xff00ull)<<64;
+  test(x);
+  x = ((ti)0xff0000ull)<<64;
+  test(x);
+  x = ((ti)0xff000000ull)<<64;
+  test(x);
+  x = ((ti)0xff00000000ull)<<64;
+  test(x);
+  x = ((ti)0xff0000000000ull)<<64;
+  test(x);
+  x = ((ti)0xff000000000000ull)<<64;
+  test(x);
+  x = ((ti)0xff00000000000000ull)<<64;
+  test(x);
+  x = 0xdeadbeefcafebabeull;
+  test(x);
+  x = ((ti)0xdeadbeefcafebabeull)<<64;
+  test(x);
+
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.target/i386/vnni-auto-vectorize-1.c b/gcc/testsuite/gcc.target/i386/vnni-auto-vectorize-1.c
new file mode 100644
index 0000000..844f37d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vnni-auto-vectorize-1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */                                     
+/* { dg-options "-mavx512f -mavx512vnni -mavx512vl -O2" } */
+
+/* { dg-final { scan-assembler "vpdpwssd\t" } } */
+/* { dg-final { scan-assembler "vpdpbusd\t" } } */
+/* { dg-final { scan-assembler-not "vpmaddwd\t" } } */
+
+int __attribute__((noinline, noclone, optimize("tree-vectorize")))
+sdot_prod_hi (short * restrict a, short * restrict b,
+	      int c, int n)
+{
+  int i;
+  for (i = 0; i < n; i++)
+    {
+      c += ((int) a[i] * (int) b[i]);
+    }
+  return c;
+}
+
+int __attribute__((noinline, noclone, optimize("tree-vectorize")))
+usdot_prod_qi (unsigned char * restrict a, char *restrict b,
+	       int c, int n)
+{
+  int i;
+  for (i = 0; i < n; i++)
+    {
+      c += ((int) a[i] * (int) b[i]);
+    }
+  return c;
+}
diff --git a/gcc/testsuite/gcc.target/i386/vnni-auto-vectorize-2.c b/gcc/testsuite/gcc.target/i386/vnni-auto-vectorize-2.c
new file mode 100644
index 0000000..dc8047d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vnni-auto-vectorize-2.c
@@ -0,0 +1,72 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mavx512vnni -mavx512vl" } */
+/* { dg-require-effective-target avx512vnni } */
+/* { dg-require-effective-target avx512vl } */
+
+static void vnni_test (void);
+#define DO_TEST vnni_test
+#define AVX512VNNI
+#define AVX512VL
+#include "avx512f-check.h"
+#include "vnni-auto-vectorize-1.c"
+
+#define N 256
+unsigned char a_u8[N];
+char b_i8[N];
+short a_i16[N], b_i16[N];
+int i8_exp, i8_ref, i16_exp, i16_ref;
+
+int __attribute__((noinline, noclone, optimize("no-tree-vectorize")))
+sdot_prod_hi_scalar (short * restrict a, short * restrict b,
+		     int c, int n)
+{
+  int i;
+  for (i = 0; i < n; i++)
+    {
+      c += ((int) a[i] * (int) b[i]);
+    }
+  return c;
+}
+
+int __attribute__((noinline, noclone, optimize("no-tree-vectorize")))
+usdot_prod_qi_scalar (unsigned char * restrict a, char *restrict b,
+	       int c, int n)
+{
+  int i;
+  for (i = 0; i < n; i++)
+    {
+      c += ((int) a[i] * (int) b[i]);
+    }
+  return c;
+}
+
+void init()
+{
+  int i;
+
+  i8_exp = i8_ref = 127;
+  i16_exp = i16_ref = 65535;
+
+  for (i = 0; i < N; i++)
+    {
+      a_u8[i] = (i + 3) % 256;
+      b_i8[i] = (i + 1) % 128; 
+      a_i16[i] = i * 2;
+      b_i16[i] = -i + 2;
+    }
+}
+
+static void vnni_test()
+{
+  init ();
+  i16_exp = sdot_prod_hi (a_i16, b_i16, i16_exp, N);
+  i16_ref = sdot_prod_hi_scalar (a_i16, b_i16, i16_ref, N);
+  if (i16_exp != i16_ref)
+    abort ();
+
+  init ();
+  i8_exp = usdot_prod_qi (a_u8, b_i8, i8_exp, N);
+  i8_ref = usdot_prod_qi_scalar (a_u8, b_i8, i8_ref, N);
+  if (i8_exp != i8_ref)
+    abort ();
+}
diff --git a/gcc/testsuite/gcc.target/nios2/custom-fp-inline-1.c b/gcc/testsuite/gcc.target/nios2/custom-fp-inline-1.c
new file mode 100644
index 0000000..43715e5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/custom-fp-inline-1.c
@@ -0,0 +1,33 @@
+/* Test that you can inline a function with custom insn attributes into
+   one with the same attributes.  */
+
+/* { dg-do compile } */
+/* { dg-options "-O1 -ffinite-math-only" } */
+
+/* -O1 in the options is significant.  Without it FP operations may not be
+   optimized to custom instructions.  */
+
+#include <math.h>
+
+static inline
+__attribute__ ((always_inline, target ("custom-fmaxs=246,custom-fmins=247")))
+void
+custom_fp1 (float operand_a, float operand_b, float *result)
+
+{
+  result[0] = fmaxf (operand_a, operand_b);
+  result[1] = fminf (operand_a, operand_b);
+}
+
+extern void
+custom_fp (float operand_a, float operand_b, float *result)
+  __attribute__ ((target ("custom-fmaxs=246,custom-fmins=247")));
+
+void
+custom_fp (float operand_a, float operand_b, float *result)
+{
+  custom_fp1 (operand_a, operand_b, result);
+}
+
+/* { dg-final { scan-assembler "custom\\t246, .* # fmaxs .*" } } */
+/* { dg-final { scan-assembler "custom\\t247, .* # fmins .*" } } */
diff --git a/gcc/testsuite/gcc.target/nios2/custom-fp-inline-2.c b/gcc/testsuite/gcc.target/nios2/custom-fp-inline-2.c
new file mode 100644
index 0000000..3e35241
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/custom-fp-inline-2.c
@@ -0,0 +1,29 @@
+/* Test that you cannot inline a function with custom insn attributes into
+   one without attributes.  */
+
+/* { dg-do compile } */
+/* { dg-options "-O1 -ffinite-math-only" } */
+
+/* -O1 in the options is significant.  Without it FP operations may not be
+   optimized to custom instructions.  */
+
+#include <math.h>
+
+static inline
+__attribute__ ((always_inline, target ("custom-fmaxs=246,custom-fmins=247")))
+void
+custom_fp1 (float operand_a, float operand_b, float *result)  /* { dg-error "target specific option mismatch" } */
+{
+  result[0] = fmaxf (operand_a, operand_b);
+  result[1] = fminf (operand_a, operand_b);
+}
+
+extern void
+custom_fp (float operand_a, float operand_b, float *result);
+
+void
+custom_fp (float operand_a, float operand_b, float *result)
+{
+  custom_fp1 (operand_a, operand_b, result);
+}
+
diff --git a/gcc/testsuite/gcc.target/nios2/custom-fp-inline-3.c b/gcc/testsuite/gcc.target/nios2/custom-fp-inline-3.c
new file mode 100644
index 0000000..86b1064
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/custom-fp-inline-3.c
@@ -0,0 +1,33 @@
+/* Test that you can inline a function without custom insn attributes into
+   one that does have custom insn attributes.  */
+
+/* { dg-do compile } */
+/* { dg-options "-O1 -ffinite-math-only" } */
+
+/* -O1 in the options is significant.  Without it FP operations may not be
+   optimized to custom instructions.  */
+
+#include <math.h>
+
+static inline 
+__attribute__ ((always_inline))
+void
+custom_fp1 (float operand_a, float operand_b, float *result)
+
+{
+  result[0] = fmaxf (operand_a, operand_b);
+  result[1] = fminf (operand_a, operand_b);
+}
+
+extern void
+custom_fp (float operand_a, float operand_b, float *result)
+  __attribute__ ((target ("custom-fmaxs=246,custom-fmins=247")));
+
+void
+custom_fp (float operand_a, float operand_b, float *result)
+{
+  custom_fp1 (operand_a, operand_b, result);
+}
+
+/* { dg-final { scan-assembler "custom\\t246, .* # fmaxs .*" } } */
+/* { dg-final { scan-assembler "custom\\t247, .* # fmins .*" } } */
diff --git a/gcc/testsuite/gcc.target/nios2/custom-fp-inline-4.c b/gcc/testsuite/gcc.target/nios2/custom-fp-inline-4.c
new file mode 100644
index 0000000..5e2c1bc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nios2/custom-fp-inline-4.c
@@ -0,0 +1,29 @@
+/* Test that you cannot inline a function with custom insn attributes into
+   one with incompatible attributes.  */
+
+/* { dg-do compile } */
+/* { dg-options "-O1 -ffinite-math-only" } */
+
+/* -O1 in the options is significant.  Without it FP operations may not be
+   optimized to custom instructions.  */
+
+#include <math.h>
+
+static inline
+__attribute__ ((always_inline, target ("custom-fmaxs=246,custom-fmins=247")))
+void
+custom_fp1 (float operand_a, float operand_b, float *result)  /* { dg-error "target specific option mismatch" } */
+{
+  result[0] = fmaxf (operand_a, operand_b);
+  result[1] = fminf (operand_a, operand_b);
+}
+
+extern void
+custom_fp (float operand_a, float operand_b, float *result)
+  __attribute__ ((target ("custom-fmaxs=200,custom-fmins=201")));
+
+void
+custom_fp (float operand_a, float operand_b, float *result)
+{
+  custom_fp1 (operand_a, operand_b, result);
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/builtins-1.c b/gcc/testsuite/gcc.target/powerpc/builtins-1.c
index 2dafa90..63fbd2e 100644
--- a/gcc/testsuite/gcc.target/powerpc/builtins-1.c
+++ b/gcc/testsuite/gcc.target/powerpc/builtins-1.c
@@ -326,7 +326,7 @@ int main ()
 /* { dg-final { scan-assembler-times {\mvpkudus\M} 1 } } */
 /* { dg-final { scan-assembler-times "vperm" 4 } } */
 /* { dg-final { scan-assembler-times "xvrdpi" 2 } } */
-/* { dg-final { scan-assembler-times "xxsel" 10 } } */
+/* { dg-final { scan-assembler-times "xxsel" 5 } } */
 /* { dg-final { scan-assembler-times "xxlxor" 6 } } */
 /* { dg-final { scan-assembler-times "divd" 8  { target lp64 } } } */
 /* { dg-final { scan-assembler-times "divdu" 2  { target lp64 } } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/p10_vec_xl_sext.c b/gcc/testsuite/gcc.target/powerpc/p10_vec_xl_sext.c
index 78e72ac..67d52ad 100644
--- a/gcc/testsuite/gcc.target/powerpc/p10_vec_xl_sext.c
+++ b/gcc/testsuite/gcc.target/powerpc/p10_vec_xl_sext.c
@@ -24,7 +24,7 @@ foo3 (signed long a, signed int *b)
 }
 
 vector signed __int128
-foo4 (signed long a, signed long *b)
+foo4 (signed long a, signed long long *b)
 {
   return vec_xl_sext (a, b);
 }
diff --git a/gcc/testsuite/gcc.target/powerpc/ppc-fortran/pr102767.f90 b/gcc/testsuite/gcc.target/powerpc/ppc-fortran/pr102767.f90
new file mode 100644
index 0000000..6853cfe
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/ppc-fortran/pr102767.f90
@@ -0,0 +1,21 @@
+! { dg-require-effective-target powerpc_vsx_ok }
+! { dg-options "-mvsx -O2 -ftree-vectorize -mno-efficient-unaligned-vsx" }
+
+INTERFACE
+  FUNCTION elemental_mult (a, b, c)
+    type(*), DIMENSION(..) :: a, b, c
+  END
+END INTERFACE
+
+allocatable  z
+integer, dimension(2,2) :: a, b
+call test_CFI_address
+contains
+  subroutine test_CFI_address
+    if (elemental_mult (z, x, y) .ne. 0) stop
+    a = reshape ([4,3,2,1], [2,2])
+    b = reshape ([2,3,4,5], [2,2])
+    if (elemental_mult (i, a, b) .ne. 0) stop
+  end
+end
+
diff --git a/gcc/testsuite/gcc.target/powerpc/pr102868.c b/gcc/testsuite/gcc.target/powerpc/pr102868.c
new file mode 100644
index 0000000..eb45d19
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/pr102868.c
@@ -0,0 +1,53 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target powerpc_vsx_ok } */
+/* { dg-options "-O2 -mvsx" } */
+
+#include <altivec.h>
+vector float b = {0.0f, 0.0f, 0.0f, 0.0f};
+
+
+vector float foo1 (vector float x)
+{
+  vector int c = {0, 1, 4, 5};
+  return __builtin_shuffle (x, b, c);
+}
+
+vector float foo2 (vector float x)
+{
+  vector int c = {2, 3, 4, 5};
+  return __builtin_shuffle (x, b, c);
+}
+
+vector float foo3 (vector float x)
+{
+  vector int c = {0, 1, 6, 7};
+  return __builtin_shuffle (x, b, c);
+}
+
+vector float foo4 (vector float x)
+{
+  vector int c = {2, 3, 6, 7};
+  return __builtin_shuffle (x, b, c);
+}
+
+vector unsigned char foo5 (vector unsigned char x)
+{
+  vector unsigned char c = {0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7};
+  return __builtin_shuffle (x, c);
+}
+
+vector unsigned char foo6 (vector unsigned char x)
+{
+  vector unsigned char c = {8, 9, 10, 11, 12, 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15};
+  return __builtin_shuffle (x, c);
+}
+
+vector unsigned char foo7 (vector unsigned char x)
+{
+  vector unsigned char c = {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7};
+  return __builtin_shuffle (x, c);
+}
+
+/* { dg-final { scan-assembler-times {\mxxpermdi\M} 7 { target has_arch_pwr9 } } } */
+/* { dg-final { scan-assembler-times {\mxxpermdi\M} 7 { target { {! has_arch_pwr9} && be } } } } */
+/* { dg-final { scan-assembler-times {\mxxpermdi\M} 11 { target { {! has_arch_pwr9} && le } } } } */
diff --git a/gcc/testsuite/gcc.target/powerpc/pr78102.c b/gcc/testsuite/gcc.target/powerpc/pr78102.c
index 434e677..0b50910 100644
--- a/gcc/testsuite/gcc.target/powerpc/pr78102.c
+++ b/gcc/testsuite/gcc.target/powerpc/pr78102.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -mvsx" } */
-/* { dg-require-effective-target vsx_hw } */
+/* { dg-options "-O2 -mdejagnu-cpu=power8 -DNO_WARN_X86_INTRINSICS" } */
+/* { dg-require-effective-target powerpc_p8vector_ok } */
 
 #include <x86intrin.h>
 
diff --git a/gcc/testsuite/gcc.target/powerpc/pr94613.c b/gcc/testsuite/gcc.target/powerpc/pr94613.c
new file mode 100644
index 0000000..13cab13
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/pr94613.c
@@ -0,0 +1,47 @@
+/* { dg-do run } */
+/* { dg-require-effective-target vmx_hw } */
+/* { dg-options "-O2 -maltivec" } */
+
+#include <altivec.h>
+
+/* The initial implementation of vec_sel used an IF_THEN_ELSE rtx.
+   This did NOT match what the vsel instruction does.  vsel is a
+   bit-wise operation.  Using IF_THEN_ELSE made the + operation to be
+   simplified away in combine.  A plus operation affects other bits in
+   the same element. Hence per-element simplifications are wrong for
+   vsel.  */
+vector unsigned char __attribute__((noinline))
+foo (vector unsigned char a, vector unsigned char b, vector unsigned char c)
+{
+  return vec_sel (a + b, c, a);
+}
+
+vector unsigned char __attribute__((noinline))
+foor (vector unsigned char a, vector unsigned char b, vector unsigned char c)
+{
+  return vec_sel (c, a + b, ~a);
+}
+
+vector unsigned char __attribute__((noinline))
+bar (vector unsigned char a, vector unsigned char b, vector unsigned char c)
+{
+  return vec_sel (a | b, c, a);
+}
+
+int
+main ()
+{
+  vector unsigned char v = (vector unsigned char){ 1 };
+
+  if (foo (v, v, v)[0] != 3)
+      __builtin_abort ();
+
+  if (bar (v, v, v)[0] != 1)
+    __builtin_abort ();
+
+  if (foor (v, v, v)[0] != 3)
+    __builtin_abort ();
+
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.target/powerpc/test_mffsl.c b/gcc/testsuite/gcc.target/powerpc/test_mffsl.c
index 41377ef..28c2b91 100644
--- a/gcc/testsuite/gcc.target/powerpc/test_mffsl.c
+++ b/gcc/testsuite/gcc.target/powerpc/test_mffsl.c
@@ -1,5 +1,6 @@
 /* { dg-do run { target { powerpc*-*-* } } } */
-/* { dg-options "-O2 -std=c99" } */
+/* { dg-options "-O2 -std=c99 -mcpu=power9" } */
+/* { dg-require-effective-target powerpc_p9vector_ok } */
 
 #ifdef DEBUG
 #include <stdio.h>
diff --git a/gcc/testsuite/gcc.target/powerpc/vec-minmax-1.c b/gcc/testsuite/gcc.target/powerpc/vec-minmax-1.c
new file mode 100644
index 0000000..3f6e376
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/vec-minmax-1.c
@@ -0,0 +1,53 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target powerpc_p9vector_ok } */
+/* { dg-options "-O2 -mdejagnu-cpu=power9" } */
+/* { dg-final { scan-assembler-times {\mxvmaxdp\M} 1 } } */
+/* { dg-final { scan-assembler-times {\mxvmaxsp\M} 1 } } */
+/* { dg-final { scan-assembler-times {\mxvmindp\M} 1 } } */
+/* { dg-final { scan-assembler-times {\mxvminsp\M} 1 } } */
+
+/* This test verifies that float or double vec_min/max are bound to
+   xv[min|max][d|s]p instructions when fast-math is not set.  */
+
+
+#include <altivec.h>
+
+#ifdef _BIG_ENDIAN
+   const int PREF_D = 0;
+#else
+   const int PREF_D = 1;
+#endif
+
+double vmaxd (double a, double b)
+{
+  vector double va = vec_promote (a, PREF_D);
+  vector double vb = vec_promote (b, PREF_D);
+  return vec_extract (vec_max (va, vb), PREF_D);
+}
+
+double vmind (double a, double b)
+{
+  vector double va = vec_promote (a, PREF_D);
+  vector double vb = vec_promote (b, PREF_D);
+  return vec_extract (vec_min (va, vb), PREF_D);
+}
+
+#ifdef _BIG_ENDIAN
+   const int PREF_F = 0;
+#else
+   const int PREF_F = 3;
+#endif
+
+float vmaxf (float a, float b)
+{
+  vector float va = vec_promote (a, PREF_F);
+  vector float vb = vec_promote (b, PREF_F);
+  return vec_extract (vec_max (va, vb), PREF_F);
+}
+
+float vminf (float a, float b)
+{
+  vector float va = vec_promote (a, PREF_F);
+  vector float vb = vec_promote (b, PREF_F);
+  return vec_extract (vec_min (va, vb), PREF_F);
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/vec-minmax-2.c b/gcc/testsuite/gcc.target/powerpc/vec-minmax-2.c
new file mode 100644
index 0000000..b27bebb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/vec-minmax-2.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target powerpc_p9vector_ok } */
+/* { dg-options "-O2 -mdejagnu-cpu=power9 -ffast-math" } */
+/* { dg-final { scan-assembler-times {\mxsmaxcdp\M} 2 } } */
+/* { dg-final { scan-assembler-times {\mxsmincdp\M} 2 } } */
+
+/* This test verifies that float or double vec_min/max can be converted
+   to scalar comparison when fast-math is set.  */
+
+
+#include <altivec.h>
+
+#ifdef _BIG_ENDIAN
+   const int PREF_D = 0;
+#else
+   const int PREF_D = 1;
+#endif
+
+double vmaxd (double a, double b)
+{
+  vector double va = vec_promote (a, PREF_D);
+  vector double vb = vec_promote (b, PREF_D);
+  return vec_extract (vec_max (va, vb), PREF_D);
+}
+
+double vmind (double a, double b)
+{
+  vector double va = vec_promote (a, PREF_D);
+  vector double vb = vec_promote (b, PREF_D);
+  return vec_extract (vec_min (va, vb), PREF_D);
+}
+
+#ifdef _BIG_ENDIAN
+   const int PREF_F = 0;
+#else
+   const int PREF_F = 3;
+#endif
+
+float vmaxf (float a, float b)
+{
+  vector float va = vec_promote (a, PREF_F);
+  vector float vb = vec_promote (b, PREF_F);
+  return vec_extract (vec_max (va, vb), PREF_F);
+}
+
+float vminf (float a, float b)
+{
+  vector float va = vec_promote (a, PREF_F);
+  vector float vb = vec_promote (b, PREF_F);
+  return vec_extract (vec_min (va, vb), PREF_F);
+}
diff --git a/gcc/testsuite/gcc.target/riscv/pr102957.c b/gcc/testsuite/gcc.target/riscv/pr102957.c
new file mode 100644
index 0000000..8a7e541
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/pr102957.c
@@ -0,0 +1,5 @@
+/* { dg-do compile } */
+/* { dg-options "-O -march=rv64gzb -mabi=lp64" } */
+int foo()
+{
+}
diff --git a/gcc/testsuite/gcc.target/riscv/zba-adduw.c b/gcc/testsuite/gcc.target/riscv/zba-adduw.c
new file mode 100644
index 0000000..cac1e84
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zba-adduw.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zba_zbs -mabi=lp64 -O2" } */
+
+int foo(int n, unsigned char *arr, unsigned y){
+  int s = 0;
+  unsigned x = 0;
+  for (;x<n;x++)
+    s += arr[x+y];
+  return s;
+}
+
+/* { dg-final { scan-assembler "add.uw" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zba-shNadd-01.c b/gcc/testsuite/gcc.target/riscv/zba-shNadd-01.c
new file mode 100644
index 0000000..aaabaf5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zba-shNadd-01.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zba -mabi=lp64 -O2" } */
+
+long test_1(long a, long b)
+{
+  return a + (b << 1);
+}
+long test_2(long a, long b)
+{
+  return a + (b << 2);
+}
+long test_3(long a, long b)
+{
+  return a + (b << 3);
+}
+
+/* { dg-final { scan-assembler-times "sh1add" 1 } } */
+/* { dg-final { scan-assembler-times "sh2add" 1 } } */
+/* { dg-final { scan-assembler-times "sh3add" 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zba-shNadd-02.c b/gcc/testsuite/gcc.target/riscv/zba-shNadd-02.c
new file mode 100644
index 0000000..8dfea4a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zba-shNadd-02.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc_zba -mabi=ilp32 -O2" } */
+
+long test_1(long a, long b)
+{
+  return a + (b << 1);
+}
+long test_2(long a, long b)
+{
+  return a + (b << 2);
+}
+long test_3(long a, long b)
+{
+  return a + (b << 3);
+}
+
+/* { dg-final { scan-assembler-times "sh1add" 1 } } */
+/* { dg-final { scan-assembler-times "sh2add" 1 } } */
+/* { dg-final { scan-assembler-times "sh3add" 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zba-shNadd-03.c b/gcc/testsuite/gcc.target/riscv/zba-shNadd-03.c
new file mode 100644
index 0000000..b2ea231
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zba-shNadd-03.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zba -mabi=lp64 -O2" } */
+
+/* RV64 only.  */
+int foos(short *x, int n){
+  return x[n];
+}
+int fooi(int *x, int n){
+  return x[n];
+}
+int fooll(long long *x, int n){
+  return x[n];
+}
+
+/* RV64 only.  */
+int ufoos(short *x, unsigned int n){
+  return x[n];
+}
+int ufooi(int *x, unsigned int n){
+  return x[n];
+}
+int ufooll(long long *x, unsigned int n){
+  return x[n];
+}
+
+/* { dg-final { scan-assembler-times "sh1add\t" 1 } } */
+/* { dg-final { scan-assembler-times "sh2add\t" 1 } } */
+/* { dg-final { scan-assembler-times "sh3add\t" 1 } } */
+/* { dg-final { scan-assembler-times "sh3add.uw" 1 } } */
+/* { dg-final { scan-assembler-times "sh3add.uw" 1 } } */
+/* { dg-final { scan-assembler-times "sh3add.uw" 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zba-slliuw.c b/gcc/testsuite/gcc.target/riscv/zba-slliuw.c
new file mode 100644
index 0000000..a7a3dc7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zba-slliuw.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zba_zbs -mabi=lp64 -O2" } */
+
+long
+foo (long i)
+{
+  return (long)(unsigned int)i << 10;
+}
+/* XXX: This pattern need combine improvement or intermediate instruction
+ *      from zbs.   */
+/* { dg-final { scan-assembler "slli.uw" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zba-zextw.c b/gcc/testsuite/gcc.target/riscv/zba-zextw.c
new file mode 100644
index 0000000..26fd64d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zba-zextw.c
@@ -0,0 +1,10 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zba_zbs -mabi=lp64 -O2" } */
+
+long
+foo (long i)
+{
+  return (long)(unsigned int)i;
+}
+/* XXX: This pattern require combine improvement.   */
+/* { dg-final { scan-assembler-not "slli.uw" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zbb-andn-orn-xnor-01.c b/gcc/testsuite/gcc.target/riscv/zbb-andn-orn-xnor-01.c
new file mode 100644
index 0000000..0037dea
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbb-andn-orn-xnor-01.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zbb -mabi=lp64 -O2" } */
+
+unsigned long long foo1(unsigned long long rs1, unsigned long long rs2)
+{
+return rs1 & ~rs2;
+}
+
+unsigned long long foo2(unsigned long long rs1, unsigned long long rs2)
+{
+return rs1 | ~rs2;
+}
+
+unsigned long long foo3(unsigned long long rs1, unsigned long long rs2)
+{
+return rs1 ^ ~rs2;
+}
+
+/* { dg-final { scan-assembler-times "andn" 2 } } */
+/* { dg-final { scan-assembler-times "orn" 2 } } */
+/* { dg-final { scan-assembler-times "xnor" 2 } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/riscv/zbb-andn-orn-xnor-02.c b/gcc/testsuite/gcc.target/riscv/zbb-andn-orn-xnor-02.c
new file mode 100644
index 0000000..b0c1e40
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbb-andn-orn-xnor-02.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc_zbb -mabi=ilp32 -O2" } */
+
+unsigned int foo1(unsigned int rs1, unsigned int rs2)
+{
+return rs1 & ~rs2;
+}
+
+unsigned int foo2(unsigned  int rs1, unsigned  int rs2)
+{
+return rs1 | ~rs2;
+}
+
+unsigned int foo3(unsigned int rs1, unsigned int rs2)
+{
+return rs1 ^ ~rs2;
+}
+
+/* { dg-final { scan-assembler-times "andn" 2 } } */
+/* { dg-final { scan-assembler-times "orn" 2 } } */
+/* { dg-final { scan-assembler-times "xnor" 2 } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/riscv/zbb-li-rotr.c b/gcc/testsuite/gcc.target/riscv/zbb-li-rotr.c
new file mode 100644
index 0000000..03254ed
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbb-li-rotr.c
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zbb -mabi=lp64 -O2" } */
+
+long
+li_rori (void)
+{
+  return 0xffff77ffffffffffL;
+}
+
+long
+li_rori_2 (void)
+{
+  return 0x77ffffffffffffffL;
+}
+
+long
+li_rori_3 (void)
+{
+  return 0xfffffffeefffffffL;
+}
+
+long
+li_rori_4 (void)
+{
+  return 0x5ffffffffffffff5L;
+}
+
+long
+li_rori_5 (void)
+{
+  return 0xaffffffffffffffaL;
+}
+
+
+/* { dg-final { scan-assembler-times "rori\t" 5 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zbb-min-max.c b/gcc/testsuite/gcc.target/riscv/zbb-min-max.c
new file mode 100644
index 0000000..f44c398
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbb-min-max.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zbb -mabi=lp64 -O2" } */
+
+long
+foo1 (long i, long j)
+{
+  return i < j ? i : j;
+}
+
+long
+foo2 (long i, long j)
+{
+  return i > j ? i : j;
+}
+
+unsigned long
+foo3 (unsigned long i, unsigned long j)
+{
+  return i < j ? i : j;
+}
+
+unsigned long
+foo4 (unsigned long i, unsigned long j)
+{
+  return i > j ? i : j;
+}
+
+/* { dg-final { scan-assembler-times "min" 3 } } */
+/* { dg-final { scan-assembler-times "max" 3 } } */
+/* { dg-final { scan-assembler-times "minu" 1 } } */
+/* { dg-final { scan-assembler-times "maxu" 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zbb-rol-ror-01.c b/gcc/testsuite/gcc.target/riscv/zbb-rol-ror-01.c
new file mode 100644
index 0000000..9589662
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbb-rol-ror-01.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zbb -mabi=lp64 -O2" } */
+
+unsigned long foo1(unsigned long rs1, unsigned long rs2)
+{
+    long shamt = rs2 & (64 - 1);
+    return (rs1 << shamt) | (rs1 >> ((64 - shamt) & (64 - 1)));
+}
+unsigned long foo2(unsigned long rs1, unsigned long rs2)
+{
+    unsigned long shamt = rs2 & (64 - 1);
+    return (rs1 >> shamt) | (rs1 << ((64 - shamt) & (64 - 1)));
+}
+
+/* { dg-final { scan-assembler-times "rol" 2 } } */
+/* { dg-final { scan-assembler-times "ror" 2 } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/riscv/zbb-rol-ror-02.c b/gcc/testsuite/gcc.target/riscv/zbb-rol-ror-02.c
new file mode 100644
index 0000000..24b482f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbb-rol-ror-02.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc_zbb -mabi=ilp32 -O2" } */
+
+unsigned int foo1(unsigned int rs1, unsigned int rs2)
+{
+    unsigned int shamt = rs2 & (32 - 1);
+    return (rs1 << shamt) | (rs1 >> ((32 - shamt) & (32 - 1)));
+}
+unsigned int foo2(unsigned int rs1, unsigned int rs2)
+{
+    unsigned int shamt = rs2 & (32 - 1);
+    return (rs1 >> shamt) | (rs1 << ((32 - shamt) & (32 - 1)));
+}
+
+/* { dg-final { scan-assembler-times "rol" 2 } } */
+/* { dg-final { scan-assembler-times "ror" 2 } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/riscv/zbb-rol-ror-03.c b/gcc/testsuite/gcc.target/riscv/zbb-rol-ror-03.c
new file mode 100644
index 0000000..ffde7c9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbb-rol-ror-03.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zbb -mabi=lp64 -O2" } */
+
+/* RV64 only*/
+unsigned int rol(unsigned int rs1, unsigned int rs2)
+{
+    int shamt = rs2 & (32 - 1);
+    return (rs1 << shamt) | (rs1 >> ((64 - shamt) & (32 - 1)));
+}
+unsigned int ror(unsigned int rs1, unsigned int rs2)
+{
+    int shamt = rs2 & (64 - 1);
+    return (rs1 >> shamt) | (rs1 << ((32 - shamt) & (32 - 1)));
+}
+
+/* { dg-final { scan-assembler-times "rolw" 1 } } */
+/* { dg-final { scan-assembler-times "rorw" 1 } } */
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/riscv/zbbw.c b/gcc/testsuite/gcc.target/riscv/zbbw.c
new file mode 100644
index 0000000..236ddf7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbbw.c
@@ -0,0 +1,25 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zbb -mabi=lp64 -O2" } */
+
+int
+clz (int i)
+{
+  return __builtin_clz (i);
+}
+
+int
+ctz (int i)
+{
+  return __builtin_ctz (i);
+}
+
+int
+popcount (int i)
+{
+  return __builtin_popcount (i);
+}
+
+
+/* { dg-final { scan-assembler-times "clzw" 1 } } */
+/* { dg-final { scan-assembler-times "ctzw" 1 } } */
+/* { dg-final { scan-assembler-times "cpopw" 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zbs-bclr.c b/gcc/testsuite/gcc.target/riscv/zbs-bclr.c
new file mode 100644
index 0000000..4a3c2f1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbs-bclr.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zbs -mabi=lp64 -O2" } */
+
+/* bclr */
+long
+foo0 (long i, long j)
+{
+  return i & ~(1L << j);
+}
+
+/* bclri */
+long
+foo1 (long i)
+{
+  return i & ~(1L << 20);
+}
+
+/* { dg-final { scan-assembler-times "bclr\t" 1 } } */
+/* { dg-final { scan-assembler-times "bclri\t" 1 } } */
+/* { dg-final { scan-assembler-not "andi" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zbs-bext.c b/gcc/testsuite/gcc.target/riscv/zbs-bext.c
new file mode 100644
index 0000000..a093cdc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbs-bext.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zbs -mabi=lp64 -O2" } */
+
+/* bext */
+long
+foo0 (long i, long j)
+{
+  return 1L & (i >> j);
+}
+
+/* bexti */
+long
+foo1 (long i)
+{
+  return 1L & (i >> 20);
+}
+
+/* { dg-final { scan-assembler-times "bexti\t" 1 } } */
+/* { dg-final { scan-assembler-times "bext\t" 1 } } */
+/* { dg-final { scan-assembler-not "andi" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zbs-binv.c b/gcc/testsuite/gcc.target/riscv/zbs-binv.c
new file mode 100644
index 0000000..e4e48b9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbs-binv.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zbs -mabi=lp64 -O2" } */
+
+/* binv */
+long
+foo0 (long i, long j)
+{
+  return i ^ (1L << j);
+}
+
+/* binvi */
+long
+foo1 (long i)
+{
+  return i ^ (1L << 20);
+}
+
+/* { dg-final { scan-assembler-times "binv\t" 1 } } */
+/* { dg-final { scan-assembler-times "binvi\t" 1 } } */
+/* { dg-final { scan-assembler-not "andi" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/zbs-bset.c b/gcc/testsuite/gcc.target/riscv/zbs-bset.c
new file mode 100644
index 0000000..733d427
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/zbs-bset.c
@@ -0,0 +1,41 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zbs -mabi=lp64 -O2" } */
+
+/* bset */
+long
+sub0 (long i, long j)
+{
+  return i | (1L << j);
+}
+
+/* bset_mask */
+long
+sub1 (long i, long j)
+{
+  return i | (1L << (j & 0x3f));
+}
+
+/* bset_1 */
+long
+sub2 (long i)
+{
+  return 1L << i;
+}
+
+/* bset_1_mask */
+long
+sub3 (long i)
+{
+  return 1L << (i & 0x3f);
+}
+
+/* bseti */
+long
+sub4 (long i)
+{
+  return i | (1L << 20);
+}
+
+/* { dg-final { scan-assembler-times "bset\t" 4 } } */
+/* { dg-final { scan-assembler-times "bseti\t" 1 } } */
+/* { dg-final { scan-assembler-not "andi" } } */
diff --git a/gcc/testsuite/gcc.target/vax/setmem.c b/gcc/testsuite/gcc.target/vax/setmem.c
new file mode 100644
index 0000000..6c60cfc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/vax/setmem.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-skip-if "code quality test" { *-*-* } { "-O0" } { "" } } */
+
+#include <stddef.h>
+
+void *
+memset8 (void *block, int c, size_t size)
+{
+  unsigned char s8 = size;
+  return __builtin_memset (block, c, s8);
+}
+
+/* Expect assembly like:
+
+	movl 4(%ap),%r6
+	movzbl 12(%ap),%r7
+	movc5 $0,(%ap),8(%ap),%r7,(%r6)
+	movl %r6,%r0
+
+ */
+
+/* { dg-final { scan-assembler "\tmovc5 \\\$0,\\\(%ap\\\)," } } */
diff --git a/gcc/testsuite/gdc.dg/torture/pr96435.d b/gcc/testsuite/gdc.dg/torture/pr96435.d
index c6d8785..896b25f 100644
--- a/gcc/testsuite/gdc.dg/torture/pr96435.d
+++ b/gcc/testsuite/gdc.dg/torture/pr96435.d
@@ -6,7 +6,7 @@
     int[2] array = [16, 678];
     union U { int i; bool b; }
     U u;
-    u.i = 0xDEADBEEF;
+    u.i = 0x81818181;
     assert(array[u.b] == 678);
     return u.b;
 }
diff --git a/gcc/testsuite/gfortran.dg/PR100906.c b/gcc/testsuite/gfortran.dg/PR100906.c
new file mode 100644
index 0000000..f71d5677
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/PR100906.c
@@ -0,0 +1,169 @@
+/* Test the fix for PR100906 */
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+/* #include <uchar.h> */
+
+#include <ISO_Fortran_binding.h>
+
+#define _CFI_type_mask 0xFF
+#define _CFI_type_kind_shift 8
+
+#define _CFI_decode_type(NAME) (signed char)((NAME) & CFI_type_mask)
+#define _CFI_decode_kind(NAME) (signed char)(((NAME) >> CFI_type_kind_shift) & CFI_type_mask)
+
+#define _CFI_encode_type(TYPE, KIND) (int16_t)\
+((((KIND) & CFI_type_mask) << CFI_type_kind_shift)\
+ | ((TYPE) & CFI_type_mask))
+
+#define N 11
+#define M 7
+
+typedef char c_char;
+/* typedef char32_t c_ucs4_char; */
+typedef uint32_t char32_t;
+typedef uint32_t c_ucs4_char;
+ 
+bool charcmp (char *, char, size_t);
+
+bool ucharcmp (char32_t *, char32_t, size_t);
+
+bool c_vrfy_c_char (const CFI_cdesc_t *restrict, const size_t);
+
+bool c_vrfy_c_ucs4_char (const CFI_cdesc_t *restrict, const size_t);
+
+bool c_vrfy_character (const CFI_cdesc_t *restrict, const size_t);
+ 
+void check_tk (const CFI_cdesc_t*restrict, const CFI_type_t, const signed char, const size_t, const size_t);
+
+bool
+charcmp (char *c, char v, size_t n)
+{
+  bool res = true;
+  char b = (char)'A';
+  size_t i;
+
+  for (i=0; ((i<n)&&(res)); i++, c++)
+    res = (*c == (v+b));
+  return res;
+}
+
+bool
+ucharcmp (char32_t *c, char32_t v, size_t n)
+{
+  bool res = true;
+  char32_t b = (char32_t)0xFF01;
+  size_t i;
+
+  for (i=0; ((i<n)&&(res)); i++, c++)
+    res = (*c == (v+b));
+  return res;
+}
+
+bool
+c_vrfy_c_char (const CFI_cdesc_t *restrict auxp, const size_t len)
+{
+  CFI_index_t i, lb, ub, ex;
+  size_t sz;
+  c_char *ip = NULL;
+
+  assert (auxp);
+  assert (auxp->base_addr);
+  assert (auxp->elem_len>0);
+  lb = auxp->dim[0].lower_bound;
+  ex = auxp->dim[0].extent;
+  assert (ex==N);
+  sz = (size_t)auxp->elem_len / sizeof (c_char);
+  assert (sz==len);
+  ub = ex + lb - 1;
+  ip = (c_char*)auxp->base_addr;
+  for (i=0; i<ex; i++, ip+=sz)
+    if (!charcmp (ip, (c_char)(i), sz))
+      return false;
+  for (i=lb; i<ub+1; i++)
+    {
+      ip = (c_char*)CFI_address(auxp, &i);
+      if (!charcmp (ip, (c_char)(i-lb), sz))
+	return false;
+    }
+  return true;
+}
+
+bool
+c_vrfy_c_ucs4_char (const CFI_cdesc_t *restrict auxp, const size_t len)
+{
+  CFI_index_t i, lb, ub, ex;
+  size_t sz;
+  c_ucs4_char *ip = NULL;
+
+  assert (auxp);
+  assert (auxp->base_addr);
+  assert (auxp->elem_len>0);
+  lb = auxp->dim[0].lower_bound;
+  ex = auxp->dim[0].extent;
+  assert (ex==N);
+  sz = (size_t)auxp->elem_len / sizeof (c_ucs4_char);
+  assert (sz==len);
+  ub = ex + lb - 1;
+  ip = (c_ucs4_char*)auxp->base_addr;
+  for (i=0; i<ex; i++, ip+=sz)
+    if (!ucharcmp (ip, (c_ucs4_char)(i), sz))
+      return false;
+  for (i=lb; i<ub+1; i++)
+    {
+      ip = (c_ucs4_char*)CFI_address(auxp, &i);
+      if (!ucharcmp (ip, (c_ucs4_char)(i-lb), sz))
+	return false;
+    }
+  return true;
+}
+
+bool
+c_vrfy_character (const CFI_cdesc_t *restrict auxp, const size_t len)
+{
+  signed char type, kind;
+  
+  assert (auxp);
+  type = _CFI_decode_type(auxp->type);
+  kind = _CFI_decode_kind(auxp->type);
+  assert (type == CFI_type_Character);
+  switch (kind)
+    {
+    case 1:
+      return c_vrfy_c_char (auxp, len);
+      break;
+    case 4:
+      return c_vrfy_c_ucs4_char (auxp, len);
+      break;
+    default:
+      assert (false);
+    }
+  return true;
+}
+
+void
+check_tk (const CFI_cdesc_t *restrict auxp, const CFI_type_t type, const signed char kind, const size_t elem_len, const size_t nelem)
+{
+  signed char ityp, iknd;
+
+  assert (auxp);
+  assert (auxp->elem_len==elem_len*nelem);
+  assert (auxp->rank==1);
+  assert (auxp->dim[0].sm>0);
+  assert ((size_t)auxp->dim[0].sm==elem_len*nelem);
+  /*  */
+  assert (auxp->type==type);
+  ityp = _CFI_decode_type(auxp->type);
+  assert (ityp == CFI_type_Character);
+  iknd = _CFI_decode_kind(auxp->type);
+  assert (_CFI_decode_type(type)==ityp);
+  assert (kind==iknd);
+  assert (c_vrfy_character (auxp, nelem));
+  return;
+}
+
+// Local Variables:
+// mode: C
+// End:
diff --git a/gcc/testsuite/gfortran.dg/PR100906.f90 b/gcc/testsuite/gfortran.dg/PR100906.f90
new file mode 100644
index 0000000..f6cb3af
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/PR100906.f90
@@ -0,0 +1,1699 @@
+! { dg-do run }
+! { dg-additional-sources PR100906.c }
+!
+! Test the fix for PR100906
+! 
+
+module isof_m
+
+  use, intrinsic :: iso_c_binding, only: &
+    c_signed_char, c_int16_t
+  
+  implicit none
+
+  private
+  
+  public ::             &
+    CFI_type_character
+
+  public ::             &
+    CFI_type_char,      &
+    CFI_type_ucs4_char
+ 
+  public ::      &
+    check_tk_as, &
+    check_tk_ar
+  
+  
+  public ::          &
+    cfi_encode_type
+  
+  integer, parameter :: CFI_type_t = c_int16_t
+  
+  integer(kind=c_int16_t), parameter :: CFI_type_mask = int(z"FF", kind=c_int16_t)
+  integer(kind=c_int16_t), parameter :: CFI_type_kind_shift = 8_c_int16_t
+
+  ! Intrinsic types. Their kind number defines their storage size. */
+  integer(kind=c_signed_char), parameter :: CFI_type_Character = 5
+
+  ! C-Fortran Interoperability types.
+  integer(kind=cfi_type_t), parameter :: CFI_type_char      = &
+    ior(int(CFI_type_Character, kind=c_int16_t), shiftl(1_c_int16_t, CFI_type_kind_shift))
+  integer(kind=cfi_type_t), parameter :: CFI_type_ucs4_char = &
+    ior(int(CFI_type_Character, kind=c_int16_t), shiftl(4_c_int16_t, CFI_type_kind_shift))
+
+  interface
+    subroutine check_tk_as(a, t, k, e, n) &
+      bind(c, name="check_tk")
+      use, intrinsic :: iso_c_binding, only: &
+        c_int16_t, c_signed_char, c_size_t
+      implicit none
+      type(*),                       intent(in) :: a(:)
+      integer(c_int16_t),     value, intent(in) :: t
+      integer(c_signed_char), value, intent(in) :: k
+      integer(c_size_t),      value, intent(in) :: e
+      integer(c_size_t),      value, intent(in) :: n
+    end subroutine check_tk_as
+    subroutine check_tk_ar(a, t, k, e, n) &
+      bind(c, name="check_tk")
+      use, intrinsic :: iso_c_binding, only: &
+        c_int16_t, c_signed_char, c_size_t
+      implicit none
+      type(*),                       intent(in) :: a(..)
+      integer(c_int16_t),     value, intent(in) :: t
+      integer(c_signed_char), value, intent(in) :: k
+      integer(c_size_t),      value, intent(in) :: e
+      integer(c_size_t),      value, intent(in) :: n
+    end subroutine check_tk_ar
+  end interface
+
+contains
+
+  elemental function cfi_encode_type(type, kind) result(itype)
+    integer(kind=c_signed_char), intent(in) :: type
+    integer(kind=c_signed_char), intent(in) :: kind
+
+    integer(kind=c_int16_t) :: itype, ikind
+
+    itype = int(type, kind=c_int16_t)
+    itype = iand(itype, CFI_type_mask)
+    ikind = int(kind, kind=c_int16_t)
+    ikind = iand(ikind, CFI_type_mask)
+    ikind = shiftl(ikind, CFI_type_kind_shift)
+    itype = ior(ikind, itype)
+    return
+  end function cfi_encode_type
+  
+end module isof_m
+
+module iso_check_m
+
+  use, intrinsic :: iso_c_binding, only: &
+    c_signed_char, c_int16_t, c_size_t
+
+  use, intrinsic :: iso_c_binding, only: &
+    c_char
+
+  use :: isof_m, only:  &
+    CFI_type_character
+
+  use :: isof_m, only:  &
+    CFI_type_char,      &
+    CFI_type_ucs4_char
+
+  use :: isof_m, only: &
+    check_tk_as,       &
+    check_tk_ar
+
+  use :: isof_m, only: &
+    cfi_encode_type
+
+  implicit none
+
+  private
+
+  public ::               &
+    check_c_char_l1,      &
+    check_c_char_lm,      &
+    check_c_ucs4_char_l1, &
+    check_c_ucs4_char_lm
+  
+  integer                           :: i
+  integer(kind=c_size_t), parameter :: b = 8
+  integer,                parameter :: n = 11
+  integer,                parameter :: m = 7
+  
+  integer, parameter :: c_ucs4_char = 4
+  
+  character(kind=c_char, len=1), parameter :: ref_c_char_l1(*) = &
+    [(achar(i+iachar("A")-1, kind=c_char), i=1,n)]
+  character(kind=c_char, len=m), parameter :: ref_c_char_lm(*) = &
+    [(repeat(achar(i+iachar("A")-1, kind=c_char), m), i=1,n)]
+  character(kind=c_ucs4_char, len=1), parameter :: ref_c_ucs4_char_l1(*) = &
+    [(achar(i+iachar("A")-1, kind=c_ucs4_char), i=1,n)]
+  character(kind=c_ucs4_char, len=m), parameter :: ref_c_ucs4_char_lm(*) = &
+    [(repeat(achar(i+iachar("A")-1, kind=c_ucs4_char), m), i=1,n)]
+
+contains
+
+  subroutine check_c_char_l1()
+    character(kind=c_char, len=1), target :: a(n)
+    !
+    character(kind=c_char, len=:), pointer :: p(:)
+    !
+    a = ref_c_char_l1
+    call f_check_c_char_c1_as(a)
+    if(any(a/=ref_c_char_l1)) stop 1
+    a = ref_c_char_l1
+    call c_check_c_char_c1_as(a)
+    if(any(a/=ref_c_char_l1)) stop 2
+    a = ref_c_char_l1
+    call f_check_c_char_c1_ar(a)
+    if(any(a/=ref_c_char_l1)) stop 3
+    a = ref_c_char_l1
+    call c_check_c_char_c1_ar(a)
+    if(any(a/=ref_c_char_l1)) stop 4
+    a = ref_c_char_l1
+    call f_check_c_char_a1_as(a)
+    if(any(a/=ref_c_char_l1)) stop 5
+    a = ref_c_char_l1
+    call c_check_c_char_a1_as(a)
+    if(any(a/=ref_c_char_l1)) stop 6
+    a = ref_c_char_l1
+    call f_check_c_char_a1_ar(a)
+    if(any(a/=ref_c_char_l1)) stop 7
+    a = ref_c_char_l1
+    call c_check_c_char_a1_ar(a)
+    if(any(a/=ref_c_char_l1)) stop 8
+    a = ref_c_char_l1
+    p => a  
+    call f_check_c_char_d1_as(p)
+    if(.not.associated(p)) stop 9
+    if(.not.associated(p, a)) stop 10
+    if(any(p/=ref_c_char_l1)) stop 11
+    if(any(a/=ref_c_char_l1)) stop 12
+    a = ref_c_char_l1
+    p => a  
+    call c_check_c_char_d1_as(p)
+    if(.not.associated(p)) stop 13
+    if(.not.associated(p, a)) stop 14
+    if(any(p/=ref_c_char_l1)) stop 15
+    if(any(a/=ref_c_char_l1)) stop 16
+    a = ref_c_char_l1
+    p => a  
+    call f_check_c_char_d1_ar(p)
+    if(.not.associated(p)) stop 17
+    if(.not.associated(p, a)) stop 18
+    if(any(p/=ref_c_char_l1)) stop 19
+    if(any(a/=ref_c_char_l1)) stop 20
+    a = ref_c_char_l1
+    p => a  
+    call c_check_c_char_d1_ar(p)
+    if(.not.associated(p)) stop 21
+    if(.not.associated(p, a)) stop 22
+    if(any(p/=ref_c_char_l1)) stop 23
+    if(any(a/=ref_c_char_l1)) stop 24
+    return
+  end subroutine check_c_char_l1
+
+  subroutine f_check_c_char_c1_as(a)
+    character(kind=c_char, len=1), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 25
+    if(k/=1_c_signed_char) stop 26
+    if(n/=1) stop 27
+    if(int(k, kind=c_size_t)/=e) stop 28
+    if(t/=CFI_type_char) stop 29
+    if(any(a/=ref_c_char_l1)) stop 30
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_l1)) stop 31
+    return
+  end subroutine f_check_c_char_c1_as
+
+  subroutine c_check_c_char_c1_as(a) bind(c)
+    character(kind=c_char, len=1), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 32
+    if(k/=1_c_signed_char) stop 33
+    if(n/=1) stop 34
+    if(int(k, kind=c_size_t)/=e) stop 35
+    if(t/=CFI_type_char) stop 36
+    if(any(a/=ref_c_char_l1)) stop 37
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_l1)) stop 38
+    return
+  end subroutine c_check_c_char_c1_as
+
+  subroutine f_check_c_char_c1_ar(a)
+    character(kind=c_char, len=1), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 39
+    if(k/=1_c_signed_char) stop 40
+    if(n/=1) stop 41
+    if(int(k, kind=c_size_t)/=e) stop 42
+    if(t/=CFI_type_char) stop 43
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 44
+    rank default
+      stop 45
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 46
+    rank default
+      stop 47
+    end select
+    return
+  end subroutine f_check_c_char_c1_ar
+
+  subroutine c_check_c_char_c1_ar(a) bind(c)
+    character(kind=c_char, len=1), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 48
+    if(k/=1_c_signed_char) stop 49
+    if(n/=1) stop 50
+    if(int(k, kind=c_size_t)/=e) stop 51
+    if(t/=CFI_type_char) stop 52
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 53
+    rank default
+      stop 54
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 55
+    rank default
+      stop 56
+    end select
+    return
+  end subroutine c_check_c_char_c1_ar
+
+  subroutine f_check_c_char_a1_as(a)
+    character(kind=c_char, len=*), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 57
+    if(k/=1_c_signed_char) stop 58
+    if(n/=1) stop 59
+    if(int(k, kind=c_size_t)/=e) stop 60
+    if(t/=CFI_type_char) stop 61
+    if(any(a/=ref_c_char_l1)) stop 62
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_l1)) stop 63
+    return
+  end subroutine f_check_c_char_a1_as
+
+  subroutine c_check_c_char_a1_as(a) bind(c)
+    character(kind=c_char, len=*), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 64
+    if(k/=1_c_signed_char) stop 65
+    if(n/=1) stop 66
+    if(int(k, kind=c_size_t)/=e) stop 67
+    if(t/=CFI_type_char) stop 68
+    if(any(a/=ref_c_char_l1)) stop 69
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_l1)) stop 70
+    return
+  end subroutine c_check_c_char_a1_as
+
+  subroutine f_check_c_char_a1_ar(a)
+    character(kind=c_char, len=*), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 71
+    if(k/=1_c_signed_char) stop 72
+    if(n/=1) stop 73
+    if(int(k, kind=c_size_t)/=e) stop 74
+    if(t/=CFI_type_char) stop 75
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 76
+    rank default
+      stop 77
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 78
+    rank default
+      stop 79
+    end select
+    return
+  end subroutine f_check_c_char_a1_ar
+
+  subroutine c_check_c_char_a1_ar(a) bind(c)
+    character(kind=c_char, len=*), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 80
+    if(k/=1_c_signed_char) stop 81
+    if(n/=1) stop 82
+    if(int(k, kind=c_size_t)/=e) stop 83
+    if(t/=CFI_type_char) stop 84
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 85
+    rank default
+      stop 86
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 87
+    rank default
+      stop 88
+    end select
+    return
+  end subroutine c_check_c_char_a1_ar
+
+  subroutine f_check_c_char_d1_as(a)
+    character(kind=c_char, len=:), pointer, intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 89
+    if(k/=1_c_signed_char) stop 90
+    if(n/=1) stop 91
+    if(int(k, kind=c_size_t)/=e) stop 92
+    if(t/=CFI_type_char) stop 93
+    if(any(a/=ref_c_char_l1)) stop 94
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_l1)) stop 95
+    return
+  end subroutine f_check_c_char_d1_as
+
+  subroutine c_check_c_char_d1_as(a) bind(c)
+    character(kind=c_char, len=:), pointer, intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 96
+    if(k/=1_c_signed_char) stop 97
+    if(n/=1) stop 98
+    if(int(k, kind=c_size_t)/=e) stop 99
+    if(t/=CFI_type_char) stop 100
+    if(any(a/=ref_c_char_l1)) stop 101
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_l1)) stop 102
+    return
+  end subroutine c_check_c_char_d1_as
+
+  subroutine f_check_c_char_d1_ar(a)
+    character(kind=c_char, len=:), pointer, intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 103
+    if(k/=1_c_signed_char) stop 104
+    if(n/=1) stop 105
+    if(int(k, kind=c_size_t)/=e) stop 106
+    if(t/=CFI_type_char) stop 107
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 108
+    rank default
+      stop 109
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 110
+    rank default
+      stop 111
+    end select
+    return
+  end subroutine f_check_c_char_d1_ar
+
+  subroutine c_check_c_char_d1_ar(a) bind(c)
+    character(kind=c_char, len=:), pointer, intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 112
+    if(k/=1_c_signed_char) stop 113
+    if(n/=1) stop 114
+    if(int(k, kind=c_size_t)/=e) stop 115
+    if(t/=CFI_type_char) stop 116
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 117
+    rank default
+      stop 118
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_l1)) stop 119
+    rank default
+      stop 120
+    end select
+    return
+  end subroutine c_check_c_char_d1_ar
+
+  subroutine check_c_char_lm()
+    character(kind=c_char, len=m), target :: a(n)
+    !
+    character(kind=c_char, len=:), pointer :: p(:)
+    !
+    a = ref_c_char_lm
+    call f_check_c_char_cm_as(a)
+    if(any(a/=ref_c_char_lm)) stop 121
+    a = ref_c_char_lm
+    call c_check_c_char_cm_as(a)
+    if(any(a/=ref_c_char_lm)) stop 122
+    a = ref_c_char_lm
+    call f_check_c_char_cm_ar(a)
+    if(any(a/=ref_c_char_lm)) stop 123
+    a = ref_c_char_lm
+    call c_check_c_char_cm_ar(a)
+    if(any(a/=ref_c_char_lm)) stop 124
+    a = ref_c_char_lm
+    call f_check_c_char_am_as(a)
+    if(any(a/=ref_c_char_lm)) stop 125
+    a = ref_c_char_lm
+    call c_check_c_char_am_as(a)
+    if(any(a/=ref_c_char_lm)) stop 126
+    a = ref_c_char_lm
+    call f_check_c_char_am_ar(a)
+    if(any(a/=ref_c_char_lm)) stop 127
+    a = ref_c_char_lm
+    call c_check_c_char_am_ar(a)
+    if(any(a/=ref_c_char_lm)) stop 128
+    a = ref_c_char_lm
+    p => a  
+    call f_check_c_char_dm_as(p)
+    if(.not.associated(p)) stop 129
+    if(.not.associated(p, a)) stop 130
+    if(any(p/=ref_c_char_lm)) stop 131
+    if(any(a/=ref_c_char_lm)) stop 132
+    a = ref_c_char_lm
+    p => a  
+    call c_check_c_char_dm_as(p)
+    if(.not.associated(p)) stop 133
+    if(.not.associated(p, a)) stop 134
+    if(any(p/=ref_c_char_lm)) stop 135
+    if(any(a/=ref_c_char_lm)) stop 136
+    a = ref_c_char_lm
+    p => a  
+    call f_check_c_char_dm_ar(p)
+    if(.not.associated(p)) stop 137
+    if(.not.associated(p, a)) stop 138
+    if(any(p/=ref_c_char_lm)) stop 139
+    if(any(a/=ref_c_char_lm)) stop 140
+    a = ref_c_char_lm
+    p => a  
+    call c_check_c_char_dm_ar(p)
+    if(.not.associated(p)) stop 141
+    if(.not.associated(p, a)) stop 142
+    if(any(p/=ref_c_char_lm)) stop 143
+    if(any(a/=ref_c_char_lm)) stop 144
+    return
+  end subroutine check_c_char_lm
+
+  subroutine f_check_c_char_cm_as(a)
+    character(kind=c_char, len=m), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 145
+    if(k/=1_c_signed_char) stop 146
+    if(n/=m) stop 147
+    if(int(k, kind=c_size_t)/=e) stop 148
+    if(t/=CFI_type_char) stop 149
+    if(any(a/=ref_c_char_lm)) stop 150
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_lm)) stop 151
+    return
+  end subroutine f_check_c_char_cm_as
+
+  subroutine c_check_c_char_cm_as(a) bind(c)
+    character(kind=c_char, len=m), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 152
+    if(k/=1_c_signed_char) stop 153
+    if(n/=m) stop 154
+    if(int(k, kind=c_size_t)/=e) stop 155
+    if(t/=CFI_type_char) stop 156
+    if(any(a/=ref_c_char_lm)) stop 157
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_lm)) stop 158
+    return
+  end subroutine c_check_c_char_cm_as
+
+  subroutine f_check_c_char_cm_ar(a)
+    character(kind=c_char, len=m), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 159
+    if(k/=1_c_signed_char) stop 160
+    if(n/=m) stop 161
+    if(int(k, kind=c_size_t)/=e) stop 162
+    if(t/=CFI_type_char) stop 163
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 164
+    rank default
+      stop 165
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 166
+    rank default
+      stop 167
+    end select
+    return
+  end subroutine f_check_c_char_cm_ar
+
+  subroutine c_check_c_char_cm_ar(a) bind(c)
+    character(kind=c_char, len=m), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 168
+    if(k/=1_c_signed_char) stop 169
+    if(n/=m) stop 170
+    if(int(k, kind=c_size_t)/=e) stop 171
+    if(t/=CFI_type_char) stop 172
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 173
+    rank default
+      stop 174
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 175
+    rank default
+      stop 176
+    end select
+    return
+  end subroutine c_check_c_char_cm_ar
+
+  subroutine f_check_c_char_am_as(a)
+    character(kind=c_char, len=*), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 177
+    if(k/=1_c_signed_char) stop 178
+    if(n/=m) stop 179
+    if(int(k, kind=c_size_t)/=e) stop 180
+    if(t/=CFI_type_char) stop 181
+    if(any(a/=ref_c_char_lm)) stop 182
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_lm)) stop 183
+    return
+  end subroutine f_check_c_char_am_as
+
+  subroutine c_check_c_char_am_as(a) bind(c)
+    character(kind=c_char, len=*), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 184
+    if(k/=1_c_signed_char) stop 185
+    if(n/=m) stop 186
+    if(int(k, kind=c_size_t)/=e) stop 187
+    if(t/=CFI_type_char) stop 188
+    if(any(a/=ref_c_char_lm)) stop 189
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_lm)) stop 190
+    return
+  end subroutine c_check_c_char_am_as
+
+  subroutine f_check_c_char_am_ar(a)
+    character(kind=c_char, len=*), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 191
+    if(k/=1_c_signed_char) stop 192
+    if(n/=m) stop 193
+    if(int(k, kind=c_size_t)/=e) stop 194
+    if(t/=CFI_type_char) stop 195
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 196
+    rank default
+      stop 197
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 198
+    rank default
+      stop 199
+    end select
+    return
+  end subroutine f_check_c_char_am_ar
+
+  subroutine c_check_c_char_am_ar(a) bind(c)
+    character(kind=c_char, len=*), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 200
+    if(k/=1_c_signed_char) stop 201
+    if(n/=m) stop 202
+    if(int(k, kind=c_size_t)/=e) stop 203
+    if(t/=CFI_type_char) stop 204
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 205
+    rank default
+      stop 206
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 207
+    rank default
+      stop 208
+    end select
+    return
+  end subroutine c_check_c_char_am_ar
+
+  subroutine f_check_c_char_dm_as(a)
+    character(kind=c_char, len=:), pointer, intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 209
+    if(k/=1_c_signed_char) stop 210
+    if(n/=m) stop 211
+    if(int(k, kind=c_size_t)/=e) stop 212
+    if(t/=CFI_type_char) stop 213
+    if(any(a/=ref_c_char_lm)) stop 214
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_lm)) stop 215
+    return
+  end subroutine f_check_c_char_dm_as
+
+  subroutine c_check_c_char_dm_as(a) bind(c)
+    character(kind=c_char, len=:), pointer, intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 216
+    if(k/=1_c_signed_char) stop 217
+    if(n/=m) stop 218
+    if(int(k, kind=c_size_t)/=e) stop 219
+    if(t/=CFI_type_char) stop 220
+    if(any(a/=ref_c_char_lm)) stop 221
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_char_lm)) stop 222
+    return
+  end subroutine c_check_c_char_dm_as
+
+  subroutine f_check_c_char_dm_ar(a)
+    character(kind=c_char, len=:), pointer, intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 223
+    if(k/=1_c_signed_char) stop 224
+    if(n/=m) stop 225
+    if(int(k, kind=c_size_t)/=e) stop 226
+    if(t/=CFI_type_char) stop 227
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 228
+    rank default
+      stop 229
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 230
+    rank default
+      stop 231
+    end select
+    return
+  end subroutine f_check_c_char_dm_ar
+
+  subroutine c_check_c_char_dm_ar(a) bind(c)
+    character(kind=c_char, len=:), pointer, intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 232
+    if(k/=1_c_signed_char) stop 233
+    if(n/=m) stop 234
+    if(int(k, kind=c_size_t)/=e) stop 235
+    if(t/=CFI_type_char) stop 236
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 237
+    rank default
+      stop 238
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_char_lm)) stop 239
+    rank default
+      stop 240
+    end select
+    return
+  end subroutine c_check_c_char_dm_ar
+
+  subroutine check_c_ucs4_char_l1()
+    character(kind=c_ucs4_char, len=1), target :: a(n)
+    !
+    character(kind=c_ucs4_char, len=:), pointer :: p(:)
+    !
+    a = ref_c_ucs4_char_l1
+    call f_check_c_ucs4_char_c1_as(a)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 241
+    a = ref_c_ucs4_char_l1
+    call c_check_c_ucs4_char_c1_as(a)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 242
+    a = ref_c_ucs4_char_l1
+    call f_check_c_ucs4_char_c1_ar(a)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 243
+    a = ref_c_ucs4_char_l1
+    call c_check_c_ucs4_char_c1_ar(a)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 244
+    a = ref_c_ucs4_char_l1
+    call f_check_c_ucs4_char_a1_as(a)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 245
+    a = ref_c_ucs4_char_l1
+    call c_check_c_ucs4_char_a1_as(a)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 246
+    a = ref_c_ucs4_char_l1
+    call f_check_c_ucs4_char_a1_ar(a)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 247
+    a = ref_c_ucs4_char_l1
+    call c_check_c_ucs4_char_a1_ar(a)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 248
+    a = ref_c_ucs4_char_l1
+    p => a  
+    call f_check_c_ucs4_char_d1_as(p)
+    if(.not.associated(p)) stop 249
+    if(.not.associated(p, a)) stop 250
+    if(any(p/=ref_c_ucs4_char_l1)) stop 251
+    if(any(a/=ref_c_ucs4_char_l1)) stop 252
+    a = ref_c_ucs4_char_l1
+    p => a  
+    call c_check_c_ucs4_char_d1_as(p)
+    if(.not.associated(p)) stop 253
+    if(.not.associated(p, a)) stop 254
+    if(any(p/=ref_c_ucs4_char_l1)) stop 255
+    if(any(a/=ref_c_ucs4_char_l1)) stop 256
+    a = ref_c_ucs4_char_l1
+    p => a  
+    call f_check_c_ucs4_char_d1_ar(p)
+    if(.not.associated(p)) stop 257
+    if(.not.associated(p, a)) stop 258
+    if(any(p/=ref_c_ucs4_char_l1)) stop 259
+    if(any(a/=ref_c_ucs4_char_l1)) stop 260
+    a = ref_c_ucs4_char_l1
+    p => a  
+    call c_check_c_ucs4_char_d1_ar(p)
+    if(.not.associated(p)) stop 261
+    if(.not.associated(p, a)) stop 262
+    if(any(p/=ref_c_ucs4_char_l1)) stop 263
+    if(any(a/=ref_c_ucs4_char_l1)) stop 264
+    return
+  end subroutine check_c_ucs4_char_l1
+
+  subroutine f_check_c_ucs4_char_c1_as(a)
+    character(kind=c_ucs4_char, len=1), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 265
+    if(k/=4_c_signed_char) stop 266
+    if(n/=1) stop 267
+    if(int(k, kind=c_size_t)/=e) stop 268
+    if(t/=CFI_type_ucs4_char) stop 269
+    if(any(a/=ref_c_ucs4_char_l1)) stop 270
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 271
+    return
+  end subroutine f_check_c_ucs4_char_c1_as
+
+  subroutine c_check_c_ucs4_char_c1_as(a) bind(c)
+    character(kind=c_ucs4_char, len=1), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 272
+    if(k/=4_c_signed_char) stop 273
+    if(n/=1) stop 274
+    if(int(k, kind=c_size_t)/=e) stop 275
+    if(t/=CFI_type_ucs4_char) stop 276
+    if(any(a/=ref_c_ucs4_char_l1)) stop 277
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 278
+    return
+  end subroutine c_check_c_ucs4_char_c1_as
+
+  subroutine f_check_c_ucs4_char_c1_ar(a)
+    character(kind=c_ucs4_char, len=1), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 279
+    if(k/=4_c_signed_char) stop 280
+    if(n/=1) stop 281
+    if(int(k, kind=c_size_t)/=e) stop 282
+    if(t/=CFI_type_ucs4_char) stop 283
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 284
+    rank default
+      stop 285
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 286
+    rank default
+      stop 287
+    end select
+    return
+  end subroutine f_check_c_ucs4_char_c1_ar
+
+  subroutine c_check_c_ucs4_char_c1_ar(a) bind(c)
+    character(kind=c_ucs4_char, len=1), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 288
+    if(k/=4_c_signed_char) stop 289
+    if(n/=1) stop 290
+    if(int(k, kind=c_size_t)/=e) stop 291
+    if(t/=CFI_type_ucs4_char) stop 292
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 293
+    rank default
+      stop 294
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 295
+    rank default
+      stop 296
+    end select
+    return
+  end subroutine c_check_c_ucs4_char_c1_ar
+
+  subroutine f_check_c_ucs4_char_a1_as(a)
+    character(kind=c_ucs4_char, len=*), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 297
+    if(k/=4_c_signed_char) stop 298
+    if(n/=1) stop 299
+    if(int(k, kind=c_size_t)/=e) stop 300
+    if(t/=CFI_type_ucs4_char) stop 301
+    if(any(a/=ref_c_ucs4_char_l1)) stop 302
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 303
+    return
+  end subroutine f_check_c_ucs4_char_a1_as
+
+  subroutine c_check_c_ucs4_char_a1_as(a) bind(c)
+    character(kind=c_ucs4_char, len=*), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 304
+    if(k/=4_c_signed_char) stop 305
+    if(n/=1) stop 306
+    if(int(k, kind=c_size_t)/=e) stop 307
+    if(t/=CFI_type_ucs4_char) stop 308
+    if(any(a/=ref_c_ucs4_char_l1)) stop 309
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 310
+    return
+  end subroutine c_check_c_ucs4_char_a1_as
+
+  subroutine f_check_c_ucs4_char_a1_ar(a)
+    character(kind=c_ucs4_char, len=*), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 311
+    if(k/=4_c_signed_char) stop 312
+    if(n/=1) stop 313
+    if(int(k, kind=c_size_t)/=e) stop 314
+    if(t/=CFI_type_ucs4_char) stop 315
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 316
+    rank default
+      stop 317
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 318
+    rank default
+      stop 319
+    end select
+    return
+  end subroutine f_check_c_ucs4_char_a1_ar
+
+  subroutine c_check_c_ucs4_char_a1_ar(a) bind(c)
+    character(kind=c_ucs4_char, len=*), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 320
+    if(k/=4_c_signed_char) stop 321
+    if(n/=1) stop 322
+    if(int(k, kind=c_size_t)/=e) stop 323
+    if(t/=CFI_type_ucs4_char) stop 324
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 325
+    rank default
+      stop 326
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 327
+    rank default
+      stop 328
+    end select
+    return
+  end subroutine c_check_c_ucs4_char_a1_ar
+
+  subroutine f_check_c_ucs4_char_d1_as(a)
+    character(kind=c_ucs4_char, len=:), pointer, intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 329
+    if(k/=4_c_signed_char) stop 330
+    if(n/=1) stop 331
+    if(int(k, kind=c_size_t)/=e) stop 332
+    if(t/=CFI_type_ucs4_char) stop 333
+    if(any(a/=ref_c_ucs4_char_l1)) stop 334
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 335
+    return
+  end subroutine f_check_c_ucs4_char_d1_as
+
+  subroutine c_check_c_ucs4_char_d1_as(a) bind(c)
+    character(kind=c_ucs4_char, len=:), pointer, intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 336
+    if(k/=4_c_signed_char) stop 337
+    if(n/=1) stop 338
+    if(int(k, kind=c_size_t)/=e) stop 339
+    if(t/=CFI_type_ucs4_char) stop 340
+    if(any(a/=ref_c_ucs4_char_l1)) stop 341
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_l1)) stop 342
+    return
+  end subroutine c_check_c_ucs4_char_d1_as
+
+  subroutine f_check_c_ucs4_char_d1_ar(a)
+    character(kind=c_ucs4_char, len=:), pointer, intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 343
+    if(k/=4_c_signed_char) stop 344
+    if(n/=1) stop 345
+    if(int(k, kind=c_size_t)/=e) stop 346
+    if(t/=CFI_type_ucs4_char) stop 347
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 348
+    rank default
+      stop 349
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 350
+    rank default
+      stop 351
+    end select
+    return
+  end subroutine f_check_c_ucs4_char_d1_ar
+
+  subroutine c_check_c_ucs4_char_d1_ar(a) bind(c)
+    character(kind=c_ucs4_char, len=:), pointer, intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*1)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 352
+    if(k/=4_c_signed_char) stop 353
+    if(n/=1) stop 354
+    if(int(k, kind=c_size_t)/=e) stop 355
+    if(t/=CFI_type_ucs4_char) stop 356
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 357
+    rank default
+      stop 358
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_l1)) stop 359
+    rank default
+      stop 360
+    end select
+    return
+  end subroutine c_check_c_ucs4_char_d1_ar
+
+  subroutine check_c_ucs4_char_lm()
+    character(kind=c_ucs4_char, len=m), target :: a(n)
+    !
+    character(kind=c_ucs4_char, len=:), pointer :: p(:)
+    !
+    a = ref_c_ucs4_char_lm
+    call f_check_c_ucs4_char_cm_as(a)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 361
+    a = ref_c_ucs4_char_lm
+    call c_check_c_ucs4_char_cm_as(a)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 362
+    a = ref_c_ucs4_char_lm
+    call f_check_c_ucs4_char_cm_ar(a)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 363
+    a = ref_c_ucs4_char_lm
+    call c_check_c_ucs4_char_cm_ar(a)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 364
+    a = ref_c_ucs4_char_lm
+    call f_check_c_ucs4_char_am_as(a)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 365
+    a = ref_c_ucs4_char_lm
+    call c_check_c_ucs4_char_am_as(a)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 366
+    a = ref_c_ucs4_char_lm
+    call f_check_c_ucs4_char_am_ar(a)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 367
+    a = ref_c_ucs4_char_lm
+    call c_check_c_ucs4_char_am_ar(a)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 368
+    a = ref_c_ucs4_char_lm
+    p => a  
+    call f_check_c_ucs4_char_dm_as(p)
+    if(.not.associated(p)) stop 369
+    if(.not.associated(p, a)) stop 370
+    if(any(p/=ref_c_ucs4_char_lm)) stop 371
+    if(any(a/=ref_c_ucs4_char_lm)) stop 372
+    a = ref_c_ucs4_char_lm
+    p => a  
+    call c_check_c_ucs4_char_dm_as(p)
+    if(.not.associated(p)) stop 373
+    if(.not.associated(p, a)) stop 374
+    if(any(p/=ref_c_ucs4_char_lm)) stop 375
+    if(any(a/=ref_c_ucs4_char_lm)) stop 376
+    a = ref_c_ucs4_char_lm
+    p => a  
+    call f_check_c_ucs4_char_dm_ar(p)
+    if(.not.associated(p)) stop 377
+    if(.not.associated(p, a)) stop 378
+    if(any(p/=ref_c_ucs4_char_lm)) stop 379
+    if(any(a/=ref_c_ucs4_char_lm)) stop 380
+    a = ref_c_ucs4_char_lm
+    p => a  
+    call c_check_c_ucs4_char_dm_ar(p)
+    if(.not.associated(p)) stop 381
+    if(.not.associated(p, a)) stop 382
+    if(any(p/=ref_c_ucs4_char_lm)) stop 383
+    if(any(a/=ref_c_ucs4_char_lm)) stop 384
+    return
+  end subroutine check_c_ucs4_char_lm
+
+  subroutine f_check_c_ucs4_char_cm_as(a)
+    character(kind=c_ucs4_char, len=m), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 385
+    if(k/=4_c_signed_char) stop 386
+    if(n/=m) stop 387
+    if(int(k, kind=c_size_t)/=e) stop 388
+    if(t/=CFI_type_ucs4_char) stop 389
+    if(any(a/=ref_c_ucs4_char_lm)) stop 390
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 391
+    return
+  end subroutine f_check_c_ucs4_char_cm_as
+
+  subroutine c_check_c_ucs4_char_cm_as(a) bind(c)
+    character(kind=c_ucs4_char, len=m), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 392
+    if(k/=4_c_signed_char) stop 393
+    if(n/=m) stop 394
+    if(int(k, kind=c_size_t)/=e) stop 395
+    if(t/=CFI_type_ucs4_char) stop 396
+    if(any(a/=ref_c_ucs4_char_lm)) stop 397
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 398
+    return
+  end subroutine c_check_c_ucs4_char_cm_as
+
+  subroutine f_check_c_ucs4_char_cm_ar(a)
+    character(kind=c_ucs4_char, len=m), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 399
+    if(k/=4_c_signed_char) stop 400
+    if(n/=m) stop 401
+    if(int(k, kind=c_size_t)/=e) stop 402
+    if(t/=CFI_type_ucs4_char) stop 403
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 404
+    rank default
+      stop 405
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 406
+    rank default
+      stop 407
+    end select
+    return
+  end subroutine f_check_c_ucs4_char_cm_ar
+
+  subroutine c_check_c_ucs4_char_cm_ar(a) bind(c)
+    character(kind=c_ucs4_char, len=m), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 408
+    if(k/=4_c_signed_char) stop 409
+    if(n/=m) stop 410
+    if(int(k, kind=c_size_t)/=e) stop 411
+    if(t/=CFI_type_ucs4_char) stop 412
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 413
+    rank default
+      stop 414
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 415
+    rank default
+      stop 416
+    end select
+    return
+  end subroutine c_check_c_ucs4_char_cm_ar
+
+  subroutine f_check_c_ucs4_char_am_as(a)
+    character(kind=c_ucs4_char, len=*), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 417
+    if(k/=4_c_signed_char) stop 418
+    if(n/=m) stop 419
+    if(int(k, kind=c_size_t)/=e) stop 420
+    if(t/=CFI_type_ucs4_char) stop 421
+    if(any(a/=ref_c_ucs4_char_lm)) stop 422
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 423
+    return
+  end subroutine f_check_c_ucs4_char_am_as
+
+  subroutine c_check_c_ucs4_char_am_as(a) bind(c)
+    character(kind=c_ucs4_char, len=*), intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 424
+    if(k/=4_c_signed_char) stop 425
+    if(n/=m) stop 426
+    if(int(k, kind=c_size_t)/=e) stop 427
+    if(t/=CFI_type_ucs4_char) stop 428
+    if(any(a/=ref_c_ucs4_char_lm)) stop 429
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 430
+    return
+  end subroutine c_check_c_ucs4_char_am_as
+
+  subroutine f_check_c_ucs4_char_am_ar(a)
+    character(kind=c_ucs4_char, len=*), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 431
+    if(k/=4_c_signed_char) stop 432
+    if(n/=m) stop 433
+    if(int(k, kind=c_size_t)/=e) stop 434
+    if(t/=CFI_type_ucs4_char) stop 435
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 436
+    rank default
+      stop 437
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 438
+    rank default
+      stop 439
+    end select
+    return
+  end subroutine f_check_c_ucs4_char_am_ar
+
+  subroutine c_check_c_ucs4_char_am_ar(a) bind(c)
+    character(kind=c_ucs4_char, len=*), intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 440
+    if(k/=4_c_signed_char) stop 441
+    if(n/=m) stop 442
+    if(int(k, kind=c_size_t)/=e) stop 443
+    if(t/=CFI_type_ucs4_char) stop 444
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 445
+    rank default
+      stop 446
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 447
+    rank default
+      stop 448
+    end select
+    return
+  end subroutine c_check_c_ucs4_char_am_ar
+
+  subroutine f_check_c_ucs4_char_dm_as(a)
+    character(kind=c_ucs4_char, len=:), pointer, intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 449
+    if(k/=4_c_signed_char) stop 450
+    if(n/=m) stop 451
+    if(int(k, kind=c_size_t)/=e) stop 452
+    if(t/=CFI_type_ucs4_char) stop 453
+    if(any(a/=ref_c_ucs4_char_lm)) stop 454
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 455
+    return
+  end subroutine f_check_c_ucs4_char_dm_as
+
+  subroutine c_check_c_ucs4_char_dm_as(a) bind(c)
+    character(kind=c_ucs4_char, len=:), pointer, intent(in) :: a(:)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 456
+    if(k/=4_c_signed_char) stop 457
+    if(n/=m) stop 458
+    if(int(k, kind=c_size_t)/=e) stop 459
+    if(t/=CFI_type_ucs4_char) stop 460
+    if(any(a/=ref_c_ucs4_char_lm)) stop 461
+    call check_tk_as(a, t, k, e, n)
+    if(any(a/=ref_c_ucs4_char_lm)) stop 462
+    return
+  end subroutine c_check_c_ucs4_char_dm_as
+
+  subroutine f_check_c_ucs4_char_dm_ar(a)
+    character(kind=c_ucs4_char, len=:), pointer, intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 463
+    if(k/=4_c_signed_char) stop 464
+    if(n/=m) stop 465
+    if(int(k, kind=c_size_t)/=e) stop 466
+    if(t/=CFI_type_ucs4_char) stop 467
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 468
+    rank default
+      stop 469
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 470
+    rank default
+      stop 471
+    end select
+    return
+  end subroutine f_check_c_ucs4_char_dm_ar
+
+  subroutine c_check_c_ucs4_char_dm_ar(a) bind(c)
+    character(kind=c_ucs4_char, len=:), pointer, intent(in) :: a(..)
+    !
+    integer(kind=c_int16_t)     :: t
+    integer(kind=c_signed_char) :: k
+    integer(kind=c_size_t)      :: e, n
+    !
+    k = kind(a)
+    n = len(a, kind=kind(e))
+    e = storage_size(a, kind=kind(e))/(b*m)
+    t = cfi_encode_type(CFI_type_Character, k)
+    if(k<=0_c_signed_char) stop 472
+    if(k/=4_c_signed_char) stop 473
+    if(n/=m) stop 474
+    if(int(k, kind=c_size_t)/=e) stop 475
+    if(t/=CFI_type_ucs4_char) stop 476
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 477
+    rank default
+      stop 478
+    end select
+    call check_tk_ar(a, t, k, e, n)
+    select rank(a)
+    rank(1)
+      if(any(a/=ref_c_ucs4_char_lm)) stop 479
+    rank default
+      stop 480
+    end select
+    return
+  end subroutine c_check_c_ucs4_char_dm_ar
+  
+end module iso_check_m
+
+program main_p
+  
+  use :: iso_check_m, only: &
+    check_c_char_l1,        &
+    check_c_char_lm,        &
+    check_c_ucs4_char_l1,   &
+    check_c_ucs4_char_lm
+
+  implicit none
+
+  call check_c_char_l1()
+  call check_c_char_lm()
+  ! See PR100907
+  !call check_c_ucs4_char_l1()
+  !call check_c_ucs4_char_lm()
+  stop
+
+end program main_p
+
+!! Local Variables:
+!! mode: f90
+!! End:
+
diff --git a/gcc/testsuite/gfortran.dg/PR93963.f90 b/gcc/testsuite/gfortran.dg/PR93963.f90
index 66c9379..6769d7f 100644
--- a/gcc/testsuite/gfortran.dg/PR93963.f90
+++ b/gcc/testsuite/gfortran.dg/PR93963.f90
@@ -1,4 +1,5 @@
 ! { dg-do run }
+! { dg-additional-options "-fdump-tree-original" }
 !
 ! Test the fix for PR93963
 !
@@ -190,3 +191,7 @@ program selr_p
   deallocate(inta)
 
 end program selr_p
+
+! Special code for assumed rank - but only if not allocatable/pointer
+! Thus, expect it only once for subroutine rank_o but not for rank_a or rank_p
+! { dg-final { scan-tree-dump-times "ubound != -1" 1 "original" } }
diff --git a/gcc/testsuite/gfortran.dg/PR94289.f90 b/gcc/testsuite/gfortran.dg/PR94289.f90
new file mode 100644
index 0000000..4f17d97
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/PR94289.f90
@@ -0,0 +1,168 @@
+! { dg-do run }
+!
+! Testcase for PR 94289
+!
+! - if the dummy argument is a pointer/allocatable, it has the same 
+!   bounds as the dummy argument
+! - if is is nonallocatable nonpointer, the lower bounds are [1, 1, 1].
+
+module bounds_m
+
+  implicit none
+
+  private
+  public :: &
+    lb, ub
+
+  public :: &
+    bnds_p, &
+    bnds_a, &
+    bnds_e
+
+  integer, parameter :: lb1 = 3
+  integer, parameter :: lb2 = 5
+  integer, parameter :: lb3 = 9
+  integer, parameter :: ub1 = 4
+  integer, parameter :: ub2 = 50
+  integer, parameter :: ub3 = 11
+  integer, parameter :: ex1 = ub1 - lb1 + 1
+  integer, parameter :: ex2 = ub2 - lb2 + 1
+  integer, parameter :: ex3 = ub3 - lb3 + 1
+
+  integer, parameter :: lf(*) = [1,1,1]
+  integer, parameter :: lb(*) = [lb1,lb2,lb3]
+  integer, parameter :: ub(*) = [ub1,ub2,ub3]
+  integer, parameter :: ex(*) = [ex1,ex2,ex3]
+
+contains
+
+  subroutine bounds(a, lb, ub)
+    integer, pointer, intent(in) :: a(..)
+    integer,          intent(in) :: lb(3)
+    integer,          intent(in) :: ub(3)
+
+    integer :: ex(3)
+
+    ex = max(ub-lb+1, 0)
+    if(any(lbound(a)/=lb)) stop 101
+    if(any(ubound(a)/=ub)) stop 102
+    if(any( shape(a)/=ex)) stop 103
+    return
+  end subroutine bounds
+
+  subroutine bnds_p(this)
+    integer, pointer, intent(in) :: this(..)
+
+    if(any(lbound(this)/=lb)) stop 1
+    if(any(ubound(this)/=ub)) stop 2
+    if(any( shape(this)/=ex)) stop 3
+    call bounds(this, lb, ub)
+    return
+  end subroutine bnds_p
+  
+  subroutine bnds_a(this)
+    integer, allocatable, target, intent(in) :: this(..)
+    
+    if(any(lbound(this)/=lb)) stop 4
+    if(any(ubound(this)/=ub)) stop 5
+    if(any( shape(this)/=ex)) stop 6
+    call bounds(this, lb, ub)
+    return
+  end subroutine bnds_a
+  
+  subroutine bnds_e(this)
+    integer, target, intent(in) :: this(..)
+    
+    if(any(lbound(this)/=lf)) stop 7
+    if(any(ubound(this)/=ex)) stop 8
+    if(any( shape(this)/=ex)) stop 9
+    call bounds(this, lf, ex)
+    return
+  end subroutine bnds_e
+  
+end module bounds_m
+
+program bounds_p
+
+  use, intrinsic :: iso_c_binding, only: c_int
+  
+  use bounds_m
+  
+  implicit none
+
+  integer, parameter :: fpn = 1
+  integer, parameter :: fan = 2
+  integer, parameter :: fon = 3
+
+  integer :: i
+  
+  do i = fpn, fon
+    call test_p(i)
+  end do
+  do i = fpn, fon
+    call test_a(i)
+  end do
+  do i = fpn, fon
+    call test_e(i)
+  end do
+  stop
+
+contains
+
+  subroutine test_p(t)
+    integer, intent(in) :: t
+    
+    integer, pointer :: a(:,:,:)
+
+    allocate(a(lb(1):ub(1),lb(2):ub(2),lb(3):ub(3)))
+    select case(t)
+    case(fpn)
+      call bnds_p(a)
+    case(fan)
+    case(fon)
+      call bnds_e(a)
+    case default
+      stop
+    end select
+    deallocate(a)
+    return
+  end subroutine test_p
+
+  subroutine test_a(t)
+    integer, intent(in) :: t
+    
+    integer, allocatable, target :: a(:,:,:)
+
+    allocate(a(lb(1):ub(1),lb(2):ub(2),lb(3):ub(3)))
+    select case(t)
+    case(fpn)
+      call bnds_p(a)
+    case(fan)
+      call bnds_a(a)
+    case(fon)
+      call bnds_e(a)
+    case default
+      stop
+    end select
+    deallocate(a)
+    return
+  end subroutine test_a
+
+  subroutine test_e(t)
+    integer, intent(in) :: t
+    
+    integer, target :: a(lb(1):ub(1),lb(2):ub(2),lb(3):ub(3))
+
+    select case(t)
+    case(fpn)
+      call bnds_p(a)
+    case(fan)
+    case(fon)
+      call bnds_e(a)
+    case default
+      stop
+    end select
+    return
+  end subroutine test_e
+
+end program bounds_p
diff --git a/gcc/testsuite/gfortran.dg/PR95196.f90 b/gcc/testsuite/gfortran.dg/PR95196.f90
new file mode 100644
index 0000000..14333e4
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/PR95196.f90
@@ -0,0 +1,83 @@
+! { dg-do run }
+
+program rnk_p
+
+  implicit none
+
+  integer, parameter :: n = 10
+  integer, parameter :: m = 5
+  integer, parameter :: s = 4
+  integer, parameter :: l = 4
+  integer, parameter :: u = s+l-1
+  
+  integer :: a(n)
+  integer :: b(n,n)
+  integer :: c(n,n,n)
+  integer :: r(s*s*s)
+  integer :: i
+
+  a = reshape([(i, i=1,n)], [n])
+  b = reshape([(i, i=1,n*n)], [n,n])
+  c = reshape([(i, i=1,n*n*n)], [n,n,n])
+  r(1:s) = a(l:u)
+  call rnk_s(a(l:u), r(1:s))
+  r(1:s*s) = reshape(b(l:u,l:u), [s*s])
+  call rnk_s(b(l:u,l:u), r(1:s*s))
+  r = reshape(c(l:u,l:u,l:u), [s*s*s])
+  call rnk_s(c(l:u,l:7,l:u), r)
+  stop
+  
+contains
+
+  subroutine rnk_s(a, b)
+    integer, intent(in) :: a(..)
+    integer, intent(in) :: b(:)
+    
+    !integer :: l(rank(a)), u(rank(a)) does not work due to Bug 94048 
+    integer, allocatable :: lb(:), ub(:)
+    integer              :: i, j, k, l
+
+    lb = lbound(a)
+    ub = ubound(a)
+    select rank(a)
+    rank(1)
+      if(any(lb/=lbound(a))) stop 11
+      if(any(ub/=ubound(a))) stop 12
+      if(size(a)/=size(b))   stop 13
+      do i = 1, size(a)
+        if(a(i)/=b(i)) stop 14
+      end do
+    rank(2)
+      if(any(lb/=lbound(a))) stop 21
+      if(any(ub/=ubound(a))) stop 22
+      if(size(a)/=size(b))   stop 23
+      k = 0
+      do j = 1, size(a, dim=2)
+        do i = 1, size(a, dim=1)
+          k = k + 1
+          if(a(i,j)/=b(k)) stop 24
+        end do
+      end do
+    rank(3)
+      if(any(lb/=lbound(a))) stop 31
+      if(any(ub/=ubound(a))) stop 32
+      if(size(a)/=size(b))   stop 33
+      l = 0
+      do k = 1, size(a, dim=3)
+        do j = 1, size(a, dim=2)
+          do i = 1, size(a, dim=1)
+            l = l + 1
+            ! print *, a(i,j,k), b(l)
+            if(a(i,j,k)/=b(l)) stop 34
+          end do
+        end do
+      end do
+    rank default
+      stop 171
+    end select
+    deallocate(lb, ub)
+    return
+  end subroutine rnk_s
+  
+end program rnk_p
+
diff --git a/gcc/testsuite/gfortran.dg/associate_3.f03 b/gcc/testsuite/gfortran.dg/associate_3.f03
index da7bec9..dfd5a99 100644
--- a/gcc/testsuite/gfortran.dg/associate_3.f03
+++ b/gcc/testsuite/gfortran.dg/associate_3.f03
@@ -34,4 +34,4 @@ PROGRAM main
     INTEGER :: b ! { dg-error "Unexpected data declaration statement" }
   END ASSOCIATE
 END PROGRAM main ! { dg-error "Expecting END ASSOCIATE" }
-! { dg-excess-errors "Unexpected end of file" }
+! { dg-error "Unexpected end of file" "" { target "*-*-*" } 0 }
diff --git a/gcc/testsuite/gfortran.dg/bessel_3.f90 b/gcc/testsuite/gfortran.dg/bessel_3.f90
index 8817725..51e11e9 100644
--- a/gcc/testsuite/gfortran.dg/bessel_3.f90
+++ b/gcc/testsuite/gfortran.dg/bessel_3.f90
@@ -9,10 +9,10 @@ print *, SIN (1.0)
 print *, BESSEL_J0(1.0) ! { dg-error "has no IMPLICIT type" })
 print *, BESSEL_J1(1.0) ! { dg-error "has no IMPLICIT type" }
 print *, BESSEL_JN(1,1.0) ! { dg-error "has no IMPLICIT type|Type mismatch" }
-print *, BESSEL_JN(1,2,1.0) ! { dg-error "has no IMPLICIT type|Type mismatch" }
+print *, BESSEL_JN(1,2,1.0) ! { dg-error "has no IMPLICIT type|Type mismatch|More actual than formal" }
 
 print *, BESSEL_Y0(1.0) ! { dg-error "has no IMPLICIT type" }
 print *, BESSEL_Y1(1.0) ! { dg-error "has no IMPLICIT type" }
 print *, BESSEL_YN(1,1.0) ! { dg-error "has no IMPLICIT type|Type mismatch" }
-print *, BESSEL_YN(1,2,1.0) ! { dg-error "has no IMPLICIT type|Type mismatch" }
+print *, BESSEL_YN(1,2,1.0) ! { dg-error "has no IMPLICIT type|Type mismatch|More actual than formal" }
 end
diff --git a/gcc/testsuite/gfortran.dg/binding_label_tests_4.f03 b/gcc/testsuite/gfortran.dg/binding_label_tests_4.f03
index f8c0f04..af9a588 100644
--- a/gcc/testsuite/gfortran.dg/binding_label_tests_4.f03
+++ b/gcc/testsuite/gfortran.dg/binding_label_tests_4.f03
@@ -20,4 +20,4 @@ module C
 use A
 use B ! { dg-error "Cannot open module file" }
 end module C
-! { dg-excess-errors "compilation terminated" }
+! { dg-prune-output "compilation terminated" }
diff --git a/gcc/testsuite/gfortran.dg/block_4.f08 b/gcc/testsuite/gfortran.dg/block_4.f08
index 4c63194..3ff52b0 100644
--- a/gcc/testsuite/gfortran.dg/block_4.f08
+++ b/gcc/testsuite/gfortran.dg/block_4.f08
@@ -15,4 +15,4 @@ PROGRAM main
   myname2: BLOCK
   END BLOCK ! { dg-error "Expected block name of 'myname2'" }
 END PROGRAM main ! { dg-error "Expecting END BLOCK" }
-! { dg-excess-errors "Unexpected end of file" }
+! { dg-error "Unexpected end of file" "" { target "*-*-*" } 0 }
diff --git a/gcc/testsuite/gfortran.dg/c-interop/c407b-2.f90 b/gcc/testsuite/gfortran.dg/c-interop/c407b-2.f90
index 3d3cd63..4f9f6c7 100644
--- a/gcc/testsuite/gfortran.dg/c-interop/c407b-2.f90
+++ b/gcc/testsuite/gfortran.dg/c-interop/c407b-2.f90
@@ -78,11 +78,11 @@ subroutine s2 (x, y)
   end select
 
   ! relational operations
-  if (x & ! { dg-error "Assumed.type" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+  if (x & ! { dg-error "Assumed.type" "pr101337" }
       .eq. y) then  ! { dg-error "Assumed.type" } 
     return
   end if
-  if (.not. (x & ! { dg-error "Assumed.type" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+  if (.not. (x & ! { dg-error "Assumed.type" "pr101337" }
              .ne. y)) then  ! { dg-error "Assumed.type" } 
     return
   end if
@@ -99,7 +99,7 @@ subroutine s2 (x, y)
   ! arithmetic
   i = x + 1  ! { dg-error "Assumed.type" } 
   i = -y  ! { dg-error "Assumed.type" } 
-  i = (x & ! { dg-error "Assumed.type" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+  i = (x & ! { dg-error "Assumed.type" "pr101337" }
        + y)  ! { dg-error "Assumed.type" } 
 
   ! computed go to
@@ -131,19 +131,19 @@ subroutine s3 (x, y)
   i = exponent (x)  ! { dg-error "Assumed.type" }
 
   if (extends_type_of (x, &  ! { dg-error "Assumed.type" }
-                       y)) then  ! { dg-error "Assumed.type" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+                       y)) then  ! { dg-error "Assumed.type" "pr101337" }
     return
   end if
 
   if (same_type_as (x, &  ! { dg-error "Assumed.type" }
-                    y)) then  ! { dg-error "Assumed.type" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+                    y)) then  ! { dg-error "Assumed.type" "pr101337" }
     return
   end if
 
   i = storage_size (x)  ! { dg-error "Assumed.type" }
 
   i = iand (x, &  ! { dg-error "Assumed.type" }
-            y)    ! { dg-error "Assumed.type" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+            y)    ! { dg-error "Assumed.type" "pr101337" }
 
   i = kind (x)  ! { dg-error "Assumed.type" }
 
diff --git a/gcc/testsuite/gfortran.dg/c-interop/c535b-2.f90 b/gcc/testsuite/gfortran.dg/c-interop/c535b-2.f90
index 2dafd44..4d99f7f 100644
--- a/gcc/testsuite/gfortran.dg/c-interop/c535b-2.f90
+++ b/gcc/testsuite/gfortran.dg/c-interop/c535b-2.f90
@@ -57,18 +57,18 @@ subroutine test_calls (x, y)
   ! Make sure each invalid argument produces a diagnostic.
   ! scalar dummies
   call f (x, &  ! { dg-error "(A|a)ssumed.rank" }
-          y)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+          y)  ! { dg-error "(A|a)ssumed.rank" "pr101337" } 
   ! assumed-rank dummies
   call g (x, y)  ! OK
   ! assumed-size dummies
   call h (x, &  ! { dg-error "(A|a)ssumed.rank" "pr101334" }
-          y)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+          y)  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
   ! assumed-shape dummies
   call i (x, &  ! { dg-error "(A|a)ssumed.rank" }
-          y)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+          y)  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
  ! fixed-size array dummies
   call j (x, &  ! { dg-error "(A|a)ssumed.rank" "pr101334" }
-          y)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+          y)  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
 end subroutine
 
 ! Check that you can't use an assumed-rank array variable in an array
@@ -81,7 +81,7 @@ subroutine test_designators (x)
 
   call f (x(1), 1)  ! { dg-error "(A|a)ssumed.rank" }
   call g (x(1:3:1), &  ! { dg-error "(A|a)ssumed.rank" }
-          x)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+          x)
 end subroutine
 
 ! Check that you can't use an assumed-rank array variable in elemental
@@ -122,7 +122,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   z = x + y  ! OK
   c &  ! { dg-error "(A|a)ssumed.rank" }
-    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     + b  ! { dg-error "(A|a)ssumed.rank" }
   z = x + i  ! OK
   c &  ! { dg-error "(A|a)ssumed.rank" }
@@ -133,7 +133,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   z = x - y  ! OK
   c &  ! { dg-error "(A|a)ssumed.rank" }
-    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     - b  ! { dg-error "(A|a)ssumed.rank" }
   z = x - i  ! OK
   c &  ! { dg-error "(A|a)ssumed.rank" }
@@ -144,7 +144,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   z = x * y  ! OK
   c &  ! { dg-error "(A|a)ssumed.rank" }
-    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     * b  ! { dg-error "(A|a)ssumed.rank" }
   z = x * i  ! OK
   c &  ! { dg-error "(A|a)ssumed.rank" }
@@ -155,7 +155,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   z = x / y  ! OK
   c &  ! { dg-error "(A|a)ssumed.rank" }
-    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     / b  ! { dg-error "(A|a)ssumed.rank" }
   z = x / i  ! OK
   c &  ! { dg-error "(A|a)ssumed.rank" }
@@ -166,7 +166,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   z = x ** y  ! OK
   c &  ! { dg-error "(A|a)ssumed.rank" }
-    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     ** b  ! { dg-error "(A|a)ssumed.rank" }
   z = x ** i  ! OK
   c &  ! { dg-error "(A|a)ssumed.rank" }
@@ -179,7 +179,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   r = x .eq. y  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
-    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     .eq. b  ! { dg-error "(A|a)ssumed.rank" }
   r = x .eq. i  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
@@ -190,7 +190,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   r = x .ne. y  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
-    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     .ne. b  ! { dg-error "(A|a)ssumed.rank" }
   r = x .ne. i  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
@@ -201,7 +201,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   r = x .lt. y  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
-    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     .lt. b  ! { dg-error "(A|a)ssumed.rank" }
   r = x .lt. i  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
@@ -212,7 +212,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   r = x .le. y  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
-    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     .le. b  ! { dg-error "(A|a)ssumed.rank" }
   r = x .le. i  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
@@ -223,7 +223,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   r = x .gt. y  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
-    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     .gt. b  ! { dg-error "(A|a)ssumed.rank" }
   r = x .gt. i  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
@@ -234,7 +234,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   r = x .ge. y  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
-    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = a &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     .ge. b  ! { dg-error "(A|a)ssumed.rank" }
   r = x .ge. i  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
@@ -253,7 +253,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   r = p .and. q  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
-    = l &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = l &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     .and. m  ! { dg-error "(A|a)ssumed.rank" }
   r = p .and. j  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
@@ -264,7 +264,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   r = p .or. q  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
-    = l &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = l &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     .or. m  ! { dg-error "(A|a)ssumed.rank" }
   r = p .or. j  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
@@ -275,7 +275,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   r = p .eqv. q  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
-    = l &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = l &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     .eqv. m  ! { dg-error "(A|a)ssumed.rank" }
   r = p .eqv. j  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
@@ -286,7 +286,7 @@ subroutine test_expressions (a, b, c, l, m, n, x, y, z, p, q, r, s, i, j)
 
   r = p .neqv. q  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
-    = l &  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+    = l &  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
     .neqv. m  ! { dg-error "(A|a)ssumed.rank" }
   r = p .neqv. j  ! OK
   n &  ! { dg-error "(A|a)ssumed.rank" }
@@ -320,7 +320,7 @@ subroutine test_intrinsics (i1, i2, r1, r2, c1, c2, l1, l2, s1, s2)
   ! trig, hyperbolic, other math functions
   r1 &  ! { dg-error "(A|a)ssumed.rank" }
     = atan2 (r1, &  ! { dg-error "(A|a)ssumed.rank" }
-             r2)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+             r2)  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
   r1 &  ! { dg-error "(A|a)ssumed.rank" }
     = atan (r2)  ! { dg-error "(A|a)ssumed.rank" }
   c1 &  ! { dg-error "(A|a)ssumed.rank" }
@@ -335,7 +335,7 @@ subroutine test_intrinsics (i1, i2, r1, r2, c1, c2, l1, l2, s1, s2)
   ! bit operations
   l1 &  ! { dg-error "(A|a)ssumed.rank" }
     = blt (i1, &  ! { dg-error "(A|a)ssumed.rank" }
-           i2)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+           i2)  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
   l1 &  ! { dg-error "(A|a)ssumed.rank" }
     = btest (i1, 0)  ! { dg-error "(A|a)ssumed.rank" }
   i1 &  ! { dg-error "(A|a)ssumed.rank" }
@@ -348,7 +348,7 @@ subroutine test_intrinsics (i1, i2, r1, r2, c1, c2, l1, l2, s1, s2)
     = char (i1)  ! { dg-error "(A|a)ssumed.rank" }
   c1 &  ! { dg-error "(A|a)ssumed.rank" }
     = cmplx (r1, &  ! { dg-error "(A|a)ssumed.rank" }
-             r2)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+             r2)  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
   i1 &  ! { dg-error "(A|a)ssumed.rank" }
     = floor (r1)  ! { dg-error "(A|a)ssumed.rank" }
   r1 &  ! { dg-error "(A|a)ssumed.rank" }
@@ -357,16 +357,16 @@ subroutine test_intrinsics (i1, i2, r1, r2, c1, c2, l1, l2, s1, s2)
   ! reductions
   l = any (l2)  ! { dg-error "(A|a)ssumed.rank" }
   r = dot_product (r1, &  ! { dg-error "(A|a)ssumed.rank" }
-                   r2)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+                   r2)  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
   i = iall (i2, &  ! { dg-error "(A|a)ssumed.rank" }
-            l2)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+            l2)  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
 
   ! string operations
   s1 &  ! { dg-error "(A|a)ssumed.rank" }
     = adjustr (s2)  ! { dg-error "(A|a)ssumed.rank" }
   i1 &  ! { dg-error "(A|a)ssumed.rank" }
     = index (c1, &  ! { dg-error "(A|a)ssumed.rank" }
-             c2)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+             c2)  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
 
   ! misc
   i1 &  ! { dg-error "(A|a)ssumed.rank" }
@@ -374,12 +374,12 @@ subroutine test_intrinsics (i1, i2, r1, r2, c1, c2, l1, l2, s1, s2)
   i = findloc (r1, 0.0)  ! { dg-error "(A|a)ssumed.rank" }
   r1 &  ! { dg-error "(A|a)ssumed.rank" }
     = matmul (r1, &  ! { dg-error "(A|a)ssumed.rank" }
-              r2)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+              r2)  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
   r1 &  ! { dg-error "(A|a)ssumed.rank" }
     = reshape (r2, [10, 3])  ! { dg-error "(A|a)ssumed.rank" }
   i1 &  ! { dg-error "(A|a)ssumed.rank" }
     = sign (i1, &  ! { dg-error "(A|a)ssumed.rank" }
-            i2)  ! { dg-error "(A|a)ssumed.rank" "pr101337, failure to diagnose both operands" { xfail *-*-*} }
+            i2)  ! { dg-error "(A|a)ssumed.rank" "pr101337" }
   s1 &  ! { dg-error "(A|a)ssumed.rank" }
     = transpose (s2)  ! { dg-error "(A|a)ssumed.rank" }
 
diff --git a/gcc/testsuite/gfortran.dg/c-interop/cf-descriptor-5-c.c b/gcc/testsuite/gfortran.dg/c-interop/cf-descriptor-5-c.c
index 12464b5..320a354 100644
--- a/gcc/testsuite/gfortran.dg/c-interop/cf-descriptor-5-c.c
+++ b/gcc/testsuite/gfortran.dg/c-interop/cf-descriptor-5-c.c
@@ -1,6 +1,5 @@
 #include <stdlib.h>
 #include <stdio.h>
-#include <alloca.h>
 
 #include <ISO_Fortran_binding.h>
 #include "dump-descriptors.h"
@@ -8,12 +7,18 @@
 extern void ctest (int n);
 extern void ftest (CFI_cdesc_t *a, int n);
 
+#define BUFSIZE 512
+static char adata[BUFSIZE];
+
 void
 ctest (int n)
 {
   CFI_CDESC_T(0) adesc;
   CFI_cdesc_t *a = (CFI_cdesc_t *) &adesc;
-  char *adata = (char *) alloca (n);
+
+  /* Use a fixed-size static buffer instead of allocating one dynamically.  */
+  if (n > BUFSIZE)
+    abort ();
 
   /* Fill in adesc.  */
   check_CFI_status ("CFI_establish",
diff --git a/gcc/testsuite/gfortran.dg/charlen_04.f90 b/gcc/testsuite/gfortran.dg/charlen_04.f90
index f93465f..97aa0ec 100644
--- a/gcc/testsuite/gfortran.dg/charlen_04.f90
+++ b/gcc/testsuite/gfortran.dg/charlen_04.f90
@@ -3,6 +3,5 @@
 program p
    type t
       character(*), allocatable :: x(*)  ! { dg-error "must have a deferred shape" }
-   end type
+   end type                              ! { dg-error "needs to be a constant specification" "" { target "*-*-*" } .-1 } 
 end
-! { dg-excess-errors "needs to be a constant specification" } 
diff --git a/gcc/testsuite/gfortran.dg/charlen_05.f90 b/gcc/testsuite/gfortran.dg/charlen_05.f90
index 0eb0015..e58f926 100644
--- a/gcc/testsuite/gfortran.dg/charlen_05.f90
+++ b/gcc/testsuite/gfortran.dg/charlen_05.f90
@@ -3,6 +3,5 @@
 program p
    type t
       character(*) :: x y  ! { dg-error "error in data declaration" }
-   end type
+   end type                ! { dg-error "needs to be a constant specification" "" { target "*-*-*" } .-1 } 
 end
-! { dg-excess-errors "needs to be a constant specification" } 
diff --git a/gcc/testsuite/gfortran.dg/charlen_06.f90 b/gcc/testsuite/gfortran.dg/charlen_06.f90
index e20d604..836c1c69 100644
--- a/gcc/testsuite/gfortran.dg/charlen_06.f90
+++ b/gcc/testsuite/gfortran.dg/charlen_06.f90
@@ -3,6 +3,5 @@
 program p
    type t
       character(*) :: x+1  ! { dg-error "error in data declaration" }
-   end type
+   end type                ! { dg-error "needs to be a constant specification" "" { target "*-*-*" } .-1 } 
 end
-! { dg-excess-errors "needs to be a constant specification" } 
diff --git a/gcc/testsuite/gfortran.dg/charlen_13.f90 b/gcc/testsuite/gfortran.dg/charlen_13.f90
index d89b71c..8dc192b 100644
--- a/gcc/testsuite/gfortran.dg/charlen_13.f90
+++ b/gcc/testsuite/gfortran.dg/charlen_13.f90
@@ -4,7 +4,7 @@ program p
    type t
       character(2), allocatable :: a(*) ! { dg-error "must have a deferred shape" }
       character(*), allocatable :: b(2) ! { dg-error "must have a deferred shape" }
+                                        ! { dg-error "needs to be a constant specification" "" { target "*-*-*" } .-1 } 
       character(*), allocatable :: c(*) ! { dg-error "must have a deferred shape" }
-   end type
+   end type                             ! { dg-error "needs to be a constant specification" "" { target "*-*-*" } .-1 } 
 end
-! { dg-excess-errors "needs to be a constant specification" } 
diff --git a/gcc/testsuite/gfortran.dg/co_reduce_2.f90 b/gcc/testsuite/gfortran.dg/co_reduce_2.f90
new file mode 100644
index 0000000..42bd02a
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/co_reduce_2.f90
@@ -0,0 +1,15 @@
+! { dg-do compile }
+! { dg-options "-fcoarray=single" }
+! PR 103054 - wrong keyword name.
+! Original test case by Damian Rouson.
+program main
+  implicit none
+  logical :: co_all= .true.
+  call co_reduce(co_all, operator=both) ! { dg-error "Cannot find keyword" }
+  call co_reduce(co_all, operation=both)
+contains
+  logical pure function both(lhs,rhs)
+    logical, intent(in) :: lhs, rhs
+    both = lhs .and. rhs
+  end function
+end
diff --git a/gcc/testsuite/gfortran.dg/coarray_9.f90 b/gcc/testsuite/gfortran.dg/coarray_9.f90
index 0a4dbec..b613428 100644
--- a/gcc/testsuite/gfortran.dg/coarray_9.f90
+++ b/gcc/testsuite/gfortran.dg/coarray_9.f90
@@ -16,4 +16,4 @@ critical ! "Coarrays disabled"  (but error above is fatal)
 end critical ! "Expecting END PROGRAM statement"  (but error above is fatal)
 
 end
-! { dg-excess-errors "compilation terminated" }
+! { dg-prune-output "compilation terminated" }
diff --git a/gcc/testsuite/gfortran.dg/coarray_collectives_14.f90 b/gcc/testsuite/gfortran.dg/coarray_collectives_14.f90
index 6d53411..15679ee 100644
--- a/gcc/testsuite/gfortran.dg/coarray_collectives_14.f90
+++ b/gcc/testsuite/gfortran.dg/coarray_collectives_14.f90
@@ -63,10 +63,10 @@ program test
   call co_reduce(caf, dt%arg3) ! { dg-error "shall have two arguments" }
   call co_reduce(caf, elem) ! { dg-error "ELEMENTAL non-INTRINSIC procedure 'elem' is not allowed as an actual argument" }
   call co_reduce(caf, dt%elem) ! { dg-error "ELEMENTAL procedure pointer component 'elem' is not allowed as an actual argument" }
-  call co_reduce(caf, realo) ! { dg-error "A argument at .1. has type INTEGER.4. but the function passed as OPERATOR at .2. returns REAL.4." }
-  call co_reduce(caf, dt%realo) ! { dg-error "A argument at .1. has type INTEGER.4. but the function passed as OPERATOR at .2. returns REAL.4." }
-  call co_reduce(caf, int8) ! { dg-error "A argument at .1. has type INTEGER.4. but the function passed as OPERATOR at .2. returns INTEGER.8." }
-  call co_reduce(caf, dt%int8) ! { dg-error "A argument at .1. has type INTEGER.4. but the function passed as OPERATOR at .2. returns INTEGER.8." }
+  call co_reduce(caf, realo) ! { dg-error "A argument at .1. has type INTEGER.4. but the function passed as OPERATION at .2. returns REAL.4." }
+  call co_reduce(caf, dt%realo) ! { dg-error "A argument at .1. has type INTEGER.4. but the function passed as OPERATION at .2. returns REAL.4." }
+  call co_reduce(caf, int8) ! { dg-error "A argument at .1. has type INTEGER.4. but the function passed as OPERATION at .2. returns INTEGER.8." }
+  call co_reduce(caf, dt%int8) ! { dg-error "A argument at .1. has type INTEGER.4. but the function passed as OPERATION at .2. returns INTEGER.8." }
   call co_reduce(caf, arr) ! { dg-error "scalar nonallocatable nonpointer arguments and return a nonallocatable nonpointer scalar" }
   call co_reduce(caf, dt%arr) ! { dg-error "scalar nonallocatable nonpointer arguments and return a nonallocatable nonpointer scalar" }
   call co_reduce(caf, ptr) ! { dg-error "scalar nonallocatable nonpointer arguments and return a nonallocatable nonpointer scalar" }
@@ -83,10 +83,10 @@ program test
   call co_reduce(caf, dt%tgt) ! { dg-error "shall have the TARGET attribute either for none or both arguments" }
   call co_reduce(c4, char44) ! OK
   call co_reduce(c4, dt%char44) ! OK
-  call co_reduce(c3, char34) ! { dg-error "character length of the A argument at .1. and of the arguments of the OPERATOR at .2. shall be the same" }
-  call co_reduce(c3, dt%char34) ! { dg-error "character length of the A argument at .1. and of the arguments of the OPERATOR at .2. shall be the same" }
-  call co_reduce(c4, char34) ! { dg-error "The character length of the A argument at .1. and of the function result of the OPERATOR at .2. shall be the same" }
-  call co_reduce(c4, dt%char34) ! { dg-error "The character length of the A argument at .1. and of the function result of the OPERATOR at .2. shall be the same" }
+  call co_reduce(c3, char34) ! { dg-error "character length of the A argument at .1. and of the arguments of the OPERATION at .2. shall be the same" }
+  call co_reduce(c3, dt%char34) ! { dg-error "character length of the A argument at .1. and of the arguments of the OPERATION at .2. shall be the same" }
+  call co_reduce(c4, char34) ! { dg-error "The character length of the A argument at .1. and of the function result of the OPERATION at .2. shall be the same" }
+  call co_reduce(c4, dt%char34) ! { dg-error "The character length of the A argument at .1. and of the function result of the OPERATION at .2. shall be the same" }
 
 contains
   pure integer function valid(x,y)
diff --git a/gcc/testsuite/gfortran.dg/coarray_collectives_16.f90 b/gcc/testsuite/gfortran.dg/coarray_collectives_16.f90
index 683bedd..8419cf9 100644
--- a/gcc/testsuite/gfortran.dg/coarray_collectives_16.f90
+++ b/gcc/testsuite/gfortran.dg/coarray_collectives_16.f90
@@ -15,9 +15,9 @@ program test
   character(len=99) :: val3
   integer :: res
 
-  call co_reduce(val1, operator=fr, result_image=num_images(), stat=stat1, errmsg=errmesg1)
-  call co_reduce(val2, operator=gz, result_image=4, stat=stat2, errmsg=errmesg2)
-  call co_reduce(val3, operator=hc, result_image=res,stat=stat3, errmsg=errmesg3)
+  call co_reduce(val1, operation=fr, result_image=num_images(), stat=stat1, errmsg=errmesg1)
+  call co_reduce(val2, operation=gz, result_image=4, stat=stat2, errmsg=errmesg2)
+  call co_reduce(val3, operation=hc, result_image=res,stat=stat3, errmsg=errmesg3)
 contains
   pure real function fr(x,y)
     real, value :: x, y
diff --git a/gcc/testsuite/gfortran.dg/coarray_collectives_3.f90 b/gcc/testsuite/gfortran.dg/coarray_collectives_3.f90
index 2a3ddf4..04d540f 100644
--- a/gcc/testsuite/gfortran.dg/coarray_collectives_3.f90
+++ b/gcc/testsuite/gfortran.dg/coarray_collectives_3.f90
@@ -8,4 +8,4 @@ program test
   integer :: val
   call co_max(val) ! { dg-error "Coarrays disabled at .1., use '-fcoarray=' to enable" }
 end program test
-! { dg-excess-errors "compilation terminated" }
+! { dg-prune-output "compilation terminated" }
diff --git a/gcc/testsuite/gfortran.dg/coarray_collectives_9.f90 b/gcc/testsuite/gfortran.dg/coarray_collectives_9.f90
index f53eb4e..ee3902c 100644
--- a/gcc/testsuite/gfortran.dg/coarray_collectives_9.f90
+++ b/gcc/testsuite/gfortran.dg/coarray_collectives_9.f90
@@ -26,10 +26,10 @@ program test
   end interface
 
   call co_broadcast("abc") ! { dg-error "Missing actual argument 'source_image' in call to 'co_broadcast'" }
-  call co_reduce("abc") ! { dg-error "Missing actual argument 'operator' in call to 'co_reduce'" }
+  call co_reduce("abc") ! { dg-error "Missing actual argument 'operation' in call to 'co_reduce'" }
   call co_broadcast(1, source_image=1) ! { dg-error "'a' argument of 'co_broadcast' intrinsic at .1. must be a variable" }
-  call co_reduce(a=1, operator=red_f) ! { dg-error "'a' argument of 'co_reduce' intrinsic at .1. must be a variable" }
-  call co_reduce(a=val, operator=red_f2) ! { dg-error "OPERATOR argument at \\(1\\) must be a PURE function" }
+  call co_reduce(a=1, operation=red_f) ! { dg-error "'a' argument of 'co_reduce' intrinsic at .1. must be a variable" }
+  call co_reduce(a=val, operation=red_f2) ! { dg-error "OPERATION argument at \\(1\\) must be a PURE function" }
 
   call co_broadcast(val, source_image=[1,2]) ! { dg-error "must be a scalar" }
   call co_broadcast(val, source_image=1.0) ! { dg-error "must be INTEGER" }
diff --git a/gcc/testsuite/gfortran.dg/data_invalid.f90 b/gcc/testsuite/gfortran.dg/data_invalid.f90
index 960a8f3..6d03587 100644
--- a/gcc/testsuite/gfortran.dg/data_invalid.f90
+++ b/gcc/testsuite/gfortran.dg/data_invalid.f90
@@ -40,7 +40,7 @@ SUBROUTINE data_init_array_invalid()
   data       e(2) / 2 /                          ! { dg-error "re-initialization" }
 
   integer :: f(3) = 0                            ! { dg-error "already is initialized" }
-  data       f(2) / 1 /
+  data       f(2) / 1 /                          ! { dg-error "already is initialized" }
 
   ! full array initializer, re-initialize subsection
   integer :: g(3)
@@ -48,7 +48,7 @@ SUBROUTINE data_init_array_invalid()
   data       g(1:2) / 2*2 /                      ! { dg-error "re-initialization" }
 
   integer :: h(3) = 1                            ! { dg-error "already is initialized" }
-  data       h(2:3) / 2*2 /
+  data       h(2:3) / 2*2 /                      ! { dg-error "already is initialized" }
 
   ! full array initializer, re-initialize full array
   integer :: i(3)
@@ -56,7 +56,7 @@ SUBROUTINE data_init_array_invalid()
   data       i   / 2,2,2 /                       ! { dg-error "re-initialization" }
 
   integer :: j(3) = 1                            ! { dg-error "already is initialized" }
-  data       j   / 3*2 /
+  data       j   / 3*2 /                         ! { dg-error "already is initialized" }
 END SUBROUTINE
 
 SUBROUTINE data_init_matrix_invalid()
@@ -85,7 +85,7 @@ SUBROUTINE data_init_matrix_invalid()
   data       e(2,3) / 2 /                        ! { dg-error "re-initialization" }
 
   integer :: f(3,3) = 1                          ! { dg-error "already is initialized" }
-  data       f(3,2) / 2 /
+  data       f(3,2) / 2 /                        ! { dg-error "already is initialized" }
 
   ! full array initializer, re-initialize subsection
   integer :: g(3,3)
@@ -93,7 +93,7 @@ SUBROUTINE data_init_matrix_invalid()
   data       g(2:3,2:3) / 2, 2*3, 4 /            ! { dg-error "re-initialization" }
 
   integer :: h(3,3) = 1                          ! { dg-error "already is initialized" }
-  data       h(2:3,2:3) / 2, 2*3, 4 /
+  data       h(2:3,2:3) / 2, 2*3, 4 /            ! { dg-error "already is initialized" }
 
   ! full array initializer, re-initialize full array
   integer :: i(3,3)
@@ -101,7 +101,7 @@ SUBROUTINE data_init_matrix_invalid()
   data       i   / 9 * 1 /                       ! { dg-error "re-initialization" }
 
   integer :: j(3,3) = 0                          ! { dg-error "already is initialized" }
-  data       j   / 9 * 1 /
+  data       j   / 9 * 1 /                       ! { dg-error "already is initialized" }
 END SUBROUTINE
 
 SUBROUTINE data_init_misc_invalid()
@@ -112,11 +112,9 @@ SUBROUTINE data_init_misc_invalid()
   ! index out-of-bounds, direct access
   integer :: b(3)
   data       b(-2) / 1 /                         ! { dg-error "below array lower bound" }
-
+                                                 ! { dg-warning "is out of bounds" "" { target *-*-* } .-1 }
   ! index out-of-bounds, implied do-loop (PR32315)
   integer :: i
   character(len=20), dimension(4) :: string
   data (string(i), i = 1, 5) / 'A', 'B', 'C', 'D', 'E' /   ! { dg-error "above array upper bound" }
 END SUBROUTINE
-
-! { dg-excess-errors "" }
diff --git a/gcc/testsuite/gfortran.dg/diagnostic-format-json-1.F90 b/gcc/testsuite/gfortran.dg/diagnostic-format-json-1.F90
index 606fe0f..2993f7c 100644
--- a/gcc/testsuite/gfortran.dg/diagnostic-format-json-1.F90
+++ b/gcc/testsuite/gfortran.dg/diagnostic-format-json-1.F90
@@ -9,6 +9,7 @@
 
 ! { dg-regexp "\"kind\": \"error\"" }
 ! { dg-regexp "\"column-origin\": 1" }
+! { dg-regexp "\"escape-source\": false" }
 ! { dg-regexp "\"message\": \"#error message\"" }
 
 ! { dg-regexp "\"caret\": \{" }
diff --git a/gcc/testsuite/gfortran.dg/diagnostic-format-json-2.F90 b/gcc/testsuite/gfortran.dg/diagnostic-format-json-2.F90
index 56615f0..1681462 100644
--- a/gcc/testsuite/gfortran.dg/diagnostic-format-json-2.F90
+++ b/gcc/testsuite/gfortran.dg/diagnostic-format-json-2.F90
@@ -9,6 +9,7 @@
 
 ! { dg-regexp "\"kind\": \"warning\"" }
 ! { dg-regexp "\"column-origin\": 1" }
+! { dg-regexp "\"escape-source\": false" }
 ! { dg-regexp "\"message\": \"#warning message\"" }
 ! { dg-regexp "\"option\": \"-Wcpp\"" }
 ! { dg-regexp "\"option_url\": \"\[^\n\r\"\]*#index-Wcpp\"" }
diff --git a/gcc/testsuite/gfortran.dg/diagnostic-format-json-3.F90 b/gcc/testsuite/gfortran.dg/diagnostic-format-json-3.F90
index 5021475..f0a67de 100644
--- a/gcc/testsuite/gfortran.dg/diagnostic-format-json-3.F90
+++ b/gcc/testsuite/gfortran.dg/diagnostic-format-json-3.F90
@@ -9,6 +9,7 @@
 
 ! { dg-regexp "\"kind\": \"error\"" }
 ! { dg-regexp "\"column-origin\": 1" }
+! { dg-regexp "\"escape-source\": false" }
 ! { dg-regexp "\"message\": \"#warning message\"" }
 ! { dg-regexp "\"option\": \"-Werror=cpp\"" }
 ! { dg-regexp "\"option_url\": \"\[^\n\r\"\]*#index-Wcpp\"" }
diff --git a/gcc/testsuite/gfortran.dg/do_4.f b/gcc/testsuite/gfortran.dg/do_4.f
index 6d688a0..e391b68 100644
--- a/gcc/testsuite/gfortran.dg/do_4.f
+++ b/gcc/testsuite/gfortran.dg/do_4.f
@@ -4,6 +4,7 @@
        if(i.eq.5)then
           goto 10
  10    endif           ! { dg-error "is within another block" }
-       end
-! { dg-excess-errors "" }
+       end             ! { dg-error "END DO statement expected" }
+                       ! { dg-warning "Fortran 2018 deleted feature: DO termination statement which is not END DO or CONTINUE" "" { target "*-*-*" } 6 }
+! { dg-error "Unexpected end of file" "" { target "*-*-*" } 0 }
 
diff --git a/gcc/testsuite/gfortran.dg/do_subscript_3.f90 b/gcc/testsuite/gfortran.dg/do_subscript_3.f90
index 2f62f58..8f7183e 100644
--- a/gcc/testsuite/gfortran.dg/do_subscript_3.f90
+++ b/gcc/testsuite/gfortran.dg/do_subscript_3.f90
@@ -1,4 +1,5 @@
 ! { dg-do compile }
+! { dg-options "-O0" }
 ! PR fortran/91424
 ! Check that only one warning is issued inside blocks, and that
 ! warnings are also issued for contained subroutines.
diff --git a/gcc/testsuite/gfortran.dg/dollar_sym_1.f90 b/gcc/testsuite/gfortran.dg/dollar_sym_1.f90
index cb53fd3..5cee775 100644
--- a/gcc/testsuite/gfortran.dg/dollar_sym_1.f90
+++ b/gcc/testsuite/gfortran.dg/dollar_sym_1.f90
@@ -7,4 +7,4 @@
       PARAMETER (PLT$B_OPC=0) !  Unreachable as the error above is now fatal
       common /abc$def/ PLT$C_HOUSTPIX, PLT$C_COMMAND !  Unreachable as the error above is now fatal
       end
-! { dg-excess-errors "compilation terminated" }
+! { dg-prune-output "compilation terminated" }
diff --git a/gcc/testsuite/gfortran.dg/dollar_sym_3.f b/gcc/testsuite/gfortran.dg/dollar_sym_3.f
index 7b9b344..52ce041 100644
--- a/gcc/testsuite/gfortran.dg/dollar_sym_3.f
+++ b/gcc/testsuite/gfortran.dg/dollar_sym_3.f
@@ -8,4 +8,4 @@ c Segmentation fault in gfc_restore_last_undo_checkpoint
       COMMON RADE3155V62$JUTMU9L9E(3,3,3), LADE314JUTMP9         ! { dg-error "Invalid character '\\$' at .1.. Use '-fdollar-ok' to allow it as an extension" }
      +LHEDDJNTMP9L(3,3,3)                                                       
       end
-! { dg-excess-errors "compilation terminated" }
+! { dg-prune-output "compilation terminated" }
diff --git a/gcc/testsuite/gfortran.dg/external_implicit_none_3.f08 b/gcc/testsuite/gfortran.dg/external_implicit_none_3.f08
new file mode 100644
index 0000000..329deed
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/external_implicit_none_3.f08
@@ -0,0 +1,17 @@
+! { dg-do compile }
+! { dg-options "-std=f2018" }
+! Tests fix for PR100972 - Fails to warn about missing EXTERNAL attribute
+! Contributed by Gerhard Steinmetz
+
+program p
+   implicit none (external)
+   real, external :: f
+   real :: a
+   real :: b
+   integer :: i
+   character :: c
+   a = f() ! OK
+   b = g() ! { dg-error "Missing explicit declaration with EXTERNAL attribute" }
+   i = h() ! { dg-error "Missing explicit declaration with EXTERNAL attribute" }
+   c = j() ! { dg-error "Missing explicit declaration with EXTERNAL attribute" }
+end
diff --git a/gcc/testsuite/gfortran.dg/fmt_tab_1.f90 b/gcc/testsuite/gfortran.dg/fmt_tab_1.f90
index a763714..0b36aee 100644
--- a/gcc/testsuite/gfortran.dg/fmt_tab_1.f90
+++ b/gcc/testsuite/gfortran.dg/fmt_tab_1.f90
@@ -6,7 +6,6 @@
         write (*, 10)
         ! There is a tab character before 'bug!'.  This is accepted without
         ! the -Wno-tabs option or a -std= option.
- 10     format ('Hello ',	'bug!') ! { dg-warning "tab character in format" }
-
+ 10     format ('Hello ',	'bug!') ! { dg-warning "tab character in format at " }
+                                        ! { dg-warning "tab character at " "" { target "*-*-*" } .-1 }
       end
-! { dg-excess-errors "tab character in format" }
diff --git a/gcc/testsuite/gfortran.dg/fmt_tab_2.f90 b/gcc/testsuite/gfortran.dg/fmt_tab_2.f90
index bfff7ce..3f2d6fc 100644
--- a/gcc/testsuite/gfortran.dg/fmt_tab_2.f90
+++ b/gcc/testsuite/gfortran.dg/fmt_tab_2.f90
@@ -5,5 +5,4 @@
       program TestFormat
         write (*, 10)
  10     format ('Hello ',	'bug!') ! { dg-warning "tab character in format" }
-      end
-! { dg-excess-errors "tab character in FORMAT" }
+      end                               ! { dg-warning "tab character at " "" { target "*-*-*" } .-1 }
diff --git a/gcc/testsuite/gfortran.dg/forall_16.f90 b/gcc/testsuite/gfortran.dg/forall_16.f90
index 017aa5a..e341c37 100644
--- a/gcc/testsuite/gfortran.dg/forall_16.f90
+++ b/gcc/testsuite/gfortran.dg/forall_16.f90
@@ -4,6 +4,4 @@
   implicit none
   integer i,dest(10)
   forall (i=2:ix)  dest(i)=i ! { dg-error "has no IMPLICIT type" }
-end
-
-! { dg-excess-errors "Can't convert UNKNOWN to INTEGER" }
+end                          ! { dg-error "Cannot convert UNKNOWN to INTEGER" "" { target "*-*-*" } .-1 }
diff --git a/gcc/testsuite/gfortran.dg/g77/970125-0.f b/gcc/testsuite/gfortran.dg/g77/970125-0.f
index 656c475..0c12e26 100644
--- a/gcc/testsuite/gfortran.dg/g77/970125-0.f
+++ b/gcc/testsuite/gfortran.dg/g77/970125-0.f
@@ -1,7 +1,6 @@
 c { dg-do compile }
 c
-c Following line added on transfer to gfortran testsuite
-c { dg-excess-errors "" }
+c { dg-additional-options "-w" }
 c
 C JCB comments:
 C g77 doesn't accept the added line "integer(kind=7) ..." --
@@ -35,10 +34,10 @@ c     Frontend bug fixed by JCB 1998-06-01 com.c &c changes.
         print *, max4
         print *, i4, %loc(i4)
         print *, i8, %loc(i8)
-        call foo(i4, %loc(i4), i8, %loc(i8))
+        call foo(i4, %loc(i4), i8, %loc(i8))  ! { dg-error "Type mismatch in argument 'i8a' at .1.; passed INTEGER.8. to INTEGER.4." }
         end
         subroutine foo(i4, i4a, i8, i8a)
-        integer(kind=7) i4a, i8a
+        integer(kind=7) i4a, i8a  ! { dg-error "Kind 7 not supported for type INTEGER" }
         integer(kind=8) i8
         print *, i4, i4a
         print *, i8, i8a
diff --git a/gcc/testsuite/gfortran.dg/gomp/clauses-1.f90 b/gcc/testsuite/gfortran.dg/gomp/clauses-1.f90
new file mode 100644
index 0000000..639f5d1
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/gomp/clauses-1.f90
@@ -0,0 +1,667 @@
+! { dg-do compile }
+
+module m
+  use iso_c_binding, only: c_intptr_t
+  implicit none (external, type)
+
+  integer(c_intptr_t), parameter :: &
+    omp_null_allocator = 0,         &
+    omp_default_mem_alloc = 1,      &
+    omp_large_cap_mem_alloc = 2,    &
+    omp_const_mem_alloc = 3,        &
+    omp_high_bw_mem_alloc = 4,      &
+    omp_low_lat_mem_alloc = 5,      &
+    omp_cgroup_mem_alloc = 6,       &
+    omp_pteam_mem_alloc = 7,        &
+    omp_thread_mem_alloc = 8
+
+  integer, parameter :: &
+    omp_allocator_handle_kind = c_intptr_t
+
+  integer :: t
+  !$omp threadprivate (t)
+
+  integer :: f, l, ll, r, r2
+  !$omp declare target (f, l, ll, r, r2)
+
+contains
+
+subroutine foo (d, m, i1, i2, i3, p, idp, s, nte, tl, nth, g, nta, fi, pp, q, dd, ntm)
+  !$omp declare target (foo)
+  integer :: d, m, p, idp, s, nte, tl, nth, g, nta, pp, q, dd, ntm
+  logical :: i1, i2, i3, fi
+  pointer :: q
+  integer :: i
+
+  !$omp distribute parallel do &
+  !$omp&  private (p) firstprivate (f) collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) default(shared) shared(s) reduction(+:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) order(concurrent)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp distribute parallel do simd &
+  !$omp&  private (p) firstprivate (f) collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) if(simd: i1) default(shared) shared(s) reduction(+:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) nontemporal(ntm) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) order(concurrent)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp distribute simd &
+  !$omp&  private (p) firstprivate (f) collapse(1) dist_schedule(static, 16) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) reduction(+:r) if(i1) nontemporal(ntm) &
+  !$omp&  order(concurrent)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+end
+
+subroutine qux (p)
+  !$omp declare target (qux)
+  integer, value :: p
+
+  !$omp loop bind(teams) order(concurrent) &
+  !$omp&  private (p) lastprivate (l) collapse(1) reduction(+:r)
+  do l = 1, 64
+    ll = ll + 1
+  end do
+end
+
+subroutine baz (d, m, i1, i2, i3, p, idp, s, nte, tl, nth, g, nta, fi, pp, q, dd, ntm)
+  integer :: d, m, p, idp, s, nte, tl, nth, g, nta, pp, q, dd, ntm
+  logical :: i1, i2, i3, fi
+  pointer :: q
+  integer :: i
+  !$omp distribute parallel do &
+  !$omp&  private (p) firstprivate (f) collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) default(shared) shared(s) reduction(+:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) copyin(t)
+  ! FIXME/TODO: allocate (p)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp distribute parallel do &
+  !$omp&  private (p) firstprivate (f) collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) default(shared) shared(s) reduction(+:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) order(concurrent)
+  ! FIXME/TODO: allocate (p)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp distribute parallel do simd &
+  !$omp&  private (p) firstprivate (f) collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) if(simd: i1) default(shared) shared(s) reduction(+:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) nontemporal(ntm) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) copyin(t)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+
+  !$omp distribute parallel do simd &
+  !$omp&  private (p) firstprivate (f) collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) if(simd: i1) default(shared) shared(s) reduction(+:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) nontemporal(ntm) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+
+  !$omp distribute simd &
+  !$omp&  private (p) firstprivate (f) collapse(1) dist_schedule(static, 16) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) reduction(+:r) if(i1) nontemporal(ntm) &
+  !$omp&  order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+
+  !$omp loop bind(parallel) order(concurrent) &
+  !$omp&  private (p) lastprivate (l) collapse(1) reduction(+:r)
+  do l = 1, 64
+    ll = ll + 1
+  end do
+end
+
+subroutine bar (d, m, i1, i2, i3, p, idp, s, nte, tl, nth, g, nta, fi, pp, q, dd, ntm)
+  integer :: d, m, p, idp, s, nte, tl, nth, g, nta, pp, q, dd(0:5), ntm
+  logical :: i1, i2, i3, fi
+  pointer :: q
+  integer :: i
+
+  !$omp do simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) linear (ll:1) reduction(+:r) schedule(static, 4) collapse(1) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) nontemporal(ntm) if(i1) order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+  !$omp end do simd nowait
+
+  !$omp parallel do &
+  !$omp&  private (p) firstprivate (f) if (parallel: i2) default(shared) shared(s) copyin(t) reduction(+:r) num_threads (nth) &
+  !$omp&  proc_bind(spread) lastprivate (l) linear (ll:1) ordered schedule(static, 4) collapse(1)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+
+  !$omp parallel do &
+  !$omp&  private (p) firstprivate (f) if (parallel: i2) default(shared) shared(s) copyin(t) reduction(+:r) num_threads (nth) &
+  !$omp&  proc_bind(spread) lastprivate (l) linear (ll:1) schedule(static, 4) collapse(1) order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+
+  !$omp parallel do simd &
+  !$omp&  private (p) firstprivate (f) if (i2) default(shared) shared(s) copyin(t) reduction(+:r) num_threads (nth) &
+  !$omp&  proc_bind(spread) lastprivate (l) linear (ll:1) schedule(static, 4) collapse(1) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) nontemporal(ntm) order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+
+  !$omp parallel sections &
+  !$omp&  private (p) firstprivate (f) if (parallel: i2) default(shared) shared(s) copyin(t) reduction(+:r) num_threads (nth) &
+  !$omp&  proc_bind(spread) lastprivate (l)
+  ! FIXME/TODO: allocate (f)
+    !$omp section
+      block; end block
+    !$omp section
+      block; end block
+  !$omp end parallel sections
+
+  !$omp target parallel &
+  !$omp&  device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  if (parallel: i2) default(shared) shared(s) reduction(+:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  depend(inout: dd(0)) in_reduction(+:r2)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  !$omp end target parallel nowait
+
+  !$omp target parallel do &
+  !$omp&  device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  if (parallel: i2) default(shared) shared(s) reduction(+:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) linear (ll:1) ordered schedule(static, 4) collapse(1) depend(inout: dd(0)) &
+  !$omp&  in_reduction(+:r2)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+  !$omp end target parallel do nowait
+
+  !$omp target parallel do &
+  !$omp&  device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  if (parallel: i2) default(shared) shared(s) reduction(+:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) linear (ll:1) schedule(static, 4) collapse(1) depend(inout: dd(0)) order(concurrent) &
+  !$omp&  in_reduction(+:r2)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+  !$omp end target parallel do nowait
+
+  !$omp target parallel do simd &
+  !$omp&  device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  if (parallel: i2) default(shared) shared(s) reduction(+:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) linear (ll:1) schedule(static, 4) collapse(1) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) depend(inout: dd(0)) nontemporal(ntm) if (simd: i3) order(concurrent) &
+  !$omp&  in_reduction(+:r2)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+  !$omp end target parallel do simd nowait
+
+  !$omp target teams &
+  !$omp&  device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  shared(s) default(shared) reduction(+:r) num_teams(nte - 1:nte) thread_limit(tl) depend(inout: dd(0)) &
+  !$omp&  in_reduction(+:r2)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  !$omp end target teams nowait
+
+  !$omp target teams distribute &
+  !$omp&  device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) order(concurrent) &
+  !$omp&  collapse(1) dist_schedule(static, 16) depend(inout: dd(0)) in_reduction(+:r2)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  do i = 1, 64
+  end do
+  !$omp end target teams distribute nowait
+
+  !$omp target teams distribute parallel do &
+  !$omp&  device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) depend(inout: dd(0)) order(concurrent) &
+  !$omp&  in_reduction(+:r2)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+  !$omp end target teams distribute parallel do nowait
+
+  !$omp target teams distribute parallel do simd &
+  !$omp&  device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) order(concurrent) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) depend(inout: dd(0)) nontemporal(ntm) if (simd: i3) &
+  !$omp&  in_reduction(+:r2)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+  !$omp end target teams distribute parallel do simd nowait
+
+  !$omp target teams distribute simd &
+  !$omp&  device(d) map (tofrom: m) if (i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) order(concurrent) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) depend(inout: dd(0)) nontemporal(ntm) &
+  !$omp&  in_reduction(+:r2)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+  !$omp end target teams distribute simd nowait
+
+  !$omp target simd &
+  !$omp&  device(d) map (tofrom: m) if (target: i1) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  safelen(8) simdlen(4) lastprivate (l) linear(ll: 1) aligned(q: 32) reduction(+:r) &
+  !$omp&  depend(inout: dd(0)) nontemporal(ntm) if(simd:i3) order(concurrent) &
+  !$omp&  in_reduction(+:r2)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc:f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+  !$omp end target simd nowait
+
+  !$omp taskgroup task_reduction(+:r2)
+  ! FIXME/TODO: allocate (r2)
+  !$omp taskloop simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) grainsize (g) collapse(1) untied &
+  !$omp&  if(taskloop: i1) if(simd: i2) final(fi) mergeable priority (pp) &
+  !$omp&  safelen(8) simdlen(4) linear(ll: 1) aligned(q: 32) reduction(default, +:r) in_reduction(+:r2) nontemporal(ntm) &
+  !$omp&  order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+  !$omp end taskgroup
+
+  !$omp taskgroup task_reduction(+:r)
+  ! FIXME/TODO: allocate (r)
+  !$omp taskloop simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) grainsize (g) collapse(1) untied if(i1) &
+  !$omp&  final(fi) mergeable nogroup priority (pp) &
+  !$omp&  safelen(8) simdlen(4) linear(ll: 1) aligned(q: 32) in_reduction(+:r) nontemporal(ntm) &
+  !$omp&  order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+  !$omp end taskgroup
+
+  !$omp taskwait
+  !$omp taskloop simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) num_tasks (nta) collapse(1) if(taskloop: i1) &
+  !$omp&  final(fi) priority (pp) safelen(8) simdlen(4) linear(ll: 1) aligned(q: 32) reduction(+:r) if (simd: i3) nontemporal(ntm) &
+  !$omp&  order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll + 1
+  end do
+
+  !$omp target depend(inout: dd(0)) in_reduction(+:r2)
+  !$omp teams distribute &
+  !$omp&  private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) order(concurrent)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc: f)
+  do i = 1, 64
+  end do
+  !$omp end target nowait
+
+  !$omp target
+  !$omp teams distribute parallel do &
+  !$omp&  private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) order(concurrent)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc: f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+  !$omp end target
+
+  !$omp target
+  !$omp teams distribute parallel do simd &
+  !$omp&  private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) order(concurrent) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) if (simd: i3) nontemporal(ntm)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc: f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+  !$omp end target
+
+  !$omp target
+  !$omp teams distribute simd &
+  !$omp&  private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) order(concurrent) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) if(i3) nontemporal(ntm)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc: f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+  !$omp end target
+
+  !$omp teams distribute parallel do &
+  !$omp&  private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) copyin(t)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp teams distribute parallel do &
+  !$omp&  private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) order(concurrent) &
+  !$omp&  if (parallel: i2) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp teams distribute parallel do simd &
+  !$omp&  private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) if (simd: i3) nontemporal(ntm) copyin(t)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp teams distribute parallel do simd &
+  !$omp&  private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) &
+  !$omp&  if (parallel: i2) num_threads (nth) proc_bind(spread) &
+  !$omp&  lastprivate (l) schedule(static, 4) order(concurrent) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) if (simd: i3) nontemporal(ntm)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp teams distribute simd &
+  !$omp&  private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) &
+  !$omp&  collapse(1) dist_schedule(static, 16) order(concurrent) &
+  !$omp&  safelen(8) simdlen(4) aligned(q: 32) if(i3) nontemporal(ntm)
+  ! FIXME/TODO: allocate(f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp parallel master &
+  !$omp&  private (p) firstprivate (f) if (parallel: i2) default(shared) shared(s) reduction(+:r) &
+  !$omp&  num_threads (nth) proc_bind(spread) copyin(t)
+  ! FIXME/TODO: allocate (f)
+  !$omp end parallel master
+
+  !$omp parallel masked &
+  !$omp&  private (p) firstprivate (f) if (parallel: i2) default(shared) shared(s) reduction(+:r) &
+  !$omp&  num_threads (nth) proc_bind(spread) copyin(t) filter (d)
+  ! FIXME/TODO: allocate (f)
+  !$omp end parallel masked
+
+  !$omp taskgroup task_reduction (+:r2)
+  ! FIXME/TODO: allocate (r2)
+  !$omp master taskloop &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) grainsize (g) collapse(1) untied &
+  !$omp&  if(taskloop: i1) final(fi) mergeable priority (pp) &
+  !$omp&  reduction(default, +:r) in_reduction(+:r2)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+  !$omp end taskgroup
+
+  !$omp taskgroup task_reduction (+:r2)
+  ! FIXME/TODO: allocate (r2)
+  !$omp masked taskloop &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) grainsize (g) collapse(1) untied &
+  !$omp&  if(taskloop: i1) final(fi) mergeable priority (pp) reduction(default, +:r) in_reduction(+:r2) filter (d)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+  !$omp end taskgroup
+
+  !$omp taskgroup task_reduction (+:r2)
+  ! FIXME/TODO: allocate (r2)
+  !$omp master taskloop simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) grainsize (g) collapse(1) untied &
+  !$omp&  if(taskloop: i1) if(simd: i2) final(fi) mergeable priority (pp) &
+  !$omp&  safelen(8) simdlen(4) linear(ll: 1) aligned(q: 32) reduction(default, +:r) in_reduction(+:r2) nontemporal(ntm) &
+  !$omp&  order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+  !$omp end taskgroup
+
+  !$omp taskgroup task_reduction (+:r2)
+  ! FIXME/TODO: allocate (r2)
+  !$omp masked taskloop simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) grainsize (g) collapse(1) untied &
+  !$omp&  if(taskloop: i1) if(simd: i2) final(fi) mergeable priority (pp) &
+  !$omp&  safelen(8) simdlen(4) linear(ll: 1) aligned(q: 32) reduction(default, +:r) in_reduction(+:r2) nontemporal(ntm) &
+  !$omp&  order(concurrent) filter (d)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+  !$omp end taskgroup
+
+  !$omp parallel master taskloop &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) grainsize (g) collapse(1) untied &
+  !$omp&  if(taskloop: i1) final(fi) mergeable priority (pp) &
+  !$omp&  reduction(default, +:r) if (parallel: i2) num_threads (nth) proc_bind(spread) copyin(t)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp parallel masked taskloop &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) grainsize (g) collapse(1) untied &
+  !$omp&  if(taskloop: i1) final(fi) mergeable priority (pp) &
+  !$omp&  reduction(default, +:r) if (parallel: i2) num_threads (nth) proc_bind(spread) copyin(t) filter (d)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp parallel master taskloop simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) grainsize (g) collapse(1) untied &
+  !$omp&  if(taskloop: i1) if(simd: i2) final(fi) mergeable priority (pp) &
+  !$omp&  safelen(8) simdlen(4) linear(ll: 1) aligned(q: 32) reduction(default, +:r) nontemporal(ntm) if (parallel: i2) &
+  !$omp&  num_threads (nth) proc_bind(spread) copyin(t) order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp parallel masked taskloop simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) grainsize (g) collapse(1) untied &
+  !$omp&  if(taskloop: i1) if(simd: i2) final(fi) mergeable priority (pp) &
+  !$omp&  safelen(8) simdlen(4) linear(ll: 1) aligned(q: 32) reduction(default, +:r) nontemporal(ntm) if (parallel: i2) &
+  !$omp&  num_threads (nth) proc_bind(spread) copyin(t) order(concurrent) filter (d)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp taskgroup task_reduction (+:r2)
+  ! FIXME/TODO: allocate (r2)
+  !$omp master taskloop &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) num_tasks (nta) collapse(1) &
+  !$omp&  untied if(i1) final(fi) mergeable priority (pp) reduction(default, +:r) in_reduction(+:r2)
+  do i = 1, 64
+    ll = ll +1
+  end do
+  !$omp end taskgroup
+
+  !$omp taskgroup task_reduction (+:r2)
+  ! FIXME/TODO: allocate (r2)
+  !$omp masked taskloop &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) num_tasks (nta) collapse(1) &
+  !$omp&  untied if(i1) final(fi) mergeable priority (pp) reduction(default, +:r) in_reduction(+:r2) filter (d)
+  do i = 1, 64
+    ll = ll +1
+  end do
+  !$omp end taskgroup
+
+  !$omp taskgroup task_reduction (+:r2)
+  ! FIXME/TODO: allocate (r2)
+  !$omp master taskloop simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) num_tasks (nta) collapse(1) untied if(i1) &
+  !$omp&  final(fi) mergeable priority (pp) safelen(8) simdlen(4) linear(ll: 1) aligned(q: 32) reduction(default, +:r) &
+  !$omp&  in_reduction(+:r2) nontemporal(ntm) order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+  !$omp end taskgroup
+
+  !$omp taskgroup task_reduction (+:r2)
+  ! FIXME/TODO: allocate (r2)
+  !$omp masked taskloop simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) num_tasks (nta) collapse(1) untied &
+  !$omp&  if(i1) final(fi) mergeable priority (pp) safelen(8) simdlen(4) linear(ll: 1) aligned(q: 32) reduction(default, +:r) &
+  !$omp&  in_reduction(+:r2) nontemporal(ntm) order(concurrent) filter (d)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+  !$omp end taskgroup
+
+  !$omp parallel master taskloop &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) num_tasks (nta) collapse(1) untied &
+  !$omp&  if(i1) final(fi) mergeable priority (pp) reduction(default, +:r) num_threads (nth) proc_bind(spread) copyin(t)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp parallel masked taskloop &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) num_tasks (nta) collapse(1) untied &
+  !$omp&  if(i1) final(fi) mergeable priority (pp) reduction(default, +:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  copyin(t) filter (d)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp parallel master taskloop simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) num_tasks (nta) collapse(1) untied &
+  !$omp&  if(i1) final(fi) mergeable priority (pp) safelen(8) simdlen(4) linear(ll: 1) aligned(q: 32) reduction(default, +:r) &
+  !$omp&  nontemporal(ntm) num_threads (nth) proc_bind(spread)copyin(t) order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp parallel masked taskloop simd &
+  !$omp&  private (p) firstprivate (f) lastprivate (l) shared (s) default(shared) num_tasks (nta) collapse(1) untied if(i1) &
+  !$omp&  final(fi) mergeable priority (pp) safelen(8) simdlen(4) linear(ll: 1) aligned(q: 32) reduction(default, +:r) &
+  !$omp&  nontemporal(ntm) num_threads (nth) proc_bind(spread) copyin(t) order(concurrent) filter (d)
+  ! FIXME/TODO: allocate (f)
+  do i = 1, 64
+    ll = ll +1
+  end do
+
+  !$omp loop bind(thread) order(concurrent) &
+  !$omp&  private (p) lastprivate (l) collapse(1) reduction(+:r)
+  do l = 1, 64
+    ll = ll + 1
+  end do
+
+  !$omp parallel loop &
+  !$omp&  private (p) firstprivate (f) default(shared) shared(s) copyin(t) reduction(+:r) num_threads (nth) &
+  !$omp&  proc_bind(spread) lastprivate (l) collapse(1) bind(parallel) order(concurrent) if (parallel: i2)
+  ! FIXME/TODO: allocate (f)
+  do l = 1, 64
+    ll = ll + 1
+  end do
+
+  !$omp parallel loop &
+  !$omp&  private (p) firstprivate (f) default(shared) shared(s) copyin(t) reduction(+:r) num_threads (nth) &
+  !$omp&  proc_bind(spread) lastprivate (l) collapse(1) if (parallel: i2)
+  ! FIXME/TODO: allocate (f)
+  do l = 1, 64
+    ll = ll + 1
+  end do
+
+  !$omp teams loop &
+  !$omp&  private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) &
+  !$omp&  collapse(1) lastprivate (l) bind(teams)
+  ! FIXME/TODO: allocate (f)
+  do l = 1, 64
+  end do
+
+  !$omp teams loop &
+  !$omp&  private(p) firstprivate (f) shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) &
+  !$omp&  collapse(1) lastprivate (l) order(concurrent)
+  ! FIXME/TODO: allocate (f)
+  do l = 1, 64
+  end do
+
+  !$omp target parallel loop &
+  !$omp&  device(d) map (tofrom: m) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  default(shared) shared(s) reduction(+:r) num_threads (nth) proc_bind(spread) &
+  !$omp&  depend(inout: dd(0)) lastprivate (l) order(concurrent) collapse(1) in_reduction(+:r2) &
+  !$omp&  if (target: i1) if (parallel: i2)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc: f)
+  do l = 1, 64
+  end do
+  !$omp end target parallel loop nowait
+
+  !$omp target teams loop &
+  !$omp&  device(d) map (tofrom: m) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  shared(s) default(shared) reduction(+:r) num_teams(nte-1:nte) thread_limit(tl) depend(inout: dd(0)) &
+  !$omp&  lastprivate (l) bind(teams) collapse(1) in_reduction(+:r2) if (target: i1)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc: f)
+  do l = 1, 64
+  end do
+  !$omp end target teams loop nowait
+
+  !$omp target teams loop &
+  !$omp&  device(d) map (tofrom: m) private (p) firstprivate (f) defaultmap(tofrom: scalar) is_device_ptr (idp) &
+  !$omp&  shared(s) default(shared) reduction(+:r) num_teams(nte) thread_limit(tl) depend(inout: dd(0)) &
+  !$omp&  lastprivate (l) order(concurrent) collapse(1) in_reduction(+:r2) if (target: i1)
+  ! FIXME/TODO: allocate (omp_default_mem_alloc: f)
+  do l = 1, 64
+  end do
+  !$omp end target teams loop nowait
+
+end
+end module
diff --git a/gcc/testsuite/gfortran.dg/gomp/nowait-2.f90 b/gcc/testsuite/gfortran.dg/gomp/nowait-2.f90
new file mode 100644
index 0000000..d18459b
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/gomp/nowait-2.f90
@@ -0,0 +1,315 @@
+! Cross check that it is accepted without nowait
+subroutine bar()
+implicit none
+integer :: i, a(5)
+!$omp atomic write
+i = 5
+!$omp end atomic
+
+!$omp critical
+!$omp end critical
+
+!$omp distribute
+do i = 1, 5
+end do
+!$omp end distribute
+
+!$omp distribute parallel do
+do i = 1, 5
+end do
+!$omp end distribute parallel do
+
+!$omp distribute parallel do simd
+do i = 1, 5
+end do
+!$omp end distribute parallel do simd
+
+!$omp distribute simd
+do i = 1, 5
+end do
+!$omp end distribute simd
+
+!$omp masked
+!$omp end masked
+
+!$omp masked taskloop
+do i = 1, 5
+end do
+!$omp end masked taskloop
+
+!$omp masked taskloop simd
+do i = 1, 5
+end do
+!$omp end masked taskloop simd
+
+!$omp master
+!$omp end master
+
+!$omp master taskloop
+do i = 1, 5
+end do
+!$omp end master taskloop
+
+!$omp master taskloop simd
+do i = 1, 5
+end do
+!$omp end master taskloop simd
+
+!$omp ordered
+!$omp end ordered
+
+!$omp parallel
+!$omp end parallel
+
+!$omp parallel workshare
+a(:) = 5
+!$omp end parallel workshare
+
+!$omp parallel do
+do i = 1, 5
+end do
+!$omp end parallel do
+
+!$omp parallel do simd
+do i = 1, 5
+end do
+!$omp end parallel do simd
+
+!$omp parallel sections
+  !$omp section
+  block; end block
+!$omp end parallel sections
+
+!$omp parallel masked
+!$omp end parallel masked
+
+!$omp parallel masked taskloop
+do i = 1, 5
+end do
+!$omp end parallel masked taskloop
+
+!$omp parallel masked taskloop simd
+do i = 1, 5
+end do
+!$omp end parallel masked taskloop simd
+
+!$omp parallel master
+!$omp end parallel master
+
+!$omp parallel master taskloop
+do i = 1, 5
+end do
+!$omp end parallel master taskloop
+
+!$omp parallel master taskloop simd
+do i = 1, 5
+end do
+!$omp end parallel master taskloop simd
+
+!$omp simd
+do i = 1, 5
+end do
+!$omp end simd
+
+!$omp task
+!$omp end task
+
+!$omp taskgroup
+!$omp end taskgroup
+
+!$omp taskloop
+do i = 1, 5
+end do
+!$omp end taskloop
+
+!$omp taskloop simd
+do i = 1, 5
+end do
+!$omp end taskloop simd
+
+!$omp teams
+!$omp end teams
+
+!$omp teams distribute
+do i = 1, 5
+end do
+!$omp end teams distribute
+
+!$omp teams distribute parallel do
+do i = 1, 5
+end do
+!$omp end teams distribute parallel do
+
+!$omp teams distribute parallel do simd
+do i = 1, 5
+end do
+!$omp end teams distribute parallel do simd
+
+!$omp teams distribute simd
+do i = 1, 5
+end do
+!$omp end teams distribute simd
+
+!$omp target data map(tofrom:i)
+!$omp end target data
+
+end
+
+! invalid nowait
+
+subroutine foo
+implicit none
+integer :: i, a(5)
+!$omp atomic write
+i = 5
+!$omp end atomic nowait  ! { dg-error "Unexpected junk" }
+
+!$omp critical
+!$omp end critical nowait  ! { dg-error "Unexpected junk" }
+
+!$omp distribute
+do i = 1, 5
+end do
+!$omp end distribute nowait  ! { dg-error "Unexpected junk" }
+
+!$omp distribute parallel do
+do i = 1, 5
+end do
+!$omp end distribute parallel do nowait  ! { dg-error "Unexpected junk" }
+
+!$omp distribute parallel do simd
+do i = 1, 5
+end do
+!$omp end distribute parallel do simd nowait  ! { dg-error "Unexpected junk" }
+
+!$omp parallel sections
+  !$omp section
+  block; end block
+!$omp end parallel sections nowait  ! { dg-error "Unexpected junk" }
+
+!$omp distribute simd
+do i = 1, 5
+end do
+!$omp end distribute simd nowait  ! { dg-error "Unexpected junk" }
+
+!$omp masked
+!$omp end masked nowait  ! { dg-error "Unexpected junk" }
+
+!$omp masked taskloop
+do i = 1, 5
+end do
+!$omp end masked taskloop nowait  ! { dg-error "Unexpected junk" }
+
+!$omp masked taskloop simd
+do i = 1, 5
+end do
+!$omp end masked taskloop simd nowait  ! { dg-error "Unexpected junk" }
+
+!$omp master
+!$omp end master nowait  ! { dg-error "Unexpected junk" }
+
+!$omp master taskloop
+do i = 1, 5
+end do
+!$omp end master taskloop nowait  ! { dg-error "Unexpected junk" }
+
+!$omp master taskloop simd
+do i = 1, 5
+end do
+!$omp end master taskloop simd nowait  ! { dg-error "Unexpected junk" }
+
+!$omp ordered
+!$omp end ordered nowait  ! { dg-error "Unexpected junk" }
+
+!$omp parallel
+!$omp end parallel nowait  ! { dg-error "Unexpected junk" }
+
+!$omp parallel workshare
+a(:) = 5
+!$omp end parallel workshare nowait  ! { dg-error "Unexpected junk" }
+
+!$omp parallel do
+do i = 1, 5
+end do
+!$omp end parallel do nowait  ! { dg-error "Unexpected junk" }
+
+!$omp parallel do simd
+do i = 1, 5
+end do
+!$omp end parallel do simd nowait  ! { dg-error "Unexpected junk" }
+
+!$omp parallel masked
+!$omp end parallel masked nowait  ! { dg-error "Unexpected junk" }
+
+!$omp parallel masked taskloop
+do i = 1, 5
+end do
+!$omp end parallel masked taskloop nowait  ! { dg-error "Unexpected junk" }
+
+!$omp parallel masked taskloop simd
+do i = 1, 5
+end do
+!$omp end parallel masked taskloop simd nowait  ! { dg-error "Unexpected junk" }
+
+!$omp parallel master
+!$omp end parallel master nowait  ! { dg-error "Unexpected junk" }
+
+!$omp parallel master taskloop
+do i = 1, 5
+end do
+!$omp end parallel master taskloop nowait  ! { dg-error "Unexpected junk" }
+
+!$omp parallel master taskloop simd
+do i = 1, 5
+end do
+!$omp end parallel master taskloop simd nowait  ! { dg-error "Unexpected junk" }
+
+!$omp simd
+do i = 1, 5
+end do
+!$omp end simd nowait  ! { dg-error "Unexpected junk" }
+
+!$omp task
+!$omp end task nowait  ! { dg-error "Unexpected junk" }
+
+!$omp taskgroup
+!$omp end taskgroup nowait  ! { dg-error "Unexpected junk" }
+
+!$omp taskloop
+do i = 1, 5
+end do
+!$omp end taskloop nowait  ! { dg-error "Unexpected junk" }
+
+!$omp taskloop simd
+do i = 1, 5
+end do
+!$omp end taskloop simd nowait  ! { dg-error "Unexpected junk" }
+
+!$omp teams
+!$omp end teams nowait  ! { dg-error "Unexpected junk" }
+
+!$omp teams distribute
+do i = 1, 5
+end do
+!$omp end teams distribute nowait  ! { dg-error "Unexpected junk" }
+
+!$omp teams distribute parallel do
+do i = 1, 5
+end do
+!$omp end teams distribute parallel do nowait  ! { dg-error "Unexpected junk" }
+
+!$omp teams distribute parallel do simd
+do i = 1, 5
+end do
+!$omp end teams distribute parallel do simd nowait  ! { dg-error "Unexpected junk" }
+
+!$omp teams distribute simd
+do i = 1, 5
+end do
+!$omp end teams distribute simd nowait  ! { dg-error "Unexpected junk" }
+
+!$omp target data map(tofrom:i)
+!$omp end target data nowait  ! { dg-error "Unexpected junk" }
+
+end  ! { dg-error "Unexpected END statement" }
+! { dg-prune-output "Unexpected end of file" }
diff --git a/gcc/testsuite/gfortran.dg/gomp/nowait-3.f90 b/gcc/testsuite/gfortran.dg/gomp/nowait-3.f90
new file mode 100644
index 0000000..a58b8fd
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/gomp/nowait-3.f90
@@ -0,0 +1,118 @@
+! { dg-additional-options "-fdump-tree-original" }
+
+subroutine foo
+implicit none
+integer :: i, a(5)
+
+!$omp do
+do i = 1, 5
+end do
+!$omp end do nowait
+
+!$omp do simd
+do i = 1, 5
+end do
+!$omp end do simd nowait
+
+!$omp scope
+!$omp end scope nowait
+
+!$omp sections
+  !$omp section
+  block; end block
+!$omp end sections nowait
+
+!$omp single
+!$omp end single nowait
+
+!$omp target
+!$omp end target nowait
+
+!$omp target parallel
+!$omp end target parallel nowait
+
+!$omp target parallel do
+do i = 1, 5
+end do
+!$omp end target parallel do nowait
+
+!$omp target parallel do simd
+do i = 1, 5
+end do
+!$omp end target parallel do simd nowait
+
+!$omp target parallel loop
+do i = 1, 5
+end do
+!$omp end target parallel loop nowait
+
+!$omp target teams distribute parallel do
+do i = 1, 5
+end do
+!$omp end target teams distribute parallel do nowait
+
+!$omp target teams distribute parallel do simd
+do i = 1, 5
+end do
+!$omp end target teams distribute parallel do simd nowait
+
+!$omp target simd
+do i = 1, 5
+end do
+!$omp end target simd nowait
+
+!$omp target teams
+!$omp end target teams nowait
+
+!$omp target teams distribute
+do i = 1, 5
+end do
+!$omp end target teams distribute nowait
+
+!$omp target teams distribute simd
+do i = 1, 5
+end do
+!$omp end target teams distribute simd nowait
+
+!$omp target teams loop
+do i = 1, 5
+end do
+!$omp end target teams loop nowait
+
+!$omp workshare
+A(:) = 5
+!$omp end workshare nowait
+end
+
+! Note: internally, for '... parallel do ...', 'nowait' is always added
+! such that for 'omp end target parallel do nowait', 'nowait' is on both
+! 'target' as specified in the OpenMP spec and and on 'do' due to internal usage.
+
+! Expected with 'nowait'
+
+! { dg-final { scan-tree-dump-times "#pragma omp for nowait" 6 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp for schedule\\(static\\) nowait" 1 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp sections nowait" 1 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp single nowait" 1 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp target nowait" 12 "original" } }
+
+! Never:
+
+! { dg-final { scan-tree-dump-not "#pragma omp distribute\[^\n\r]*nowait" "original" } }
+! { dg-final { scan-tree-dump-not "#pragma omp loop\[^\n\r]*nowait" "original" } }
+! { dg-final { scan-tree-dump-not "#pragma omp parallel\[^\n\r]*nowait" "original" } }
+! { dg-final { scan-tree-dump-not "#pragma omp section\[^s\]\[^\n\r]*nowait" "original" } }
+! { dg-final { scan-tree-dump-not "#pragma omp simd\[^\n\r]*nowait" "original" } }
+! { dg-final { scan-tree-dump-not "#pragma omp teams\[^\n\r]*nowait" "original" } }
+
+! Sometimes or never with nowait:
+
+! { dg-final { scan-tree-dump-times "#pragma omp distribute\[\n\r]" 4 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp loop\[\n\r]" 2 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp parallel\[\n\r]" 6 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp section\[\n\r]" 1 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp simd linear\\(i:1\\)\[\n\r]" 5 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp teams\[\n\r]" 6 "original" } }
+
+! { dg-final { scan-tree-dump-times "#pragma omp target\[\n\r]" 0 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp for\[\n\r]" 0 "original" } }
diff --git a/gcc/testsuite/gfortran.dg/gomp/num-teams-1.f90 b/gcc/testsuite/gfortran.dg/gomp/num-teams-1.f90
new file mode 100644
index 0000000..df31cc7
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/gomp/num-teams-1.f90
@@ -0,0 +1,53 @@
+module m
+  implicit none (type, external)
+
+  interface
+  integer function fn(i); integer :: i; end
+  end interface
+
+contains
+
+subroutine foo
+  !$omp teams num_teams (4 : 6)
+  !$omp end teams
+
+  !$omp teams num_teams (7)
+  !$omp end teams
+end 
+
+subroutine bar
+  !$omp target teams num_teams (5 : 19)
+  !$omp end target teams
+
+  !$omp target teams num_teams (21)
+  !$omp end target teams
+end
+
+subroutine baz
+  !$omp teams num_teams (fn (1) : fn (2))
+  !$omp end teams
+
+  !$omp teams num_teams (fn (3))
+  !$omp end teams
+end
+
+subroutine qux
+  !$omp target teams num_teams (fn (4) : fn (5))
+  !$omp end target teams
+
+  !$omp target teams num_teams (fn (6))
+  !$omp end target teams
+end
+
+subroutine corge
+  !$omp target
+    !$omp teams num_teams (fn (7) : fn (8))
+    !$omp end teams
+  !$omp end target
+
+  !$omp target
+    !$omp teams num_teams (fn (9))
+    !$omp end teams
+  !$omp end target
+end
+end module m
diff --git a/gcc/testsuite/gfortran.dg/gomp/num-teams-2.f90 b/gcc/testsuite/gfortran.dg/gomp/num-teams-2.f90
new file mode 100644
index 0000000..e7814a1
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/gomp/num-teams-2.f90
@@ -0,0 +1,37 @@
+module m
+  implicit none (type, external)
+
+contains
+
+subroutine foo (i)
+  integer :: i
+
+  !$omp teams num_teams (6 : 4)		! { dg-warning "NUM_TEAMS lower bound at .1. larger than upper bound at .2." }
+  !$omp end teams
+
+  !$omp teams num_teams (-7)		! { dg-warning "INTEGER expression of NUM_TEAMS clause at .1. must be positive" }
+  !$omp end teams
+
+  !$omp teams num_teams (i : -7)		! { dg-warning "INTEGER expression of NUM_TEAMS clause at .1. must be positive" }
+  !$omp end teams
+
+  !$omp teams num_teams (-7 : 8)		! { dg-warning "INTEGER expression of NUM_TEAMS clause at .1. must be positive" }
+  !$omp end teams
+end
+
+subroutine bar (i)
+  integer :: i
+
+  !$omp target teams num_teams (6 : 4)	! { dg-warning "NUM_TEAMS lower bound at .1. larger than upper bound at .2." }
+  !$omp end target teams
+
+  !$omp target teams num_teams (-7)	! { dg-warning "INTEGER expression of NUM_TEAMS clause at .1. must be positive" }
+  !$omp end target teams
+
+  !$omp target teams num_teams (i : -7)	! { dg-warning "INTEGER expression of NUM_TEAMS clause at .1. must be positive" }
+  !$omp end target teams
+
+  !$omp target teams num_teams (-7 : 8)	! { dg-warning "INTEGER expression of NUM_TEAMS clause at .1. must be positive" }
+  !$omp end target teams
+end
+end module
diff --git a/gcc/testsuite/gfortran.dg/gomp/order-6.f90 b/gcc/testsuite/gfortran.dg/gomp/order-6.f90
index c8aeecb..8d7f9c5 100644
--- a/gcc/testsuite/gfortran.dg/gomp/order-6.f90
+++ b/gcc/testsuite/gfortran.dg/gomp/order-6.f90
@@ -8,7 +8,7 @@ module m
     end
     integer function omp_get_num_threads ()
     end
-    integer function omp_target_is_present (x, i)
+    integer function omp_target_is_present (x, i) bind(c)
       import :: c_ptr
       type(c_ptr) :: x
       integer, value :: i
diff --git a/gcc/testsuite/gfortran.dg/gomp/teams-3.f90 b/gcc/testsuite/gfortran.dg/gomp/teams-3.f90
new file mode 100644
index 0000000..fac3393
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/gomp/teams-3.f90
@@ -0,0 +1,65 @@
+! PR middle-end/102972
+
+module m
+implicit none (type, external)
+interface
+subroutine omp_set_num_teams (num_teams); integer :: num_teams; end
+subroutine omp_set_teams_thread_limit (thread_limit); integer :: thread_limit; end
+subroutine omp_set_num_teams_8 (num_teams); integer(8) :: num_teams; end
+subroutine omp_set_num_teams_9 (num_teams); integer :: num_teams; end
+subroutine omp_set_teams_thread_limit_8 (thread_limit); integer(8) :: thread_limit; end
+integer function omp_get_num_teams (); end
+integer function omp_get_team_size (level); integer :: level; end
+integer function omp_get_team_num (); end
+integer function omp_get_max_teams (); end
+integer function omp_get_teams_thread_limit (); end
+logical function omp_is_initial_device (); end
+integer function omp_get_num_threads (); end
+end interface
+
+contains
+
+subroutine valid ()
+  integer :: i, n
+  !$omp teams
+    !$omp distribute
+    do i = 1, 64
+    end do
+
+    n = omp_get_num_teams ()
+    if (n >= omp_get_team_num ()) &
+      error stop
+
+    !$omp parallel do
+    do i = 1, 64
+      if (.not.omp_is_initial_device () .or. omp_get_num_threads () < 0) &
+        error stop
+    end do
+
+    !$omp loop
+    do i = 1, 64
+    end do
+  !$omp end teams
+end
+
+subroutine invalid_nest ()
+  integer :: i, n
+  !$omp teams
+    !$omp distribute parallel do simd
+    do i = 1, 64
+    end do
+
+    n = 0
+    n = n + omp_get_team_size (0)  ! { dg-error "OpenMP runtime API call '\[^\n\r]*omp_get_team_size\[^\n\r]*' strictly nested in a 'teams' region" }
+    n = n + omp_get_num_teams ()
+    n = n + omp_get_team_num ()
+    call omp_set_num_teams (n)  ! { dg-error "OpenMP runtime API call '\[^\n\r]*omp_set_num_teams\[^\n\r]*' strictly nested in a 'teams' region" }
+    call omp_set_num_teams_8 (4_8)  ! { dg-error "OpenMP runtime API call '\[^\n\r]*omp_set_num_teams_8\[^\n\r]*' strictly nested in a 'teams' region" }
+    call omp_set_num_teams_9 (4)  ! OK - but misnamed user function
+    n = n + omp_get_max_teams ()  ! { dg-error "OpenMP runtime API call '\[^\n\r]*omp_get_max_teams\[^\n\r]*' strictly nested in a 'teams' region" }
+    n = n + omp_get_teams_thread_limit ()  ! { dg-error "OpenMP runtime API call '\[^\n\r]*omp_get_teams_thread_limit\[^\n\r]*' strictly nested in a 'teams' region" }
+    call omp_set_teams_thread_limit (n)  ! { dg-error "OpenMP runtime API call '\[^\n\r]*omp_set_teams_thread_limit'\[^\n\r]* strictly nested in a 'teams' region" }
+    call omp_set_teams_thread_limit_8 (3_8)  ! { dg-error "OpenMP runtime API call '\[^\n\r]*omp_set_teams_thread_limit_8'\[^\n\r]* strictly nested in a 'teams' region" }
+  !$omp end teams
+end
+end module
diff --git a/gcc/testsuite/gfortran.dg/gomp/teams-4.f90 b/gcc/testsuite/gfortran.dg/gomp/teams-4.f90
new file mode 100644
index 0000000..422c2c9
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/gomp/teams-4.f90
@@ -0,0 +1,47 @@
+! PR middle-end/102972
+
+module m
+implicit none (type, external)
+
+! Note: Those are module functions - not an interface
+! Hence, they are internally manged to contain the module name!
+
+contains
+
+subroutine omp_set_num_teams (num_teams); integer :: num_teams; end
+subroutine omp_set_teams_thread_limit (thread_limit); integer :: thread_limit; end
+subroutine omp_set_num_teams_8 (num_teams); integer(8) :: num_teams; end
+subroutine omp_set_num_teams_9 (num_teams); integer :: num_teams; end
+subroutine omp_set_teams_thread_limit_8 (thread_limit); integer(8) :: thread_limit; end
+integer function omp_get_num_teams (); omp_get_num_teams = 0; end
+integer function omp_get_team_size (level); integer :: level; omp_get_team_size = 0; end
+integer function omp_get_team_num (); omp_get_team_num = 0; end
+integer function omp_get_max_teams (); omp_get_max_teams = 0; end
+integer function omp_get_teams_thread_limit (); omp_get_teams_thread_limit = 0; end
+logical function omp_is_initial_device (); omp_is_initial_device = .true.; end
+integer function omp_get_num_threads (); omp_get_num_threads = 0; end
+end module
+
+subroutine nest_test ()
+  use m
+  implicit none (type, external)
+  
+  integer :: i, n
+  !$omp teams
+    !$omp distribute parallel do simd
+    do i = 1, 64
+    end do
+
+    n = 0
+    n = n + omp_get_team_size (0)
+    n = n + omp_get_num_teams ()
+    n = n + omp_get_team_num ()
+    call omp_set_num_teams (n)
+    call omp_set_num_teams_8 (4_8)
+    call omp_set_num_teams_9 (4)
+    n = n + omp_get_max_teams ()
+    n = n + omp_get_teams_thread_limit ()
+    call omp_set_teams_thread_limit (n)
+    call omp_set_teams_thread_limit_8 (3_8)
+  !$omp end teams
+end
diff --git a/gcc/testsuite/gfortran.dg/gomp/unexpected-end.f90 b/gcc/testsuite/gfortran.dg/gomp/unexpected-end.f90
index d2e8daa..70f54f9 100644
--- a/gcc/testsuite/gfortran.dg/gomp/unexpected-end.f90
+++ b/gcc/testsuite/gfortran.dg/gomp/unexpected-end.f90
@@ -16,12 +16,12 @@
 
 !$omp end DO SIMD  ! { dg-error "Unexpected !.OMP END DO SIMD" }
 
-!$omp end LOOP  ! { dg-error "Unclassifiable OpenMP directive" }
+!$omp end LOOP  ! { dg-error "Unexpected !.OMP END LOOP" }
 
 !$omp parallel loop
 do i = 1, 5
 end do
-!$omp end LOOP  ! { dg-error "Unclassifiable OpenMP directive" }
+!$omp end LOOP  ! { dg-error "Unexpected !.OMP END LOOP" }
 
 !$omp end MASKED  ! { dg-error "Unexpected !.OMP END MASKED" }
 
@@ -44,7 +44,7 @@ end do
 !$omp end PARALLEL DO SIMD  ! { dg-error "Unexpected !.OMP END PARALLEL DO SIMD" }
 
 !$omp loop
-!$omp end PARALLEL LOOP  ! { dg-error "Unexpected junk" }
+!$omp end PARALLEL LOOP  ! { dg-error "Unexpected !.OMP END PARALLEL LOOP" }
 
 !$omp end PARALLEL MASKED  ! { dg-error "Unexpected !.OMP END PARALLEL MASKED" }
 
@@ -80,7 +80,7 @@ end do
 
 !$omp end TARGET PARALLEL DO SIMD  ! { dg-error "Unexpected !.OMP END TARGET PARALLEL DO SIMD" }
 
-!$omp end TARGET PARALLEL LOOP  ! { dg-error "Unexpected junk" }
+!$omp end TARGET PARALLEL LOOP  ! { dg-error "Unexpected !.OMP END TARGET PARALLEL LOOP" }
 
 !$omp end TARGET SIMD  ! { dg-error "Unexpected !.OMP END TARGET SIMD" }
 
@@ -94,7 +94,7 @@ end do
 
 !$omp end TARGET TEAMS DISTRIBUTE SIMD  ! { dg-error "Unexpected !.OMP END TARGET TEAMS DISTRIBUTE SIMD" }
 
-!$omp end TARGET TEAMS LOOP  ! { dg-error "Unexpected junk" }
+!$omp end TARGET TEAMS LOOP  ! { dg-error "Unexpected !.OMP END TARGET TEAMS LOOP" }
 
 !$omp end TASK  ! { dg-error "Unexpected !.OMP END TASK" }
 
@@ -114,10 +114,10 @@ end do
 
 !$omp end TEAMS DISTRIBUTE SIMD  ! { dg-error "Unexpected !.OMP END TEAMS DISTRIBUTE SIMD" }
 
-!$omp end TEAMS LOOP  ! { dg-error "Unexpected junk" }
+!$omp end TEAMS LOOP  ! { dg-error "Unexpected !.OMP END TEAMS LOOP" }
 
 !$omp end WORKSHARE  ! { dg-error "Unexpected !.OMP END WORKSHARE" }
 
 end  ! { dg-error "Unexpected END statement" }
 
-! { dg-excess-errors "Unexpected end of file" }
+! { dg-error "Unexpected end of file" "" { target "*-*-*" } 0 }
diff --git a/gcc/testsuite/gfortran.dg/implicit_14.f90 b/gcc/testsuite/gfortran.dg/implicit_14.f90
index 8282c1f..422d913 100644
--- a/gcc/testsuite/gfortran.dg/implicit_14.f90
+++ b/gcc/testsuite/gfortran.dg/implicit_14.f90
@@ -4,5 +4,5 @@
 ! Support Fortran 2018's IMPLICIT NONE with spec list
 ! (currently implemented as vendor extension)
 
-implicit none (type) ! { dg-error "Fortran 2018: IMPORT NONE with spec list at \\(1\\)" }
+implicit none (type) ! { dg-error "Fortran 2018: IMPLICIT NONE with spec list at \\(1\\)" }
 end
diff --git a/gcc/testsuite/gfortran.dg/index_5.f90 b/gcc/testsuite/gfortran.dg/index_5.f90
new file mode 100644
index 0000000..e039455
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/index_5.f90
@@ -0,0 +1,23 @@
+! { dg-do compile }
+!
+! PR fortran/97896
+! An ICE occured with INDEX when the KIND argument was present
+! because of a mismatch between the number of arguments expected
+! during the scalarization process and the number of arguments actually
+! used.
+!
+! Test contributed by Harald Anlauf <anlauf@gcc.gnu.org>, based on an initial
+! submission by G. Steinmetz <gscfq@t-online.de>.
+
+program p
+  implicit none
+  logical    :: a(2)
+  integer    :: b(2)
+  integer(8) :: d(2)
+  b = index ('xyxyz','yx', back=a)
+  b = index ('xyxyz','yx', back=a, kind=4)
+  d = index ('xyxyz','yx', back=a, kind=8)
+  b = index ('xyxyz','yx', back=a, kind=8)
+  d = index ('xyxyz','yx', back=a, kind=4)
+end
+
diff --git a/gcc/testsuite/gfortran.dg/inline_matmul_17.f90 b/gcc/testsuite/gfortran.dg/inline_matmul_17.f90
index d2ca8e2..cff4b6c 100644
--- a/gcc/testsuite/gfortran.dg/inline_matmul_17.f90
+++ b/gcc/testsuite/gfortran.dg/inline_matmul_17.f90
@@ -45,4 +45,4 @@ program main
   c = matmul(a, bfunc())
   if (any(c-d /= 0)) STOP 6
 end program main
-! { dg-final { scan-tree-dump-times "matmul_r4" 2 "optimized" } }
+! { dg-final { scan-tree-dump-not "matmul_r4" "optimized" } }
diff --git a/gcc/testsuite/gfortran.dg/interface_operator_1.f90 b/gcc/testsuite/gfortran.dg/interface_operator_1.f90
index 97d260a..6684751 100644
--- a/gcc/testsuite/gfortran.dg/interface_operator_1.f90
+++ b/gcc/testsuite/gfortran.dg/interface_operator_1.f90
@@ -7,4 +7,4 @@ program p
    interface operator ( .gt. )
    end interface operator        ! { dg-error "END INTERFACE OPERATOR" }
 end program p                    ! { dg-error "END INTERFACE" }
-! { dg-excess-errors "Unexpected end of file" }
+! { dg-error "Unexpected end of file" "" { target "*-*-*" } 0 }
diff --git a/gcc/testsuite/gfortran.dg/interface_operator_2.f90 b/gcc/testsuite/gfortran.dg/interface_operator_2.f90
index a739596..11b9ff5 100644
--- a/gcc/testsuite/gfortran.dg/interface_operator_2.f90
+++ b/gcc/testsuite/gfortran.dg/interface_operator_2.f90
@@ -7,4 +7,4 @@ program p
    interface operator ( .gt. )
    end interface operator (.lt.) ! { dg-error "END INTERFACE OPERATOR" }
 end program p                    ! { dg-error "END INTERFACE" }
-! { dg-excess-errors "Unexpected end of file" }
+! { dg-error "Unexpected end of file" "" { target "*-*-*" } 0 }
diff --git a/gcc/testsuite/gfortran.dg/intrinsic_short-long.f90 b/gcc/testsuite/gfortran.dg/intrinsic_short-long.f90
new file mode 100644
index 0000000..3f3f3cc
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/intrinsic_short-long.f90
@@ -0,0 +1,11 @@
+! { dg-do compile }
+!
+! Checking for removal of SHORT and LONG intrinsics.
+!
+  real,parameter :: a=3.1415927
+  integer :: i
+
+  i=SHORT(a) ! { dg-error "has been removed" }
+  i=LONG(a)  ! { dg-error "has been removed" }
+
+  end
diff --git a/gcc/testsuite/gfortran.dg/line_length_4.f90 b/gcc/testsuite/gfortran.dg/line_length_4.f90
index 6e3c76e..8004040 100644
--- a/gcc/testsuite/gfortran.dg/line_length_4.f90
+++ b/gcc/testsuite/gfortran.dg/line_length_4.f90
@@ -16,4 +16,4 @@
    end subroutine foo
   end
 ! { dg-error "Line truncated" " " { target *-*-* } 8 }
-! { dg-excess-errors "some warnings being treated as errors" }
+! { dg-prune-output "some warnings being treated as errors" }
diff --git a/gcc/testsuite/gfortran.dg/line_length_5.f90 b/gcc/testsuite/gfortran.dg/line_length_5.f90
index d7aca12..8183245 100644
--- a/gcc/testsuite/gfortran.dg/line_length_5.f90
+++ b/gcc/testsuite/gfortran.dg/line_length_5.f90
@@ -4,4 +4,4 @@ print *, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
 end 
 ! { dg-error "Line truncated" " " { target *-*-* } 3 }
 ! { dg-error "Unterminated character constant" " " { target *-*-* } 3 }
-! { dg-excess-errors "some warnings being treated as errors" }
+! { dg-prune-output "some warnings being treated as errors" }
diff --git a/gcc/testsuite/gfortran.dg/line_length_6.f90 b/gcc/testsuite/gfortran.dg/line_length_6.f90
index 92f3401..8cdb0209 100644
--- a/gcc/testsuite/gfortran.dg/line_length_6.f90
+++ b/gcc/testsuite/gfortran.dg/line_length_6.f90
@@ -5,4 +5,4 @@
 !
       print *, 1                                                                                                                                          + 2 ! { dg-error "Line truncated at .1." }
       end
-! { dg-excess-errors "some warnings being treated as errors" }
+! { dg-prune-output "some warnings being treated as errors" }
diff --git a/gcc/testsuite/gfortran.dg/line_length_8.f90 b/gcc/testsuite/gfortran.dg/line_length_8.f90
index 3f0efaf..afd6cc2 100644
--- a/gcc/testsuite/gfortran.dg/line_length_8.f90
+++ b/gcc/testsuite/gfortran.dg/line_length_8.f90
@@ -6,4 +6,4 @@
 !
       print *, 1                                                                                                                                          + 2 ! { dg-error "Line truncated at .1." }
       end
-! { dg-excess-errors "some warnings being treated as errors" }
+! { dg-prune-output "some warnings being treated as errors" }
diff --git a/gcc/testsuite/gfortran.dg/line_length_9.f90 b/gcc/testsuite/gfortran.dg/line_length_9.f90
index f338972..6c156af 100644
--- a/gcc/testsuite/gfortran.dg/line_length_9.f90
+++ b/gcc/testsuite/gfortran.dg/line_length_9.f90
@@ -6,4 +6,4 @@
 !
       print *, 1                                                                                                                                          + 2 ! { dg-error "Line truncated at .1." }
       end
-! { dg-excess-errors "some warnings being treated as errors" }
+! { dg-prune-output "some warnings being treated as errors" }
diff --git a/gcc/testsuite/gfortran.dg/lto/bind-c-char_0.f90 b/gcc/testsuite/gfortran.dg/lto/bind-c-char_0.f90
new file mode 100644
index 0000000..48b495b
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/lto/bind-c-char_0.f90
@@ -0,0 +1,49 @@
+! { dg-lto-do link }
+! { dg-lto-options {{ -O0 -flto }} }
+!
+! PR fortran/102885
+
+module m
+  use iso_c_binding, only: c_char
+  implicit none (type, external)
+
+contains
+
+! Assumed-shape array, nonallocatable/nonpointer
+
+subroutine ar3 (xn, n) bind(C)
+  integer :: n
+  character(len=n) :: xn(..)
+  if (size(xn) /= 6) stop
+  if (len(xn) /= 5) stop  
+  select rank(xn)
+    rank(1)
+      xn = ['FDGhf', &
+            'hdrhg', &
+            'fDgFl', &
+            'DFHs3', &
+            '4a54G', &
+            'hSs6k']
+  rank default
+    stop
+  end select
+end
+
+end
+
+program main
+  use m
+  implicit none (type, external)
+  character(kind=c_char, len=5) :: str5a6(6)
+
+  ! assumed rank - with array descriptor
+
+  str5a6 = ['DDGhf', &
+            'hdrh$', &
+            'fDGSl', &
+            'DFHs3', &
+            '43grG', &
+            'hFG$k']
+  call ar3 (str5a6, 5)
+
+end
diff --git a/gcc/testsuite/gfortran.dg/pdt_32.f03 b/gcc/testsuite/gfortran.dg/pdt_32.f03
new file mode 100644
index 0000000..f8d4041
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pdt_32.f03
@@ -0,0 +1,17 @@
+! { dg-do compile }
+! PR fortran/102956
+! PDT KIND and LEN type parameters are mutually exclusive (F2018:R734)
+!
+module m
+  type :: good_pdt (k,l)
+     integer, kind           :: k = 1
+     integer, len            :: l = 1
+     character(kind=k,len=l) :: c
+  end type good_pdt
+
+  type :: bad_pdt (k,l)               ! { dg-error "does not have a component" }
+     integer, kind, len      :: k = 1 ! { dg-error "attribute conflicts with" }
+     integer, len, kind      :: l = 1 ! { dg-error "attribute conflicts with" }
+     character(kind=k,len=l) :: c     ! { dg-error "has not been declared" }
+  end type bad_pdt
+end
diff --git a/gcc/testsuite/gfortran.dg/pdt_4.f03 b/gcc/testsuite/gfortran.dg/pdt_4.f03
index c1af65a..37412e4 100644
--- a/gcc/testsuite/gfortran.dg/pdt_4.f03
+++ b/gcc/testsuite/gfortran.dg/pdt_4.f03
@@ -28,9 +28,9 @@ end module
 
   type :: bad_pdt (a,b, c, d)  ! { dg-error "does not have a component" }
     real, kind :: a            ! { dg-error "must be INTEGER" }
-    INTEGER(8), kind :: b      ! { dg-error "be default integer kind" }
+    INTEGER(8), kind :: b
     real, LEN :: c             ! { dg-error "must be INTEGER" }
-    INTEGER(8), LEN :: d       ! { dg-error "be default integer kind" }
+    INTEGER(8), LEN :: d
   end type
 
   type :: mytype (a,b)
diff --git a/gcc/testsuite/gfortran.dg/pr102715.f90 b/gcc/testsuite/gfortran.dg/pr102715.f90
new file mode 100644
index 0000000..7b29a1c
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pr102715.f90
@@ -0,0 +1,13 @@
+! { dg-do compile }
+! PR fortran/102715 - ICE in gfc_simplify_transpose
+
+program p
+  type t
+  end type
+  type(t), parameter :: a(4)   = t()
+  type(t), parameter :: b(2,2) = reshape(a, [2]) ! { dg-error "Rank mismatch" }
+  type(t), parameter :: c(2,2) = transpose(b)    ! { dg-error "must be of rank 2" }
+  type(t), parameter :: s2(*)  = b(2,:)          ! { dg-error "Syntax error" }
+  type(t), parameter :: x(*,*) = reshape(a, [2]) ! { dg-error "Rank mismatch" }
+  type(t), parameter :: s3(*)  = x(2,:)          ! { dg-error "Syntax error" }
+end
diff --git a/gcc/testsuite/gfortran.dg/pr102816.f90 b/gcc/testsuite/gfortran.dg/pr102816.f90
new file mode 100644
index 0000000..4683174
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pr102816.f90
@@ -0,0 +1,9 @@
+! { dg-do compile }
+! PR fortran/102816
+
+program p
+  type t
+     integer :: a([2])     ! { dg-error "must be scalar" }
+  end type
+  type(t) :: x = t([3, 4]) ! { dg-error "Bad array spec of component" }
+end
diff --git a/gcc/testsuite/gfortran.dg/pr102817.f90 b/gcc/testsuite/gfortran.dg/pr102817.f90
new file mode 100644
index 0000000..c081a69
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pr102817.f90
@@ -0,0 +1,17 @@
+! { dg-do compile }
+! PR fortran/102817 - ICE in gfc_clear_shape
+
+program test
+  type t
+     integer :: a(1,2) = 3
+  end type t
+  type(t), parameter :: u    = t(4)
+  type(t), parameter :: x(1) = t(4)
+  integer, parameter :: p(1,2) = (x(1)%a)
+  integer            :: z(1,2) = (x(1)%a) 
+  integer            :: y(1,2), v(1,2), w(1,2)
+  v = (u   %a)
+  w =  x(1)%a
+  y = (x(1)%a)
+  print *, v, w, y, z, p
+end
diff --git a/gcc/testsuite/gfortran.dg/pr24823.f b/gcc/testsuite/gfortran.dg/pr24823.f
index c6f638f..93cd8a3 100644
--- a/gcc/testsuite/gfortran.dg/pr24823.f
+++ b/gcc/testsuite/gfortran.dg/pr24823.f
@@ -61,8 +61,8 @@
                   IF( ISYM.EQ.0 ) THEN
                   END IF
                END IF
-               A( I-J+KUU+1, J ) = ZLATM2( M, N, I, J, KL, KU,
-     $              DR, IPVTNG, IWORK, SPARSE )
+               A( I-J+KUU+1, J ) = ZLATM2( M, N, I, J, KL, KU,  ! { dg-warning "More actual than formal" }
+     $              DR, IPVTNG, IWORK, SPARSE )  ! { dg-warning "Type mismatch" }
             END IF
          END IF
       END IF
diff --git a/gcc/testsuite/gfortran.dg/pr39937.f b/gcc/testsuite/gfortran.dg/pr39937.f
index 17d3eb4..ed28693 100644
--- a/gcc/testsuite/gfortran.dg/pr39937.f
+++ b/gcc/testsuite/gfortran.dg/pr39937.f
@@ -20,7 +20,7 @@ C { dg-options "-std=legacy" }
           END IF
           CALL DLALN2( .FALSE., 2, 2, SMIN, ONE,
      $                            T( J-1, J-1 ), LDT, ONE, ONE,  ! { dg-warning "Type mismatch" }
-     $                            XNORM, IERR )
+     $                            XNORM, IERR )  ! { dg-warning "Type mismatch" }
           CALL DAXPY( J-2, -X( 1, 1 ), T( 1, J-1 ), 1,
      $                           WORK( 1+N ), 1 )
           CALL DAXPY( J-2, -X( 2, 2 ), T( 1, J ), 1,
diff --git a/gcc/testsuite/gfortran.dg/pr41011.f b/gcc/testsuite/gfortran.dg/pr41011.f
index 5a32185..c032310 100644
--- a/gcc/testsuite/gfortran.dg/pr41011.f
+++ b/gcc/testsuite/gfortran.dg/pr41011.f
@@ -1,6 +1,6 @@
 ! { dg-do compile }
 ! { dg-options "-O3 -std=legacy" }
-      CALL UVSET(NX,NY,NZ,HVAR,ZET,NP,DZ,DKM,UM,VM,UG,VG,TM,DCDX, ! { dg-warning "Rank mismatch" }
+      CALL UVSET(NX,NY,NZ,HVAR,ZET,NP,DZ,DKM,UM,VM,UG,VG,TM,DCDX, ! { dg-warning "Rank mismatch|Invalid procedure argument" }
      *ITY,ISH,NSMT,F)
          CALL DCTDX(NX,NY,NX1,NFILT,C(MLAG),DCDX(MLAG),HELP,HELPA,
      *   HELP,HELPA,FY,FYC,SAVEY)
@@ -18,6 +18,6 @@
      *WORK(*)
       IF(IH.EQ.0) THEN
          CALL PADEC(DKM,VM,HVAR,WORK(LWM),WORK(LWG), ! { dg-warning "Rank mismatch" }
-     *   WORK(LF),NZ,WORK(LA),WORK(LB),WORK(LC),ITY)
+     *   WORK(LF),NZ,WORK(LA),WORK(LB),WORK(LC),ITY) ! { dg-warning "Type mismatch" }
       ENDIF
       END
diff --git a/gcc/testsuite/gfortran.dg/pr61318.f90 b/gcc/testsuite/gfortran.dg/pr61318.f90
index 57da52d..7752ecd 100644
--- a/gcc/testsuite/gfortran.dg/pr61318.f90
+++ b/gcc/testsuite/gfortran.dg/pr61318.f90
@@ -18,5 +18,5 @@ end module gbl_interfaces
 program test
   use gbl_message
   use gbl_interfaces
-  call gagout(seve%e,'Some string') ! { dg-error "Type mismatch in argument" }
+  call gagout(seve%e,'Some string') ! { dg-error "Type mismatch in argument|More actual than formal" }
 end program test
diff --git a/gcc/testsuite/gfortran.dg/pr65045.f90 b/gcc/testsuite/gfortran.dg/pr65045.f90
index 5b838d5..c496529 100644
--- a/gcc/testsuite/gfortran.dg/pr65045.f90
+++ b/gcc/testsuite/gfortran.dg/pr65045.f90
@@ -12,4 +12,4 @@ i:block
 end block i      ! { dg-error "Expecting END PROGRAM statement" }
 print*,i         ! { dg-error "not appropriate for an expression" }
 end
-! { dg-excess-errors "Unexpected end of file" }
+! { dg-error "Unexpected end of file" "" { target "*-*-*" } 0 }
diff --git a/gcc/testsuite/gfortran.dg/pr68019.f90 b/gcc/testsuite/gfortran.dg/pr68019.f90
index 2e304c3..77fd55b 100644
--- a/gcc/testsuite/gfortran.dg/pr68019.f90
+++ b/gcc/testsuite/gfortran.dg/pr68019.f90
@@ -9,5 +9,5 @@ program p
       integer :: n
    end type
    type(t), parameter :: vec(*) = [(t(i), i = 1, 4)]
-   type(t), parameter :: arr(*) = reshape(vec, [2, 2])   ! { dg-error "ranks 1 and 2 in assignment" }
+   type(t), parameter :: arr(*) = reshape(vec, [2, 2])   ! { dg-error "Rank mismatch" }
 end
diff --git a/gcc/testsuite/gfortran.dg/pr69419.f90 b/gcc/testsuite/gfortran.dg/pr69419.f90
new file mode 100644
index 0000000..7329808
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pr69419.f90
@@ -0,0 +1,9 @@
+! { dg-do compile }
+! { dg-options "-fcoarray=lib" }
+! PR fortran/69419 - ICE on invalid coarray in common
+
+blockdata b
+   real x           ! { dg-error "must be in COMMON" }
+   common /c/ x[*]  ! { dg-error "cannot be a coarray" }
+   data x /1.0/
+end
diff --git a/gcc/testsuite/gfortran.dg/pr69497.f90 b/gcc/testsuite/gfortran.dg/pr69497.f90
index 1698ebb..291d906 100644
--- a/gcc/testsuite/gfortran.dg/pr69497.f90
+++ b/gcc/testsuite/gfortran.dg/pr69497.f90
@@ -5,4 +5,4 @@ program p
    do
    end block ! { dg-error "Expecting END DO statement" }
 end ! { dg-error "END DO statement expected" }
-! { dg-excess-errors "Unexpected end of file" }
+! { dg-error "Unexpected end of file" "" { target "*-*-*" } 0 }
diff --git a/gcc/testsuite/gfortran.dg/pr86551.f90 b/gcc/testsuite/gfortran.dg/pr86551.f90
new file mode 100644
index 0000000..d96e17a
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pr86551.f90
@@ -0,0 +1,12 @@
+! { dg-do compile }
+! PR fortran/86551 - ICE on invalid code with select type / end select type
+
+subroutine b
+  type :: t1
+  end type t1
+  class(t1) :: c2
+  select type (d => c2)
+  end select type       ! { dg-error "Syntax error" }
+end                     ! { dg-error "END SELECT statement expected" }
+
+! { dg-prune-output "Unexpected end of file" }
diff --git a/gcc/testsuite/gfortran.dg/pr99853.f90 b/gcc/testsuite/gfortran.dg/pr99853.f90
new file mode 100644
index 0000000..421a656
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pr99853.f90
@@ -0,0 +1,29 @@
+! { dg-do compile }
+! { dg-options "-std=f2018" }
+! PR fortran/99853
+
+subroutine s1 ()
+  select case (.true.) ! { dg-error "Cannot convert" }
+  case (1_8)           ! { dg-error "must be of type LOGICAL" }
+  end select
+end
+
+subroutine s2 ()
+  select case (.false._1) ! { dg-error "Cannot convert" }
+  case (2:3)              ! { dg-error "must be of type LOGICAL" }
+  end select
+end
+
+subroutine s3 ()
+  select case (3_2) ! { dg-error "Cannot convert" }
+  case (.false.)    ! { dg-error "must be of type INTEGER" }
+  end select
+end
+
+subroutine s4 (i)
+  select case (i) ! { dg-error "Cannot convert" }
+  case (.true._8) ! { dg-error "must be of type INTEGER" }
+  end select
+end
+
+! { dg-prune-output "Cannot convert" }
diff --git a/gcc/testsuite/gfortran.dg/submodule_21.f08 b/gcc/testsuite/gfortran.dg/submodule_21.f08
index c96acb2..bb62600 100644
--- a/gcc/testsuite/gfortran.dg/submodule_21.f08
+++ b/gcc/testsuite/gfortran.dg/submodule_21.f08
@@ -16,4 +16,5 @@ PROGRAM MyProg
   USE MainModule
   WRITE(*,*) a
 END PROGRAM MyProg
-! { dg-excess-errors "does not contain a MODULE PROCEDURE" }
+! { dg-error "does not contain a MODULE PROCEDURE" "" { target "*-*-*" } 0 }
+! { dg-prune-output "compilation terminated" }
diff --git a/gcc/testsuite/gfortran.dg/tab_continuation.f b/gcc/testsuite/gfortran.dg/tab_continuation.f
index 85d2307..719b03f 100644
--- a/gcc/testsuite/gfortran.dg/tab_continuation.f
+++ b/gcc/testsuite/gfortran.dg/tab_continuation.f
@@ -17,4 +17,4 @@
 ! { dg-error "Nonconforming tab character in column 1 of line 13" "Nonconforming tab" { target *-*-* } 0 }
 ! { dg-error "Nonconforming tab character in column 1 of line 14" "Nonconforming tab" { target *-*-* } 0 }
 ! { dg-error "Nonconforming tab character in column 1 of line 15" "Nonconforming tab" { target *-*-* } 0 }
-! { dg-excess-errors "some warnings being treated as errors" }
+! { dg-prune-output "some warnings being treated as errors" }
diff --git a/gcc/testsuite/gfortran.dg/typebound_proc_2.f90 b/gcc/testsuite/gfortran.dg/typebound_proc_2.f90
index a34d935..14f01ef 100644
--- a/gcc/testsuite/gfortran.dg/typebound_proc_2.f90
+++ b/gcc/testsuite/gfortran.dg/typebound_proc_2.f90
@@ -9,24 +9,23 @@ MODULE testmod
 
   TYPE t
     INTEGER :: x
-  CONTAINS ! { dg-error "Fortran 2003" }
-    PROCEDURE proc1 ! { dg-error "Fortran 2003" }
-    PROCEDURE :: proc2 => p2 ! { dg-error "Fortran 2003" }
-  END TYPE t
+  CONTAINS ! { dg-error "Fortran 2003: CONTAINS block in derived type definition" }
+    PROCEDURE proc1 ! { dg-error "Fortran 2003: PROCEDURE statement" }
+    PROCEDURE :: proc2 => p2 ! { dg-error "Fortran 2003: PROCEDURE statement" }
+  END TYPE t                 ! { dg-error "Fortran 2008: Derived type definition at .1. with empty CONTAINS section" }
 
 CONTAINS
   
-  SUBROUTINE proc1 (me)
+  SUBROUTINE proc1 (me) ! { dg-error "no IMPLICIT type" }
     IMPLICIT NONE
-    TYPE(t1) :: me
+    TYPE(t1) :: me     ! { dg-error "being used before it is defined" }
   END SUBROUTINE proc1
 
-  REAL FUNCTION proc2 (me, x)
+  REAL FUNCTION proc2 (me, x) ! { dg-error "no IMPLICIT type" }
     IMPLICIT NONE
-    TYPE(t1) :: me
+    TYPE(t1) :: me     ! { dg-error "being used before it is defined" }
     REAL :: x
     proc2 = x / 2
   END FUNCTION proc2
 
 END MODULE testmod
-! { dg-excess-errors "no IMPLICIT type" }
diff --git a/gcc/testsuite/gfortran.dg/ubsan/bind-c-intent-out-2.f90 b/gcc/testsuite/gfortran.dg/ubsan/bind-c-intent-out-2.f90
new file mode 100644
index 0000000..fe8f606
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/ubsan/bind-c-intent-out-2.f90
@@ -0,0 +1,39 @@
+! { dg-do run }
+! { dg-additional-options "-fsanitize=undefined -fcheck=all" }
+
+! PR fortran/92621
+
+subroutine hello(val) bind(c)
+  use, intrinsic :: iso_c_binding, only: c_int
+
+  implicit none
+  
+  integer(kind=c_int), allocatable, intent(out) :: val(:)
+
+  allocate(val(1))
+  val = 2
+  return
+end subroutine hello
+
+program alloc_p
+
+  use, intrinsic :: iso_c_binding, only: c_int
+
+  implicit none
+
+  interface
+    subroutine hello(val) bind(c)
+      import :: c_int
+      implicit none
+      integer(kind=c_int), allocatable, intent(out) :: val(:)
+    end subroutine hello
+  end interface
+
+  integer(kind=c_int), allocatable :: a(:)
+
+  allocate(a(1))
+  a = 1
+  call hello(a)
+  stop
+
+end program alloc_p
diff --git a/gcc/testsuite/gfortran.dg/warnings_are_errors_1.f90 b/gcc/testsuite/gfortran.dg/warnings_are_errors_1.f90
index 150f234..1f9d38d 100644
--- a/gcc/testsuite/gfortran.dg/warnings_are_errors_1.f90
+++ b/gcc/testsuite/gfortran.dg/warnings_are_errors_1.f90
@@ -22,4 +22,4 @@
 
        end
 ! { dg-final { output-exists-not } }
-! { dg-excess-errors "warnings being treated as errors" }
+! { dg-prune-output "warnings being treated as errors" }
diff --git a/gcc/testsuite/jit.dg/test-sum-of-squares.c b/gcc/testsuite/jit.dg/test-sum-of-squares.c
index f095f41..cfcf5b2 100644
--- a/gcc/testsuite/jit.dg/test-sum-of-squares.c
+++ b/gcc/testsuite/jit.dg/test-sum-of-squares.c
@@ -133,10 +133,10 @@ verify_code (gcc_jit_context *ctxt, gcc_jit_result *result)
      An example of using gcc_jit_context_enable_dump to verify a property
      of the compile.
 
-     In this case, verify that vrp is able to deduce the
-     bounds of the iteration variable. Specifically, verify that some
-     variable is known to be in the range negative infinity to some
-     expression based on param "n" (actually n-1).  */
-  CHECK_STRING_CONTAINS (dump_vrp1, "[-INF, n_");
+     In particular, this example verifies that we can read the dump from a
+     pass with multiple instances.  Given that the format of the VRP dump
+     is subject to change, merely verify that it contains the function
+     name.  */
+  CHECK_STRING_CONTAINS (dump_vrp1, ";; Function loop_test");
   free (dump_vrp1);
 }
diff --git a/gcc/testsuite/lib/g++-dg.exp b/gcc/testsuite/lib/g++-dg.exp
index c360770..fd06d27 100644
--- a/gcc/testsuite/lib/g++-dg.exp
+++ b/gcc/testsuite/lib/g++-dg.exp
@@ -57,7 +57,8 @@ proc g++-dg-runtest { testcases flags default-extra-flags } {
 	    set option_list { }
 	    foreach x $std_list {
 		# Handle "concepts" as C++17 plus Concepts TS.
-		if { $x eq "concepts" } then { set x "17 -fconcepts" }
+		if { $x eq "concepts" } then { set x "17 -fconcepts"
+		} elseif { $x eq "impcx" } then { set x "23 -fimplicit-constexpr" }
 		lappend option_list "${std_prefix}$x"
 	    }
 	} else {
diff --git a/gcc/testsuite/lib/multiline.exp b/gcc/testsuite/lib/multiline.exp
index 0e151b6..86387f8 100644
--- a/gcc/testsuite/lib/multiline.exp
+++ b/gcc/testsuite/lib/multiline.exp
@@ -331,7 +331,7 @@ proc _build_multiline_regex { multiline index } {
 		# Support arbitrary followup text on each non-empty line,
 		# to deal with comments containing containing DejaGnu
 		# directives.
-		append rexp ".*"
+		append rexp "\[^\\n\\r\]*"
 	    }
 	}
 	append rexp "\n"
diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
index 1c8b1eb..c928d99 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -4848,7 +4848,8 @@ foreach { armfunc armflag armdefs } {
 	v8m_base "-march=armv8-m.base -mthumb -mfloat-abi=soft"
 		__ARM_ARCH_8M_BASE__
 	v8m_main "-march=armv8-m.main -mthumb" __ARM_ARCH_8M_MAIN__
-	v8_1m_main "-march=armv8.1-m.main -mthumb" __ARM_ARCH_8M_MAIN__ } {
+	v8_1m_main "-march=armv8.1-m.main -mthumb" __ARM_ARCH_8M_MAIN__
+	v9a "-march=armv9-a" __ARM_ARCH_9A__ } {
     eval [string map [list FUNC $armfunc FLAG $armflag DEFS $armdefs ] {
 	proc check_effective_target_arm_arch_FUNC_ok { } {
 	    return [check_no_compiler_messages arm_arch_FUNC_ok assembly {
@@ -7584,95 +7585,149 @@ proc check_effective_target_vect_element_align_preferred { } {
 # Return zero if the desirable pattern isn't found.
 # It's used by Warray-bounds/Wstringop-overflow testcases which are
 # regressed by O2 vectorization, refer to PR102697/PR102462/PR102706
-proc check_vect_slp_aligned_store_usage { pattern macro } {
+proc check_vect_slp_store_usage { pattern macro } {
     global tool
 
     set result [check_compile slp_aligned_store_usage assembly {
-	char a[16] __attribute__ ((aligned (16)));
-	short b[4] __attribute__((aligned(8)));
-	int c[4] __attribute__((aligned(16)));
-	#ifdef TEST_V8QI
+	extern void sink (void* );
+	#define Ac8 (AC8){ 0, 1, 2, 3, 4, 5, 6, 7 }
+	#define Ac16 (AC16){ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }
+	#ifdef TEST_V16QI
+	typedef struct AC16 { char a[16]; } AC16;
+	extern char a16[16];
 	void
-	foo ()
+	foo1 ()
 	{
-	    a[0] = 0;
-	    a[1] = 1;
-	    a[2] = 2;
-	    a[3] = 3;
-	    a[4] = 4;
-	    a[5] = 5;
-	    a[6] = 6;
-	    a[7] = 7;
-	}
-	#elif TEST_V16QI
+	    *(AC16*)a16 = Ac16;
+	}
+	#elif TEST_V8QI
+	typedef struct AC8 { char a[8]; } AC8;
+	extern char a8[8];
 	void
-	foo1 ()
+	foo ()
 	{
-	    a[0] = 0;
-	    a[1] = 1;
-	    a[2] = 2;
-	    a[3] = 3;
-	    a[4] = 4;
-	    a[5] = 5;
-	    a[6] = 6;
-	    a[7] = 7;
-	    a[8] = 8;
-	    a[9] = 9;
-	    a[10] = 10;
-	    a[11] = 11;
-	    a[12] = 12;
-	    a[13] = 13;
-	    a[14] = 14;
-	    a[15] = 15;
+	    *(AC8*)a8 = Ac8;
 	}
 	#elif TEST_V4QI
+	struct A1
+	{
+	    char n;
+	    char a[3];
+	};
+
+	extern void sink (void*);
 	void
 	foo2 ()
 	{
-	    a[0] = 0;
-	    a[1] = 1;
-	    a[2] = 2;
-	    a[3] = 3;
+	    struct A1 a = { 0, {  } };
+	    a.a[0] = 3;
+	    a.a[1] = 4;
+	    a.a[2] = 5;
+	    sink (&a);
+	}
+	#elif TEST_V4QI_2
+	extern char p[4];
+	void
+	foo2_2 ()
+	{
+	    p[0] = 0;
+	    p[1] = 1;
+	    p[2] = 2;
+	    p[3] = 3;
+	}
+	#elif TEST_V4QI_3
+	#define Ac4 (AC4){ 0, 1, 2, 3 }
+	typedef struct AC4 { char a[4]; } AC4;
+	extern char a[4];
+	void
+	foo ()
+	{
+	    *(AC4*)a = Ac4;
 	}
 	#elif TEST_V2QI
+	struct A2
+	{
+	    char a[2];
+	};
 	void
 	foo3 ()
 	{
-	    a[0] = 0;
-	    a[1] = 1;
+	    struct A2 a;
+	    a.a[0] = 3;
+	    a.a[1] = 4;
+	    sink (&a);
+	}
+	#elif TEST_V4HI
+	struct Ax
+	{
+	    int n;
+	    short a[4];
+	};
+	void
+	foo5 (struct Ax *p)
+	{
+	    p->a[0] = 0;
+	    p->a[1] = 1;
+	    p->a[2] = 2;
+	    p->a[3] = 3;
 	}
 	#elif TEST_V2HI
+	extern char b[4];
 	void
 	foo4 ()
 	{
-	    b[0] = 0;
-	    b[1] = 1;
+	    *(short*) b = 0;
+	    *(short*) (b + 2) = 1;
 	}
-	#elif TEST_V4HI
+	#elif TEST_V2HI_2
+	struct Ax
+	{
+	    int n;
+	    short a[2];
+	};
 	void
-	foo5 ()
+	foo4_2 (struct Ax *p)
 	{
-	    b[0] = 0;
-	    b[1] = 1;
-	    b[2] = 2;
-	    b[3] = 3;
+	    p->a[0] = 0;
+	    p->a[1] = 1;
 	}
-	#elif TEST_V2SI
+	#elif TEST_V4SI
+	struct A { int i; };
+	struct B { int j; struct A a[4]; };
+
+	struct C
+	{
+	    struct B b1;
+	    struct B b2;
+	};
+	char cbuf2[2 * sizeof (struct C)] = { };
 	void
 	foo6 ()
 	{
-	    c[0] = 0;
-	    c[1] = 1;
+	    struct C *p = (struct C*)&cbuf2;
+	    p->b2.a[0].i = 0;
+	    p->b2.a[1].i = 0;
+	    p->b2.a[2].i = 0;
+	    p->b2.a[3].i = 0;
 	}
-	#elif TEST_V4SI
+	#elif TEST_V2SI
+	struct A { int i; };
+	struct B { int j; struct A a[2]; };
+
+	struct C
+	{
+	    struct B b1;
+	    struct B b2;
+	};
+	char cbuf2[2 * sizeof (struct C)] = { };
 	void
-	foo7 ()
+	foo6 ()
 	{
-	    c[0] = 0;
-	    c[1] = 1;
-	    c[2] = 2;
-	    c[3] = 3;
+	    struct C *p = (struct C*)&cbuf2;
+	    p->b2.a[0].i = 0;
+	    p->b2.a[1].i = 0;
 	}
+
 	#endif
     } "-O2 -fopt-info-all -D$macro" ]
 
@@ -7691,75 +7746,164 @@ proc check_vect_slp_aligned_store_usage { pattern macro } {
 
 # Return the true if target support vectorization of 2-byte char stores
 # with 2-byte aligned address at plain O2.
-proc check_effective_target_vect_slp_v2qi_store { } {
+# NB: This target should be removed after real issues are fixed for
+# -Wstringop-overflow with O2 vect. Be careful if you want to reuse
+# this target since tests in check_vect_slp_store_usage
+# is the exact match of relative testcases
+proc check_effective_target_vect_slp_v2qi_store_align { } {
     set pattern {add new stmt: MEM <vector\(2\) char>}
     set macro "TEST_V2QI"
-    return [check_cached_effective_target vect_slp_v2qi_store {
-	expr [check_vect_slp_aligned_store_usage $pattern $macro] }]
+    return [check_cached_effective_target vect_slp_v2qi_store_align {
+	expr [check_vect_slp_store_usage $pattern $macro] }]
 
 }
 
 # Return the true if target support vectorization of 4-byte char stores
 # with 4-byte aligned address at plain O2.
-proc check_effective_target_vect_slp_v4qi_store { } {
+# NB: This target should be removed after real issues are fixed for
+# -Wstringop-overflow with O2 vect. Be careful if you want to reuse
+# this target since tests in check_vect_slp_store_usage
+# is the exact match of relative testcases
+proc check_effective_target_vect_slp_v4qi_store_align { } {
     set pattern {add new stmt: MEM <vector\(4\) char>}
     set macro "TEST_V4QI"
-    return [check_cached_effective_target vect_slp_v4qi_store {
-	expr [check_vect_slp_aligned_store_usage $pattern $macro ] }]
+    return [check_cached_effective_target vect_slp_v4qi_store_align {
+	expr [check_vect_slp_store_usage $pattern $macro ] }]
+}
+
+# Return the true if target support vectorization of 4-byte char stores
+# with unaligned address at plain O2.
+proc check_effective_target_vect_slp_v4qi_store_unalign { } {
+    set pattern {add new stmt: MEM <vector\(4\) char>}
+    set macro "TEST_V4QI_2"
+    return [check_cached_effective_target vect_slp_v4qi_store_unalign {
+	expr [check_vect_slp_store_usage $pattern $macro ] }]
+}
+
+# Return the true if target support block move for
+# 8-byte aligned 4-byte size struct initialization.
+proc check_effective_target_struct_4char_block_move { } {
+    set pattern {not vectorized: more than one data ref in stmt:}
+    set macro "TEST_V4QI_3"
+    return [check_cached_effective_target struct_4char_block_move {
+	expr [check_vect_slp_store_usage $pattern $macro ] }]
+}
+
+# Return the true if target support vectorization of 4-byte char stores
+# with unaligned address or store them with a constant pool at plain O2.
+proc check_effective_target_vect_slp_v4qi_store_unalign_1 { } {
+    set pattern {add new stmt: MEM <vector\(4\) char>}
+    set macro "TEST_V4QI_3"
+    return [check_cached_effective_target vect_slp_v4qi_store_unalign_1 {
+	expr { [check_vect_slp_store_usage $pattern $macro ]
+	       || [check_effective_target_struct_4char_block_move] } }]
+}
+
+# Return the true if target support block move for
+# 8-byte aligned 8-byte size struct initialization.
+proc check_effective_target_struct_8char_block_move { } {
+    set pattern {not vectorized: more than one data ref in stmt:}
+    set macro "TEST_V8QI"
+    return [check_cached_effective_target struct_8char_block_move {
+	expr [check_vect_slp_store_usage $pattern $macro ] }]
 }
 
 # Return the true if target support vectorization of 8-byte char stores
-# with 8-byte aligned address at plain O2.
-proc check_effective_target_vect_slp_v8qi_store { } {
+# with unaligned address or store them with a constant pool at plain O2.
+# NB: This target should be removed after real issues are fixed for
+# -Wstringop-overflow with O2 vect. Be careful if you want to reuse
+# this target since tests in check_vect_slp_store_usage
+# is the exact match of relative testcases
+proc check_effective_target_vect_slp_v8qi_store_unalign_1 { } {
     set pattern {add new stmt: MEM <vector\(8\) char>}
     set macro "TEST_V8QI"
-    return [check_cached_effective_target vect_slp_v8qi_store {
-	expr [check_vect_slp_aligned_store_usage $pattern $macro ] }]
+    return [check_cached_effective_target vect_slp_v8qi_store_unalign_1 {
+	expr { [check_vect_slp_store_usage $pattern $macro ]
+	       || [check_effective_target_struct_8char_block_move] } }]
+}
+
+# Return the true if target support block move for
+# 8-byte aligned 16-byte size struct initialization.
+proc check_effective_target_struct_16char_block_move { } {
+    set pattern {not vectorized: more than one data ref in stmt:}
+    set macro "TEST_V16QI"
+    return [check_cached_effective_target struct_16char_block_move {
+	expr [check_vect_slp_store_usage $pattern $macro ] }]
 }
 
 # Return the true if target support vectorization of 16-byte char stores
-# with 16-byte aligned address at plain O2.
-proc check_effective_target_vect_slp_v16qi_store { } {
+# with unaligned address or store them with a constant pool at plain O2.
+# NB: This target should be removed after real issues are fixed for
+# -Wstringop-overflow with O2 vect. Be careful if you want to reuse
+# this target since tests in check_vect_slp_store_usage
+# is the exact match of relative testcases
+proc check_effective_target_vect_slp_v16qi_store_unalign_1 { } {
     set pattern {add new stmt: MEM <vector\(16\) char>}
     set macro "TEST_V16QI"
-    return [check_cached_effective_target vect_slp_v16qi_store {
-	expr [check_vect_slp_aligned_store_usage $pattern $macro ] }]
+    return [check_cached_effective_target vect_slp_v16qi_store_unalign_1 {
+	expr { [check_vect_slp_store_usage $pattern $macro ]
+	       || [check_effective_target_struct_16char_block_move] } }]
 }
 
 # Return the true if target support vectorization of 4-byte short stores
-# with 4-byte aligned address at plain O2.
-proc check_effective_target_vect_slp_v2hi_store { } {
+# with unaligned address at plain O2.
+# NB: This target should be removed after real issues are fixed for
+# -Wstringop-overflow with O2 vect. Be careful if you want to reuse
+# this target since tests in check_vect_slp_store_usage
+# is the exact match of relative testcases
+proc check_effective_target_vect_slp_v2hi_store_unalign { } {
     set pattern {add new stmt: MEM <vector\(2\) short int>}
     set macro "TEST_V2HI"
-    return [check_cached_effective_target vect_slp_v2hi_store {
-	expr [check_vect_slp_aligned_store_usage $pattern $macro ] }]
+    return [check_cached_effective_target vect_slp_v2hi_store_unalign {
+	expr [check_vect_slp_store_usage $pattern $macro ] }]
+}
+
+# Return the true if target support vectorization of 4-byte short stores
+# with 4-byte aligned address at plain O2.
+proc check_effective_target_vect_slp_v2hi_store_align { } {
+    set pattern {add new stmt: MEM <vector\(2\) short int>}
+    set macro "TEST_V2HI_2"
+    return [check_cached_effective_target vect_slp_v2hi_store_align {
+	expr [check_vect_slp_store_usage $pattern $macro ] }]
 }
 
 # Return the true if target support vectorization of 8-byte short stores
-# with 8-byte aligned address at plain O2.
-proc check_effective_target_vect_slp_v4hi_store { } {
+# with unaligned address at plain O2.
+# NB: This target should be removed after real issues are fixed for
+# -Wstringop-overflow with O2 vect. Be careful if you want to reuse
+# this target since tests in check_vect_slp_store_usage
+# is the exact match of relative testcases
+proc check_effective_target_vect_slp_v4hi_store_unalign { } {
     set pattern {add new stmt: MEM <vector\(4\) short int>}
     set macro "TEST_V4HI"
-    return [check_cached_effective_target vect_slp_v4hi_store {
-	expr [check_vect_slp_aligned_store_usage $pattern $macro ] }]
+    return [check_cached_effective_target vect_slp_v4hi_store_unalign {
+	expr [check_vect_slp_store_usage $pattern $macro ] }]
 }
 
 # Return the true if target support vectorization of 8-byte int stores
 # with 8-byte aligned address at plain O2.
-proc check_effective_target_vect_slp_v2si_store { } {
+# NB: This target should be removed after real issues are fixed for
+# -Wstringop-overflow with O2 vect. Be careful if you want to reuse
+# this target since tests in check_vect_slp_store_usage
+# is the exact match of relative testcases
+proc check_effective_target_vect_slp_v2si_store_align { } {
     set pattern {add new stmt: MEM <vector\(2\) int>}
     set macro "TEST_V2SI"
-    return [check_cached_effective_target vect_slp_v2si_store {
-	expr [check_vect_slp_aligned_store_usage $pattern $macro ] }]
+    return [check_cached_effective_target vect_slp_v2si_store_align {
+	expr [check_vect_slp_store_usage $pattern $macro ] }]
 }
 
 # Return the true if target support vectorization of 16-byte int stores
-# with 16-byte aligned address at plain O2.
-proc check_effective_target_vect_slp_v4si_store { } {
+# with unaligned address at plain O2.
+# NB: This target should be removed after real issues are fixed for
+# -Wstringop-overflow with O2 vect. Be careful if you want to reuse
+# this target since tests in check_vect_slp_store_usage
+# is the exact match of relative testcases
+proc check_effective_target_vect_slp_v4si_store_unalign { } {
     set pattern {add new stmt: MEM <vector\(4\) int>}
     set macro "TEST_V4SI"
-    return [check_cached_effective_target vect_slp_v4si_store {
-	expr [check_vect_slp_aligned_store_usage $pattern $macro ] }]
+    return [check_cached_effective_target vect_slp_v4si_store_unalign {
+	expr [check_vect_slp_store_usage $pattern $macro ] }]
 }
 
 # Return 1 if we can align stack data to the preferred vector alignment.
@@ -10088,6 +10232,10 @@ proc check_effective_target_concepts { } {
     return [check-flags { "" { } { -fconcepts } }]
 }
 
+proc check_effective_target_implicit_constexpr { } {
+    return [check-flags { "" { } { -fimplicit-constexpr } }]
+}
+
 # Return 1 if expensive testcases should be run.
 
 proc check_effective_target_run_expensive_tests { } {
diff --git a/gcc/toplev.c b/gcc/toplev.c
index cb4f8c4..99276bd 100644
--- a/gcc/toplev.c
+++ b/gcc/toplev.c
@@ -457,6 +457,8 @@ compile_file (void)
   if (flag_dump_locations)
     dump_location_info (stderr);
 
+  free_attr_data ();
+
   /* Compilation is now finished except for writing
      what's left of the symbol table output.  */
 
@@ -1227,9 +1229,6 @@ static void
 process_options (bool no_backend)
 {
   const char *language_string = lang_hooks.name;
-  /* Just in case lang_hooks.post_options ends up calling a debug_hook.
-     This can happen with incorrect pre-processed input. */
-  debug_hooks = &do_nothing_debug_hooks;
 
   maximum_field_alignment = initial_max_fld_align * BITS_PER_UNIT;
 
@@ -1378,25 +1377,6 @@ process_options (bool no_backend)
 	}
     }
 
-  if (flag_syntax_only)
-    {
-      write_symbols = NO_DEBUG;
-      profile_flag = 0;
-    }
-
-  if (flag_gtoggle)
-    {
-      if (debug_info_level == DINFO_LEVEL_NONE)
-	{
-	  debug_info_level = DINFO_LEVEL_NORMAL;
-
-	  if (write_symbols == NO_DEBUG)
-	    write_symbols = PREFERRED_DEBUGGING_TYPE;
-	}
-      else
-	debug_info_level = DINFO_LEVEL_NONE;
-    }
-
   /* CTF is supported for only C at this time.  */
   if (!lang_GNU_C ()
       && ctf_debug_info_level > CTFINFO_LEVEL_NONE)
@@ -1499,6 +1479,7 @@ process_options (bool no_backend)
 	}
       flag_var_tracking = 0;
       flag_var_tracking_uninit = 0;
+      flag_var_tracking_assignments = 0;
     }
 
   /* The debug hooks are used to implement -fdump-go-spec because it
@@ -1507,33 +1488,6 @@ process_options (bool no_backend)
   if (flag_dump_go_spec != NULL)
     debug_hooks = dump_go_spec_init (flag_dump_go_spec, debug_hooks);
 
-  /* One could use EnabledBy, but it would lead to a circular dependency.  */
-  if (!OPTION_SET_P (flag_var_tracking_uninit))
-     flag_var_tracking_uninit = flag_var_tracking;
-
-  if (!OPTION_SET_P (flag_var_tracking_assignments))
-    flag_var_tracking_assignments
-      = (flag_var_tracking
-	 && !(flag_selective_scheduling || flag_selective_scheduling2));
-
-  if (flag_var_tracking_assignments_toggle)
-    flag_var_tracking_assignments = !flag_var_tracking_assignments;
-
-  if (flag_var_tracking_assignments && !flag_var_tracking)
-    flag_var_tracking = flag_var_tracking_assignments = -1;
-
-  if (flag_var_tracking_assignments
-      && (flag_selective_scheduling || flag_selective_scheduling2))
-    warning_at (UNKNOWN_LOCATION, 0,
-		"var-tracking-assignments changes selective scheduling");
-
-  if (!OPTION_SET_P (debug_nonbind_markers_p))
-    debug_nonbind_markers_p
-      = (optimize
-	 && debug_info_level >= DINFO_LEVEL_NORMAL
-	 && dwarf_debuginfo_p ()
-	 && !(flag_selective_scheduling || flag_selective_scheduling2));
-
   if (!OPTION_SET_P (dwarf2out_as_loc_support))
     dwarf2out_as_loc_support = dwarf2out_default_as_loc_support ();
   if (!OPTION_SET_P (dwarf2out_as_locview_support))
@@ -2338,6 +2292,9 @@ toplev::main (int argc, char **argv)
   /* Exit early if we can (e.g. -help).  */
   if (!exit_after_options)
     {
+      /* Just in case lang_hooks.post_options ends up calling a debug_hook.
+	 This can happen with incorrect pre-processed input. */
+      debug_hooks = &do_nothing_debug_hooks;
       /* Allow the front end to perform consistency checks and do further
 	 initialization based on the command line options.  This hook also
 	 sets the original filename if appropriate (e.g. foo.i -> foo.c)
diff --git a/gcc/tree-cfg.c b/gcc/tree-cfg.c
index b3a27bc..8ed8c69 100644
--- a/gcc/tree-cfg.c
+++ b/gcc/tree-cfg.c
@@ -9126,7 +9126,7 @@ struct cfg_hooks gimple_cfg_hooks = {
   gimple_flow_call_edges_add,   /* flow_call_edges_add */
   gimple_execute_on_growing_pred,	/* execute_on_growing_pred */
   gimple_execute_on_shrinking_pred, /* execute_on_shrinking_pred */
-  gimple_duplicate_loop_to_header_edge, /* duplicate loop for trees */
+  gimple_duplicate_loop_body_to_header_edge, /* duplicate loop for trees */
   gimple_lv_add_condition_to_bb, /* lv_add_condition_to_bb */
   gimple_lv_adjust_loop_header_phi, /* lv_adjust_loop_header_phi*/
   extract_true_false_edges_from_block, /* extract_cond_bb_edges */
diff --git a/gcc/tree-core.h b/gcc/tree-core.h
index d3d2a8d..8ab119d 100644
--- a/gcc/tree-core.h
+++ b/gcc/tree-core.h
@@ -97,28 +97,29 @@ struct die_struct;
 #define ECF_COLD		  (1 << 15)
 
 /* Call argument flags.  */
-/* Nonzero if the argument is not dereferenced recursively, thus only
-   directly reachable memory is read or written.  */
-#define EAF_DIRECT		(1 << 0)
 
-/* Nonzero if memory reached by the argument is not clobbered.  */
-#define EAF_NOCLOBBER		(1 << 1)
+/* Nonzero if the argument is not used by the function.  */
+#define EAF_UNUSED		(1 << 1)
 
-/* Nonzero if the argument does not escape.  */
-#define EAF_NOESCAPE		(1 << 2)
+/* Following flags come in pairs.  First one is about direct dereferences
+   from the parameter, while the second is about memory reachable by
+   recursive dereferences.  */
 
-/* Nonzero if the argument is not used by the function.  */
-#define EAF_UNUSED		(1 << 3)
+/* Nonzero if memory reached by the argument is not clobbered.  */
+#define EAF_NO_DIRECT_CLOBBER	(1 << 2)
+#define EAF_NO_INDIRECT_CLOBBER	(1 << 3)
 
-/* Nonzero if the argument itself does not escape but memory
-   referenced by it can escape.  */
-#define EAF_NODIRECTESCAPE	(1 << 4)
+/* Nonzero if the argument does not escape.  */
+#define EAF_NO_DIRECT_ESCAPE	(1 << 4)
+#define EAF_NO_INDIRECT_ESCAPE	(1 << 5)
 
 /* Nonzero if the argument does not escape to return value.  */
-#define EAF_NOT_RETURNED	(1 << 5)
+#define EAF_NOT_RETURNED_DIRECTLY (1 << 6)
+#define EAF_NOT_RETURNED_INDIRECTLY (1 << 7)
 
 /* Nonzero if the argument is not read.  */
-#define EAF_NOREAD		(1 << 6)
+#define EAF_NO_DIRECT_READ	(1 << 8)
+#define EAF_NO_INDIRECT_READ	(1 << 9)
 
 /* Call return flags.  */
 /* Mask for the argument number that is returned.  Lower two bits of
diff --git a/gcc/tree-eh.c b/gcc/tree-eh.c
index 3a09de9..3eff07f 100644
--- a/gcc/tree-eh.c
+++ b/gcc/tree-eh.c
@@ -2454,15 +2454,31 @@ operation_could_trap_helper_p (enum tree_code op,
     case FLOOR_MOD_EXPR:
     case ROUND_MOD_EXPR:
     case TRUNC_MOD_EXPR:
-    case RDIV_EXPR:
-      if (honor_snans)
-	return true;
-      if (fp_operation)
-	return flag_trapping_math;
       if (!TREE_CONSTANT (divisor) || integer_zerop (divisor))
         return true;
+      if (TREE_CODE (divisor) == VECTOR_CST)
+	{
+	  /* Inspired by initializer_each_zero_or_onep.  */
+	  unsigned HOST_WIDE_INT nelts = vector_cst_encoded_nelts (divisor);
+	  if (VECTOR_CST_STEPPED_P (divisor)
+	      && !TYPE_VECTOR_SUBPARTS (TREE_TYPE (divisor))
+		    .is_constant (&nelts))
+	    return true;
+	  for (unsigned int i = 0; i < nelts; ++i)
+	    {
+	      tree elt = vector_cst_elt (divisor, i);
+	      if (integer_zerop (elt))
+		return true;
+	    }
+	}
       return false;
 
+    case RDIV_EXPR:
+      if (honor_snans)
+	return true;
+      gcc_assert (fp_operation);
+      return flag_trapping_math;
+
     case LT_EXPR:
     case LE_EXPR:
     case GT_EXPR:
diff --git a/gcc/tree-if-conv.c b/gcc/tree-if-conv.c
index b165dc0..e88ddc9 100644
--- a/gcc/tree-if-conv.c
+++ b/gcc/tree-if-conv.c
@@ -120,6 +120,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-ssa-sccvn.h"
 #include "tree-cfgcleanup.h"
 #include "tree-ssa-dse.h"
+#include "tree-vectorizer.h"
 
 /* Only handle PHIs with no more arguments unless we are asked to by
    simd pragma.  */
@@ -1732,7 +1733,11 @@ is_cond_scalar_reduction (gimple *phi, gimple **reduc, tree arg_0, tree arg_1,
       reduction_op = gimple_assign_rhs_code (stmt);
     }
 
-  if (reduction_op != PLUS_EXPR && reduction_op != MINUS_EXPR)
+  if (reduction_op != PLUS_EXPR
+      && reduction_op != MINUS_EXPR
+      && reduction_op != BIT_IOR_EXPR
+      && reduction_op != BIT_XOR_EXPR
+      && reduction_op != BIT_AND_EXPR)
     return false;
   r_op1 = gimple_assign_rhs1 (stmt);
   r_op2 = gimple_assign_rhs2 (stmt);
@@ -1742,7 +1747,7 @@ is_cond_scalar_reduction (gimple *phi, gimple **reduc, tree arg_0, tree arg_1,
 
   /* Make R_OP1 to hold reduction variable.  */
   if (r_nop2 == PHI_RESULT (header_phi)
-      && reduction_op == PLUS_EXPR)
+      && commutative_tree_code (reduction_op))
     {
       std::swap (r_op1, r_op2);
       std::swap (r_nop1, r_nop2);
@@ -1811,7 +1816,8 @@ convert_scalar_cond_reduction (gimple *reduc, gimple_stmt_iterator *gsi,
   tree rhs1 = gimple_assign_rhs1 (reduc);
   tree tmp = make_temp_ssa_name (TREE_TYPE (rhs1), NULL, "_ifc_");
   tree c;
-  tree zero = build_zero_cst (TREE_TYPE (rhs1));
+  enum tree_code reduction_op  = gimple_assign_rhs_code (reduc);
+  tree op_nochange = neutral_op_for_reduction (TREE_TYPE (rhs1), reduction_op, NULL);
   gimple_seq stmts = NULL;
 
   if (dump_file && (dump_flags & TDF_DETAILS))
@@ -1824,14 +1830,14 @@ convert_scalar_cond_reduction (gimple *reduc, gimple_stmt_iterator *gsi,
      of reduction rhs.  */
   c = fold_build_cond_expr (TREE_TYPE (rhs1),
 			    unshare_expr (cond),
-			    swap ? zero : op1,
-			    swap ? op1 : zero);
+			    swap ? op_nochange : op1,
+			    swap ? op1 : op_nochange);
 
   /* Create assignment stmt and insert it at GSI.  */
   new_assign = gimple_build_assign (tmp, c);
   gsi_insert_before (gsi, new_assign, GSI_SAME_STMT);
-  /* Build rhs for unconditional increment/decrement.  */
-  rhs = gimple_build (&stmts, gimple_assign_rhs_code (reduc),
+  /* Build rhs for unconditional increment/decrement/logic_operation.  */
+  rhs = gimple_build (&stmts, reduction_op,
 		      TREE_TYPE (rhs1), op0, tmp);
 
   if (has_nop)
diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c
index b2c58ac..8c108d8 100644
--- a/gcc/tree-inline.c
+++ b/gcc/tree-inline.c
@@ -193,7 +193,6 @@ remap_ssa_name (tree name, copy_body_data *id)
 	  && id->entry_bb == NULL
 	  && single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
 	{
-	  tree vexpr = make_node (DEBUG_EXPR_DECL);
 	  gimple *def_temp;
 	  gimple_stmt_iterator gsi;
 	  tree val = SSA_NAME_VAR (name);
@@ -210,10 +209,10 @@ remap_ssa_name (tree name, copy_body_data *id)
 	  n = id->decl_map->get (val);
 	  if (n && TREE_CODE (*n) == DEBUG_EXPR_DECL)
 	    return *n;
-	  def_temp = gimple_build_debug_source_bind (vexpr, val, NULL);
-	  DECL_ARTIFICIAL (vexpr) = 1;
-	  TREE_TYPE (vexpr) = TREE_TYPE (name);
+	  tree vexpr = build_debug_expr_decl (TREE_TYPE (name));
+	  /* FIXME: Is setting the mode really necessary? */
 	  SET_DECL_MODE (vexpr, DECL_MODE (SSA_NAME_VAR (name)));
+	  def_temp = gimple_build_debug_source_bind (vexpr, val, NULL);
 	  gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
 	  gsi_insert_before (&gsi, def_temp, GSI_SAME_STMT);
 	  insert_decl_map (id, val, vexpr);
@@ -1529,7 +1528,21 @@ remap_gimple_stmt (gimple *stmt, copy_body_data *id)
   if (!is_gimple_debug (stmt)
       && id->param_body_adjs
       && id->param_body_adjs->m_dead_stmts.contains (stmt))
-    return NULL;
+    {
+      tree *dval = id->param_body_adjs->m_dead_stmt_debug_equiv.get (stmt);
+      if (!dval)
+	return NULL;
+
+      gcc_assert (is_gimple_assign (stmt));
+      tree lhs = gimple_assign_lhs (stmt);
+      tree *dvar = id->param_body_adjs->m_dead_ssa_debug_equiv.get (lhs);
+      gdebug *bind = gimple_build_debug_bind (*dvar, *dval, stmt);
+      if (id->reset_location)
+	gimple_set_location (bind, input_location);
+      id->debug_stmts.safe_push (bind);
+      gimple_seq_add_stmt (&stmts, bind);
+      return stmts;
+    }
 
   /* Begin by recognizing trees that we'll completely rewrite for the
      inlining context.  Our output for these trees is completely
@@ -1807,15 +1820,16 @@ remap_gimple_stmt (gimple *stmt, copy_body_data *id)
 
       if (gimple_debug_bind_p (stmt))
 	{
-	  tree value;
+	  tree var = gimple_debug_bind_get_var (stmt);
+	  tree value = gimple_debug_bind_get_value (stmt);
 	  if (id->param_body_adjs
 	      && id->param_body_adjs->m_dead_stmts.contains (stmt))
-	    value = NULL_TREE;
-	  else
-	    value = gimple_debug_bind_get_value (stmt);
-	  gdebug *copy
-	    = gimple_build_debug_bind (gimple_debug_bind_get_var (stmt),
-				       value, stmt);
+	    {
+	      value = unshare_expr_without_location (value);
+	      id->param_body_adjs->remap_with_debug_expressions (&value);
+	    }
+
+	  gdebug *copy = gimple_build_debug_bind (var, value, stmt);
 	  if (id->reset_location)
 	    gimple_set_location (copy, input_location);
 	  id->debug_stmts.safe_push (copy);
@@ -6435,9 +6449,8 @@ tree_function_versioning (tree old_decl, tree new_decl,
 	      debug_args = decl_debug_args_insert (new_decl);
 	      len = vec_safe_length (*debug_args);
 	    }
-	  ddecl = make_node (DEBUG_EXPR_DECL);
-	  DECL_ARTIFICIAL (ddecl) = 1;
-	  TREE_TYPE (ddecl) = TREE_TYPE (parm);
+	  ddecl = build_debug_expr_decl (TREE_TYPE (parm));
+	  /* FIXME: Is setting the mode really necessary? */
 	  SET_DECL_MODE (ddecl, DECL_MODE (parm));
 	  vec_safe_push (*debug_args, DECL_ORIGIN (parm));
 	  vec_safe_push (*debug_args, ddecl);
@@ -6452,7 +6465,6 @@ tree_function_versioning (tree old_decl, tree new_decl,
 	     in the debug info that var (whole DECL_ORIGIN is the parm
 	     PARM_DECL) is optimized away, but could be looked up at the
 	     call site as value of D#X there.  */
-	  tree vexpr;
 	  gimple_stmt_iterator cgsi
 	    = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
 	  gimple *def_temp;
@@ -6460,17 +6472,24 @@ tree_function_versioning (tree old_decl, tree new_decl,
 	  i = vec_safe_length (*debug_args);
 	  do
 	    {
+	      tree vexpr = NULL_TREE;
 	      i -= 2;
 	      while (var != NULL_TREE
 		     && DECL_ABSTRACT_ORIGIN (var) != (**debug_args)[i])
 		var = TREE_CHAIN (var);
 	      if (var == NULL_TREE)
 		break;
-	      vexpr = make_node (DEBUG_EXPR_DECL);
 	      tree parm = (**debug_args)[i];
-	      DECL_ARTIFICIAL (vexpr) = 1;
-	      TREE_TYPE (vexpr) = TREE_TYPE (parm);
-	      SET_DECL_MODE (vexpr, DECL_MODE (parm));
+	      if (tree parm_ddef = ssa_default_def (id.src_cfun, parm))
+		if (tree *d
+		    = param_body_adjs->m_dead_ssa_debug_equiv.get (parm_ddef))
+		  vexpr = *d;
+	      if (!vexpr)
+		{
+		  vexpr = build_debug_expr_decl (TREE_TYPE (parm));
+		  /* FIXME: Is setting the mode really necessary? */
+		  SET_DECL_MODE (vexpr, DECL_MODE (parm));
+		}
 	      def_temp = gimple_build_debug_bind (var, vexpr, NULL);
 	      gsi_insert_before (&cgsi, def_temp, GSI_NEW_STMT);
 	      def_temp = gimple_build_debug_source_bind (vexpr, parm, NULL);
diff --git a/gcc/tree-into-ssa.c b/gcc/tree-into-ssa.c
index 8045e34..265dcc5 100644
--- a/gcc/tree-into-ssa.c
+++ b/gcc/tree-into-ssa.c
@@ -1284,11 +1284,10 @@ rewrite_debug_stmt_uses (gimple *stmt)
 	      if (def == NULL_TREE)
 		{
 		  gimple *def_temp;
-		  def = make_node (DEBUG_EXPR_DECL);
-		  def_temp = gimple_build_debug_source_bind (def, var, NULL);
-		  DECL_ARTIFICIAL (def) = 1;
-		  TREE_TYPE (def) = TREE_TYPE (var);
+		  def = build_debug_expr_decl (TREE_TYPE (var));
+		  /* FIXME: Is setting the mode really necessary? */
 		  SET_DECL_MODE (def, DECL_MODE (var));
+		  def_temp = gimple_build_debug_source_bind (def, var, NULL);
 		  gsi =
 		 gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
 		  gsi_insert_before (&gsi, def_temp, GSI_SAME_STMT);
diff --git a/gcc/tree-parloops.c b/gcc/tree-parloops.c
index 5e64d5e..96932c7 100644
--- a/gcc/tree-parloops.c
+++ b/gcc/tree-parloops.c
@@ -3298,10 +3298,11 @@ gather_scalar_reductions (loop_p loop, reduction_info_table_type *reduction_list
   auto_vec<gimple *, 4> double_reduc_stmts;
 
   vec_info_shared shared;
-  simple_loop_info = vect_analyze_loop_form (loop, &shared);
-  if (simple_loop_info == NULL)
+  vect_loop_form_info info;
+  if (!vect_analyze_loop_form (loop, &info))
     goto gather_done;
 
+  simple_loop_info = vect_create_loop_vinfo (loop, &shared, &info);
   for (gsi = gsi_start_phis (loop->header); !gsi_end_p (gsi); gsi_next (&gsi))
     {
       gphi *phi = gsi.phi ();
@@ -3339,9 +3340,11 @@ gather_scalar_reductions (loop_p loop, reduction_info_table_type *reduction_list
   if (!double_reduc_phis.is_empty ())
     {
       vec_info_shared shared;
-      simple_loop_info = vect_analyze_loop_form (loop->inner, &shared);
-      if (simple_loop_info)
+      vect_loop_form_info info;
+      if (vect_analyze_loop_form (loop->inner, &info))
 	{
+	  simple_loop_info
+	    = vect_create_loop_vinfo (loop->inner, &shared, &info);
 	  gphi *phi;
 	  unsigned int i;
 
diff --git a/gcc/tree-pass.h b/gcc/tree-pass.h
index d379769..d494aff 100644
--- a/gcc/tree-pass.h
+++ b/gcc/tree-pass.h
@@ -463,7 +463,6 @@ extern gimple_opt_pass *make_pass_copy_prop (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_isolate_erroneous_paths (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_early_vrp (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_vrp (gcc::context *ctxt);
-extern gimple_opt_pass *make_pass_vrp_threader (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_uncprop (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_return_slot (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_reassoc (gcc::context *ctxt);
@@ -644,6 +643,9 @@ extern gimple_opt_pass *make_pass_update_address_taken (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_convert_switch (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_lower_vaarg (gcc::context *ctxt);
 extern gimple_opt_pass *make_pass_gimple_isel (gcc::context *ctxt);
+extern gimple_opt_pass *make_pass_harden_compares (gcc::context *ctxt);
+extern gimple_opt_pass *make_pass_harden_conditional_branches (gcc::context
+							       *ctxt);
 
 /* Current optimization pass.  */
 extern opt_pass *current_pass;
diff --git a/gcc/tree-pretty-print.c b/gcc/tree-pretty-print.c
index 275dc7d..fcc0796 100644
--- a/gcc/tree-pretty-print.c
+++ b/gcc/tree-pretty-print.c
@@ -971,6 +971,9 @@ dump_omp_clause (pretty_printer *pp, tree clause, int spc, dump_flags_t flags)
 			     spc, flags, false);
 	  pp_right_bracket (pp);
 	}
+      if (OMP_CLAUSE_CODE (clause) == OMP_CLAUSE_MAP
+	  && OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P (clause))
+	pp_string (pp, "[implicit]");
       pp_right_paren (pp);
       break;
 
@@ -994,7 +997,13 @@ dump_omp_clause (pretty_printer *pp, tree clause, int spc, dump_flags_t flags)
 
     case OMP_CLAUSE_NUM_TEAMS:
       pp_string (pp, "num_teams(");
-      dump_generic_node (pp, OMP_CLAUSE_NUM_TEAMS_EXPR (clause),
+      if (OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (clause))
+	{
+	  dump_generic_node (pp, OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR (clause),
+			     spc, flags, false);
+	  pp_colon (pp);
+	}
+      dump_generic_node (pp, OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR (clause),
 			 spc, flags, false);
       pp_right_paren (pp);
       break;
diff --git a/gcc/tree-sra.c b/gcc/tree-sra.c
index 9b786e2..76e3aae 100644
--- a/gcc/tree-sra.c
+++ b/gcc/tree-sra.c
@@ -3288,6 +3288,8 @@ totally_scalarize_subtree (struct access *root)
 	      continue;
 
 	    HOST_WIDE_INT pos = root->offset + int_bit_position (fld);
+	    if (pos + fsize > root->offset + root->size)
+	      return false;
 	    enum total_sra_field_state
 	      state = total_should_skip_creating_access (root,
 							 &last_seen_sibling,
diff --git a/gcc/tree-ssa-alias.c b/gcc/tree-ssa-alias.c
index ce667ff..02bbc87 100644
--- a/gcc/tree-ssa-alias.c
+++ b/gcc/tree-ssa-alias.c
@@ -116,10 +116,14 @@ static struct {
   unsigned HOST_WIDE_INT nonoverlapping_refs_since_match_p_may_alias;
   unsigned HOST_WIDE_INT nonoverlapping_refs_since_match_p_must_overlap;
   unsigned HOST_WIDE_INT nonoverlapping_refs_since_match_p_no_alias;
+  unsigned HOST_WIDE_INT stmt_kills_ref_p_no;
+  unsigned HOST_WIDE_INT stmt_kills_ref_p_yes;
   unsigned HOST_WIDE_INT modref_use_may_alias;
   unsigned HOST_WIDE_INT modref_use_no_alias;
   unsigned HOST_WIDE_INT modref_clobber_may_alias;
   unsigned HOST_WIDE_INT modref_clobber_no_alias;
+  unsigned HOST_WIDE_INT modref_kill_no;
+  unsigned HOST_WIDE_INT modref_kill_yes;
   unsigned HOST_WIDE_INT modref_tests;
   unsigned HOST_WIDE_INT modref_baseptr_tests;
 } alias_stats;
@@ -146,6 +150,12 @@ dump_alias_stats (FILE *s)
 	   alias_stats.call_may_clobber_ref_p_no_alias,
 	   alias_stats.call_may_clobber_ref_p_no_alias
 	   + alias_stats.call_may_clobber_ref_p_may_alias);
+  fprintf (s, "  stmt_kills_ref_p: "
+	   HOST_WIDE_INT_PRINT_DEC" kills, "
+	   HOST_WIDE_INT_PRINT_DEC" queries\n",
+	   alias_stats.stmt_kills_ref_p_yes + alias_stats.modref_kill_yes,
+	   alias_stats.stmt_kills_ref_p_yes + alias_stats.modref_kill_yes
+	   + alias_stats.stmt_kills_ref_p_no + alias_stats.modref_kill_no);
   fprintf (s, "  nonoverlapping_component_refs_p: "
 	   HOST_WIDE_INT_PRINT_DEC" disambiguations, "
 	   HOST_WIDE_INT_PRINT_DEC" queries\n",
@@ -169,6 +179,12 @@ dump_alias_stats (FILE *s)
 	   + alias_stats.aliasing_component_refs_p_may_alias);
   dump_alias_stats_in_alias_c (s);
   fprintf (s, "\nModref stats:\n");
+  fprintf (s, "  modref kill: "
+	   HOST_WIDE_INT_PRINT_DEC" kills, "
+	   HOST_WIDE_INT_PRINT_DEC" queries\n",
+	   alias_stats.modref_kill_yes,
+	   alias_stats.modref_kill_yes
+	   + alias_stats.modref_kill_no);
   fprintf (s, "  modref use: "
 	   HOST_WIDE_INT_PRINT_DEC" disambiguations, "
 	   HOST_WIDE_INT_PRINT_DEC" queries\n",
@@ -782,7 +798,7 @@ ao_ref_alias_ptr_type (ao_ref *ref)
    The access is assumed to be only to or after of the pointer target adjusted
    by the offset, not before it (even in the case RANGE_KNOWN is false).  */
 
-static void
+void
 ao_ref_init_from_ptr_and_range (ao_ref *ref, tree ptr,
 				bool range_known,
 				poly_int64 offset,
@@ -2535,13 +2551,10 @@ refs_output_dependent_p (tree store1, tree store2)
    IF TBAA_P is true, use TBAA oracle.  */
 
 static bool
-modref_may_conflict (const gimple *stmt,
+modref_may_conflict (const gcall *stmt,
 		     modref_tree <alias_set_type> *tt, ao_ref *ref, bool tbaa_p)
 {
   alias_set_type base_set, ref_set;
-  modref_base_node <alias_set_type> *base_node;
-  modref_ref_node <alias_set_type> *ref_node;
-  size_t i, j, k;
 
   if (tt->every_base)
     return true;
@@ -2554,7 +2567,7 @@ modref_may_conflict (const gimple *stmt,
   ref_set = ao_ref_alias_set (ref);
 
   int num_tests = 0, max_tests = param_modref_max_tests;
-  FOR_EACH_VEC_SAFE_ELT (tt->bases, i, base_node)
+  for (auto base_node : tt->bases)
     {
       if (tbaa_p && flag_strict_aliasing)
 	{
@@ -2569,7 +2582,7 @@ modref_may_conflict (const gimple *stmt,
       if (base_node->every_ref)
 	return true;
 
-      FOR_EACH_VEC_SAFE_ELT (base_node->refs, j, ref_node)
+      for (auto ref_node : base_node->refs)
 	{
 	  /* Do not repeat same test as before.  */
 	  if ((ref_set != base_set || base_node->base != ref_node->ref)
@@ -2583,62 +2596,43 @@ modref_may_conflict (const gimple *stmt,
 	      num_tests++;
 	    }
 
-	  /* TBAA checks did not disambiguate,  try to use base pointer, for
-	     that we however need to have ref->ref or ref->base.  */
-	  if (ref_node->every_access || (!ref->ref && !ref->base))
+	  if (ref_node->every_access)
 	    return true;
 
-	  modref_access_node *access_node;
-	  FOR_EACH_VEC_SAFE_ELT (ref_node->accesses, k, access_node)
+	  /* TBAA checks did not disambiguate, try individual accesses.  */
+	  for (auto access_node : ref_node->accesses)
 	    {
 	      if (num_tests >= max_tests)
 		return true;
 
-	      if (access_node->parm_index == -1
-		  || (unsigned)access_node->parm_index
-		     >= gimple_call_num_args (stmt))
+	      tree arg = access_node.get_call_arg (stmt);
+	      if (!arg)
 		return true;
 
 	      alias_stats.modref_baseptr_tests++;
 
-	      tree arg = gimple_call_arg (stmt, access_node->parm_index);
-
 	      if (integer_zerop (arg) && flag_delete_null_pointer_checks)
 		continue;
 
+	      /* PTA oracle will be unhapy of arg is not an pointer.  */
 	      if (!POINTER_TYPE_P (TREE_TYPE (arg)))
 		return true;
 
-	      /* ao_ref_init_from_ptr_and_range assumes that memory access
-		 starts by the pointed to location.  If we did not track the
-		 offset it is possible that it starts before the actual
-		 pointer.  */
-	      if (!access_node->parm_offset_known)
+	      /* If we don't have base pointer, give up.  */
+	      if (!ref->ref && !ref->base)
+		continue;
+
+	      ao_ref ref2;
+	      if (access_node.get_ao_ref (stmt, &ref2))
 		{
-		  if (ptr_deref_may_alias_ref_p_1 (arg, ref))
-		    return true;
-		}
-	      else
-		{
-		  ao_ref ref2;
-		  poly_offset_int off = (poly_offset_int)access_node->offset
-			+ ((poly_offset_int)access_node->parm_offset
-			   << LOG2_BITS_PER_UNIT);
-		  poly_int64 off2;
-		  if (off.to_shwi (&off2))
-		    {
-		      ao_ref_init_from_ptr_and_range
-			     (&ref2, arg, true, off2,
-			      access_node->size,
-			      access_node->max_size);
-		      ref2.ref_alias_set = ref_set;
-		      ref2.base_alias_set = base_set;
-		      if (refs_may_alias_p_1 (&ref2, ref, tbaa_p))
-			return true;
-		    }
-		  else if (ptr_deref_may_alias_ref_p_1 (arg, ref))
+		  ref2.ref_alias_set = ref_node->ref;
+		  ref2.base_alias_set = base_node->base;
+		  if (refs_may_alias_p_1 (&ref2, ref, tbaa_p))
 		    return true;
 		}
+	      else if (ptr_deref_may_alias_ref_p_1 (arg, ref))
+		return true;
+
 	      num_tests++;
 	    }
 	}
@@ -2771,7 +2765,7 @@ ref_maybe_used_by_call_p_1 (gcall *call, ao_ref *ref, bool tbaa_p)
       if (node && node->binds_to_current_def_p ())
 	{
 	  modref_summary *summary = get_modref_function_summary (node);
-	  if (summary)
+	  if (summary && !summary->calls_interposable)
 	    {
 	      if (!modref_may_conflict (call, summary->loads, ref, tbaa_p))
 		{
@@ -2870,7 +2864,7 @@ process_args:
       tree op = gimple_call_arg (call, i);
       int flags = gimple_call_arg_flags (call, i);
 
-      if (flags & (EAF_UNUSED | EAF_NOREAD))
+      if (flags & (EAF_UNUSED | EAF_NO_DIRECT_READ))
 	continue;
 
       if (TREE_CODE (op) == WITH_SIZE_EXPR)
@@ -3242,6 +3236,51 @@ same_addr_size_stores_p (tree base1, poly_int64 offset1, poly_int64 size1,
 	  && known_eq (wi::to_poly_offset (DECL_SIZE (obj)), size1));
 }
 
+/* Return true if REF is killed by an store described by
+   BASE, OFFSET, SIZE and MAX_SIZE.  */
+
+static bool
+store_kills_ref_p (tree base, poly_int64 offset, poly_int64 size,
+		   poly_int64 max_size, ao_ref *ref)
+{
+  poly_int64 ref_offset = ref->offset;
+  /* We can get MEM[symbol: sZ, index: D.8862_1] here,
+     so base == ref->base does not always hold.  */
+  if (base != ref->base)
+    {
+      /* Try using points-to info.  */
+      if (same_addr_size_stores_p (base, offset, size, max_size, ref->base,
+				   ref->offset, ref->size, ref->max_size))
+	return true;
+
+      /* If both base and ref->base are MEM_REFs, only compare the
+	 first operand, and if the second operand isn't equal constant,
+	 try to add the offsets into offset and ref_offset.  */
+      if (TREE_CODE (base) == MEM_REF && TREE_CODE (ref->base) == MEM_REF
+	  && TREE_OPERAND (base, 0) == TREE_OPERAND (ref->base, 0))
+	{
+	  if (!tree_int_cst_equal (TREE_OPERAND (base, 1),
+				   TREE_OPERAND (ref->base, 1)))
+	    {
+	      poly_offset_int off1 = mem_ref_offset (base);
+	      off1 <<= LOG2_BITS_PER_UNIT;
+	      off1 += offset;
+	      poly_offset_int off2 = mem_ref_offset (ref->base);
+	      off2 <<= LOG2_BITS_PER_UNIT;
+	      off2 += ref_offset;
+	      if (!off1.to_shwi (&offset) || !off2.to_shwi (&ref_offset))
+		size = -1;
+	    }
+	}
+      else
+	size = -1;
+    }
+  /* For a must-alias check we need to be able to constrain
+     the access properly.  */
+  return (known_eq (size, max_size)
+	  && known_subrange_p (ref_offset, ref->max_size, offset, size));
+}
+
 /* If STMT kills the memory reference REF return true, otherwise
    return false.  */
 
@@ -3315,7 +3354,10 @@ stmt_kills_ref_p (gimple *stmt, ao_ref *ref)
 		      && operand_equal_p (lhs, base,
 					  OEP_ADDRESS_OF
 					  | OEP_MATCH_SIDE_EFFECTS))))
-	    return true;
+	    {
+	      ++alias_stats.stmt_kills_ref_p_yes;
+	      return true;
+	    }
 	}
 
       /* Now look for non-literal equal bases with the restriction of
@@ -3323,52 +3365,72 @@ stmt_kills_ref_p (gimple *stmt, ao_ref *ref)
       /* For a must-alias check we need to be able to constrain
 	 the access properly.  */
       if (!ref->max_size_known_p ())
-	return false;
-      poly_int64 size, offset, max_size, ref_offset = ref->offset;
+	{
+	  ++alias_stats.stmt_kills_ref_p_no;
+	  return false;
+	}
+      poly_int64 size, offset, max_size;
       bool reverse;
       tree base = get_ref_base_and_extent (lhs, &offset, &size, &max_size,
 					   &reverse);
-      /* We can get MEM[symbol: sZ, index: D.8862_1] here,
-	 so base == ref->base does not always hold.  */
-      if (base != ref->base)
+      if (store_kills_ref_p (base, offset, size, max_size, ref))
 	{
-	  /* Try using points-to info.  */
-	  if (same_addr_size_stores_p (base, offset, size, max_size, ref->base,
-				       ref->offset, ref->size, ref->max_size))
-	    return true;
-
-	  /* If both base and ref->base are MEM_REFs, only compare the
-	     first operand, and if the second operand isn't equal constant,
-	     try to add the offsets into offset and ref_offset.  */
-	  if (TREE_CODE (base) == MEM_REF && TREE_CODE (ref->base) == MEM_REF
-	      && TREE_OPERAND (base, 0) == TREE_OPERAND (ref->base, 0))
-	    {
-	      if (!tree_int_cst_equal (TREE_OPERAND (base, 1),
-				       TREE_OPERAND (ref->base, 1)))
-		{
-		  poly_offset_int off1 = mem_ref_offset (base);
-		  off1 <<= LOG2_BITS_PER_UNIT;
-		  off1 += offset;
-		  poly_offset_int off2 = mem_ref_offset (ref->base);
-		  off2 <<= LOG2_BITS_PER_UNIT;
-		  off2 += ref_offset;
-		  if (!off1.to_shwi (&offset) || !off2.to_shwi (&ref_offset))
-		    size = -1;
-		}
-	    }
-	  else
-	    size = -1;
+	  ++alias_stats.stmt_kills_ref_p_yes;
+	  return true;
 	}
-      /* For a must-alias check we need to be able to constrain
-	 the access properly.  */
-      if (known_eq (size, max_size)
-	  && known_subrange_p (ref_offset, ref->max_size, offset, size))
-	return true;
     }
 
   if (is_gimple_call (stmt))
     {
       tree callee = gimple_call_fndecl (stmt);
+      struct cgraph_node *node;
+      modref_summary *summary;
+
+      /* Try to disambiguate using modref summary.  Modref records a vector
+	 of stores with known offsets relative to function parameters that must
+	 happen every execution of function.  Find if we have a matching
+	 store and verify that function can not use the value.  */
+      if (callee != NULL_TREE
+	  && (node = cgraph_node::get (callee)) != NULL
+	  && node->binds_to_current_def_p ()
+	  && (summary = get_modref_function_summary (node)) != NULL
+	  && summary->kills.length ()
+	  && (!cfun->can_throw_non_call_exceptions
+	      || !stmt_can_throw_internal (cfun, stmt)))
+	{
+	  for (auto kill : summary->kills)
+	    {
+	      ao_ref dref;
+
+	      /* We only can do useful compares if we know the access range
+		 precisely.  */
+	      if (!kill.get_ao_ref (as_a <gcall *> (stmt), &dref))
+		continue;
+	      if (store_kills_ref_p (ao_ref_base (&dref), dref.offset,
+				     dref.size, dref.max_size, ref))
+		{
+		  /* For store to be killed it needs to not be used
+		     earlier.  */
+		  if (ref_maybe_used_by_call_p_1 (as_a <gcall *> (stmt), ref,
+						  true)
+		      || !dbg_cnt (ipa_mod_ref))
+		    break;
+		  if (dump_file && (dump_flags & TDF_DETAILS))
+		    {
+		      fprintf (dump_file,
+			       "ipa-modref: call stmt ");
+		      print_gimple_stmt (dump_file, stmt, 0);
+		      fprintf (dump_file,
+			       "ipa-modref: call to %s kills ",
+			       node->dump_name ());
+		      print_generic_expr (dump_file, ref->base);
+		    }
+		    ++alias_stats.modref_kill_yes;
+		    return true;
+		}
+	    }
+	  ++alias_stats.modref_kill_no;
+	}
       if (callee != NULL_TREE
 	  && gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
 	switch (DECL_FUNCTION_CODE (callee))
@@ -3379,7 +3441,10 @@ stmt_kills_ref_p (gimple *stmt, ao_ref *ref)
 	      tree base = ao_ref_base (ref);
 	      if (base && TREE_CODE (base) == MEM_REF
 		  && TREE_OPERAND (base, 0) == ptr)
-		return true;
+		{
+		  ++alias_stats.stmt_kills_ref_p_yes;
+		  return true;
+		}
 	      break;
 	    }
 
@@ -3398,7 +3463,10 @@ stmt_kills_ref_p (gimple *stmt, ao_ref *ref)
 	      /* For a must-alias check we need to be able to constrain
 		 the access properly.  */
 	      if (!ref->max_size_known_p ())
-		return false;
+		{
+		  ++alias_stats.stmt_kills_ref_p_no;
+		  return false;
+		}
 	      tree dest;
 	      tree len;
 
@@ -3413,11 +3481,17 @@ stmt_kills_ref_p (gimple *stmt, ao_ref *ref)
 		  tree arg1 = gimple_call_arg (stmt, 1);
 		  if (TREE_CODE (arg0) != INTEGER_CST
 		      || TREE_CODE (arg1) != INTEGER_CST)
-		    return false;
+		    {
+		      ++alias_stats.stmt_kills_ref_p_no;
+		      return false;
+		    }
 
 		  dest = gimple_call_lhs (stmt);
 		  if (!dest)
-		    return false;
+		    {
+		      ++alias_stats.stmt_kills_ref_p_no;
+		      return false;
+		    }
 		  len = fold_build2 (MULT_EXPR, TREE_TYPE (arg0), arg0, arg1);
 		}
 	      else
@@ -3427,29 +3501,14 @@ stmt_kills_ref_p (gimple *stmt, ao_ref *ref)
 		}
 	      if (!poly_int_tree_p (len))
 		return false;
-	      tree rbase = ref->base;
-	      poly_offset_int roffset = ref->offset;
 	      ao_ref dref;
 	      ao_ref_init_from_ptr_and_size (&dref, dest, len);
-	      tree base = ao_ref_base (&dref);
-	      poly_offset_int offset = dref.offset;
-	      if (!base || !known_size_p (dref.size))
-		return false;
-	      if (TREE_CODE (base) == MEM_REF)
+	      if (store_kills_ref_p (ao_ref_base (&dref), dref.offset,
+				     dref.size, dref.max_size, ref))
 		{
-		  if (TREE_CODE (rbase) != MEM_REF)
-		    return false;
-		  // Compare pointers.
-		  offset += mem_ref_offset (base) << LOG2_BITS_PER_UNIT;
-		  roffset += mem_ref_offset (rbase) << LOG2_BITS_PER_UNIT;
-		  base = TREE_OPERAND (base, 0);
-		  rbase = TREE_OPERAND (rbase, 0);
+		  ++alias_stats.stmt_kills_ref_p_yes;
+		  return true;
 		}
-	      if (base == rbase
-		  && known_subrange_p (roffset, ref->max_size, offset,
-				       wi::to_poly_offset (len)
-				       << LOG2_BITS_PER_UNIT))
-		return true;
 	      break;
 	    }
 
@@ -3460,7 +3519,10 @@ stmt_kills_ref_p (gimple *stmt, ao_ref *ref)
 		{
 		  tree base = ao_ref_base (ref);
 		  if (TREE_OPERAND (ptr, 0) == base)
-		    return true;
+		    {
+		      ++alias_stats.stmt_kills_ref_p_yes;
+		      return true;
+		    }
 		}
 	      break;
 	    }
@@ -3468,6 +3530,7 @@ stmt_kills_ref_p (gimple *stmt, ao_ref *ref)
 	  default:;
 	  }
     }
+  ++alias_stats.stmt_kills_ref_p_no;
   return false;
 }
 
diff --git a/gcc/tree-ssa-alias.h b/gcc/tree-ssa-alias.h
index 275dea1..64d4865 100644
--- a/gcc/tree-ssa-alias.h
+++ b/gcc/tree-ssa-alias.h
@@ -111,6 +111,9 @@ ao_ref::max_size_known_p () const
 /* In tree-ssa-alias.c  */
 extern void ao_ref_init (ao_ref *, tree);
 extern void ao_ref_init_from_ptr_and_size (ao_ref *, tree, tree);
+extern void ao_ref_init_from_ptr_and_range (ao_ref *, tree, bool,
+					    poly_int64, poly_int64,
+					    poly_int64);
 extern tree ao_ref_base (ao_ref *);
 extern alias_set_type ao_ref_alias_set (ao_ref *);
 extern alias_set_type ao_ref_base_alias_set (ao_ref *);
diff --git a/gcc/tree-ssa-ccp.c b/gcc/tree-ssa-ccp.c
index 70ce6a4..18d5772 100644
--- a/gcc/tree-ssa-ccp.c
+++ b/gcc/tree-ssa-ccp.c
@@ -3243,6 +3243,90 @@ optimize_unreachable (gimple_stmt_iterator i)
   return ret;
 }
 
+/* Convert
+   _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
+   _7 = ~_1;
+   _5 = (_Bool) _7;
+   to
+   _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
+   _8 = _1 & 1;
+   _5 = _8 == 0;
+   and convert
+   _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
+   _7 = ~_1;
+   _4 = (_Bool) _7;
+   to
+   _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
+   _8 = _1 & 1;
+   _4 = (_Bool) _8;
+
+   USE_STMT is the gimplt statement which uses the return value of
+   __atomic_fetch_or_*.  LHS is the return value of __atomic_fetch_or_*.
+   MASK is the mask passed to __atomic_fetch_or_*.
+ */
+
+static gimple *
+convert_atomic_bit_not (enum internal_fn fn, gimple *use_stmt,
+			tree lhs, tree mask)
+{
+  tree and_mask;
+  if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
+    {
+      /* MASK must be ~1.  */
+      if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs),
+					   ~HOST_WIDE_INT_1), mask, 0))
+	return nullptr;
+      and_mask = build_int_cst (TREE_TYPE (lhs), 1);
+    }
+  else
+    {
+      /* MASK must be 1.  */
+      if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs), 1), mask, 0))
+	return nullptr;
+      and_mask = mask;
+    }
+
+  tree use_lhs = gimple_assign_lhs (use_stmt);
+
+  use_operand_p use_p;
+  gimple *use_not_stmt;
+
+  if (!single_imm_use (use_lhs, &use_p, &use_not_stmt)
+      || !is_gimple_assign (use_not_stmt))
+    return nullptr;
+
+  if (!CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (use_not_stmt)))
+    return nullptr;
+
+  tree use_not_lhs = gimple_assign_lhs (use_not_stmt);
+  if (TREE_CODE (TREE_TYPE (use_not_lhs)) != BOOLEAN_TYPE)
+    return nullptr;
+
+  gimple_stmt_iterator gsi;
+  gsi = gsi_for_stmt (use_stmt);
+  gsi_remove (&gsi, true);
+  tree var = make_ssa_name (TREE_TYPE (lhs));
+  use_stmt = gimple_build_assign (var, BIT_AND_EXPR, lhs, and_mask);
+  gsi = gsi_for_stmt (use_not_stmt);
+  gsi_insert_before (&gsi, use_stmt, GSI_NEW_STMT);
+  lhs = gimple_assign_lhs (use_not_stmt);
+  gimple *g = gimple_build_assign (lhs, EQ_EXPR, var,
+				   build_zero_cst (TREE_TYPE (mask)));
+  gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+  gsi = gsi_for_stmt (use_not_stmt);
+  gsi_remove (&gsi, true);
+  return use_stmt;
+}
+
+/* match.pd function to match atomic_bit_test_and pattern which
+   has nop_convert:
+     _1 = __atomic_fetch_or_4 (&v, 1, 0);
+     _2 = (int) _1;
+     _5 = _2 & 1;
+ */
+extern bool gimple_nop_atomic_bit_test_and_p (tree, tree *,
+					      tree (*) (tree));
+
 /* Optimize
      mask_2 = 1 << cnt_1;
      _4 = __atomic_fetch_or_* (ptr_6, mask_2, _3);
@@ -3269,7 +3353,7 @@ optimize_atomic_bit_test_and (gimple_stmt_iterator *gsip,
   tree lhs = gimple_call_lhs (call);
   use_operand_p use_p;
   gimple *use_stmt;
-  tree mask, bit;
+  tree mask;
   optab optab;
 
   if (!flag_inline_atomics
@@ -3279,7 +3363,6 @@ optimize_atomic_bit_test_and (gimple_stmt_iterator *gsip,
       || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)
       || !single_imm_use (lhs, &use_p, &use_stmt)
       || !is_gimple_assign (use_stmt)
-      || gimple_assign_rhs_code (use_stmt) != BIT_AND_EXPR
       || !gimple_vdef (call))
     return;
 
@@ -3298,54 +3381,347 @@ optimize_atomic_bit_test_and (gimple_stmt_iterator *gsip,
       return;
     }
 
-  if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs))) == CODE_FOR_nothing)
-    return;
+  tree bit = nullptr;
 
   mask = gimple_call_arg (call, 1);
-  tree use_lhs = gimple_assign_lhs (use_stmt);
-  if (!use_lhs)
-    return;
-
-  if (TREE_CODE (mask) == INTEGER_CST)
+  tree_code rhs_code = gimple_assign_rhs_code (use_stmt);
+  if (rhs_code != BIT_AND_EXPR)
     {
-      if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
-	mask = const_unop (BIT_NOT_EXPR, TREE_TYPE (mask), mask);
-      mask = fold_convert (TREE_TYPE (lhs), mask);
-      int ibit = tree_log2 (mask);
-      if (ibit < 0)
+      if (rhs_code != NOP_EXPR && rhs_code != BIT_NOT_EXPR)
 	return;
-      bit = build_int_cst (TREE_TYPE (lhs), ibit);
-    }
-  else if (TREE_CODE (mask) == SSA_NAME)
-    {
-      gimple *g = SSA_NAME_DEF_STMT (mask);
-      if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
+
+      tree use_lhs = gimple_assign_lhs (use_stmt);
+      if (TREE_CODE (use_lhs) == SSA_NAME
+	  && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_lhs))
+	return;
+
+      tree use_rhs = gimple_assign_rhs1 (use_stmt);
+      if (lhs != use_rhs)
+	return;
+
+      if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)))
+	  == CODE_FOR_nothing)
+	return;
+
+      gimple *g;
+      gimple_stmt_iterator gsi;
+      tree var;
+      int ibit = -1;
+
+      if (rhs_code == BIT_NOT_EXPR)
 	{
-	  if (!is_gimple_assign (g)
-	      || gimple_assign_rhs_code (g) != BIT_NOT_EXPR)
+	  g = convert_atomic_bit_not (fn, use_stmt, lhs, mask);
+	  if (!g)
 	    return;
-	  mask = gimple_assign_rhs1 (g);
-	  if (TREE_CODE (mask) != SSA_NAME)
+	  use_stmt = g;
+	  ibit = 0;
+	}
+      else if (TREE_CODE (TREE_TYPE (use_lhs)) == BOOLEAN_TYPE)
+	{
+	  tree and_mask;
+	  if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
+	    {
+	      /* MASK must be ~1.  */
+	      if (!operand_equal_p (build_int_cst (TREE_TYPE (lhs),
+						   ~HOST_WIDE_INT_1),
+				    mask, 0))
+		return;
+
+	      /* Convert
+		 _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
+		 _4 = (_Bool) _1;
+		 to
+		 _1 = __atomic_fetch_and_* (ptr_6, ~1, _3);
+		 _5 = _1 & 1;
+		 _4 = (_Bool) _5;
+	       */
+	      and_mask = build_int_cst (TREE_TYPE (lhs), 1);
+	    }
+	  else
+	    {
+	      and_mask = build_int_cst (TREE_TYPE (lhs), 1);
+	      if (!operand_equal_p (and_mask, mask, 0))
+		return;
+
+	      /* Convert
+		 _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
+		 _4 = (_Bool) _1;
+		 to
+		 _1 = __atomic_fetch_or_* (ptr_6, 1, _3);
+		 _5 = _1 & 1;
+		 _4 = (_Bool) _5;
+	       */
+	    }
+	  var = make_ssa_name (TREE_TYPE (use_rhs));
+	  replace_uses_by (use_rhs, var);
+	  g = gimple_build_assign (var, BIT_AND_EXPR, use_rhs,
+				   and_mask);
+	  gsi = gsi_for_stmt (use_stmt);
+	  gsi_insert_before (&gsi, g, GSI_NEW_STMT);
+	  use_stmt = g;
+	  ibit = 0;
+	}
+      else if (TYPE_PRECISION (TREE_TYPE (use_lhs))
+	       == TYPE_PRECISION (TREE_TYPE (use_rhs)))
+	{
+	  gimple *use_nop_stmt;
+	  if (!single_imm_use (use_lhs, &use_p, &use_nop_stmt)
+	      || !is_gimple_assign (use_nop_stmt))
 	    return;
-	  g = SSA_NAME_DEF_STMT (mask);
+	  rhs_code = gimple_assign_rhs_code (use_nop_stmt);
+	  if (rhs_code != BIT_AND_EXPR)
+	    {
+	      tree use_nop_lhs = gimple_assign_lhs (use_nop_stmt);
+	      if (TREE_CODE (use_nop_lhs) == SSA_NAME
+		  && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (use_nop_lhs))
+		return;
+	      if (rhs_code == BIT_NOT_EXPR)
+		{
+		  g = convert_atomic_bit_not (fn, use_nop_stmt, lhs,
+					      mask);
+		  if (!g)
+		    return;
+		  /* Convert
+		     _1 = __atomic_fetch_or_4 (ptr_6, 1, _3);
+		     _2 = (int) _1;
+		     _7 = ~_2;
+		     _5 = (_Bool) _7;
+		     to
+		     _1 = __atomic_fetch_or_4 (ptr_6, ~1, _3);
+		     _8 = _1 & 1;
+		     _5 = _8 == 0;
+		     and convert
+		     _1 = __atomic_fetch_and_4 (ptr_6, ~1, _3);
+		     _2 = (int) _1;
+		     _7 = ~_2;
+		     _5 = (_Bool) _7;
+		     to
+		     _1 = __atomic_fetch_and_4 (ptr_6, 1, _3);
+		     _8 = _1 & 1;
+		     _5 = _8 == 0;
+		   */
+		  gsi = gsi_for_stmt (use_stmt);
+		  gsi_remove (&gsi, true);
+		  use_stmt = g;
+		  ibit = 0;
+		}
+	      else
+		{
+		  if (TREE_CODE (TREE_TYPE (use_nop_lhs)) != BOOLEAN_TYPE)
+		    return;
+		  if (rhs_code != GE_EXPR && rhs_code != LT_EXPR)
+		    return;
+		  tree cmp_rhs1 = gimple_assign_rhs1 (use_nop_stmt);
+		  if (use_lhs != cmp_rhs1)
+		    return;
+		  tree cmp_rhs2 = gimple_assign_rhs2 (use_nop_stmt);
+		  if (!integer_zerop (cmp_rhs2))
+		    return;
+
+		  tree and_mask;
+
+		  unsigned HOST_WIDE_INT bytes
+		    = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (use_rhs)));
+		  ibit = bytes * BITS_PER_UNIT - 1;
+		  unsigned HOST_WIDE_INT highest
+		    = HOST_WIDE_INT_1U << ibit;
+
+		  if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
+		    {
+		      /* Get the signed maximum of the USE_RHS type.  */
+		      and_mask = build_int_cst (TREE_TYPE (use_rhs),
+						highest - 1);
+		      if (!operand_equal_p (and_mask, mask, 0))
+			return;
+
+		      /* Convert
+			 _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3);
+			 _5 = (signed int) _1;
+			 _4 = _5 < 0 or _5 >= 0;
+			 to
+			 _1 = __atomic_fetch_and_4 (ptr_6, 0x7fffffff, _3);
+			 _6 = _1 & 0x80000000;
+			 _4 = _6 != 0 or _6 == 0;
+		       */
+		      and_mask = build_int_cst (TREE_TYPE (use_rhs),
+						highest);
+		    }
+		  else
+		    {
+		      /* Get the signed minimum of the USE_RHS type.  */
+		      and_mask = build_int_cst (TREE_TYPE (use_rhs),
+						highest);
+		      if (!operand_equal_p (and_mask, mask, 0))
+			return;
+
+		      /* Convert
+			 _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3);
+			 _5 = (signed int) _1;
+			 _4 = _5 < 0 or _5 >= 0;
+			 to
+			 _1 = __atomic_fetch_or_4 (ptr_6, 0x80000000, _3);
+			 _6 = _1 & 0x80000000;
+			 _4 = _6 != 0 or _6 == 0;
+		       */
+		    }
+		  var = make_ssa_name (TREE_TYPE (use_rhs));
+		  gsi = gsi_for_stmt (use_stmt);
+		  gsi_remove (&gsi, true);
+		  g = gimple_build_assign (var, BIT_AND_EXPR, use_rhs,
+					   and_mask);
+		  gsi = gsi_for_stmt (use_nop_stmt);
+		  gsi_insert_before (&gsi, g, GSI_NEW_STMT);
+		  use_stmt = g;
+		  g = gimple_build_assign (use_nop_lhs,
+					   (rhs_code == GE_EXPR
+					    ? EQ_EXPR : NE_EXPR),
+					   var,
+					   build_zero_cst (TREE_TYPE (use_rhs)));
+		  gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+		  gsi = gsi_for_stmt (use_nop_stmt);
+		  gsi_remove (&gsi, true);
+		}
+	    }
+	  else
+	    {
+	      tree and_expr = gimple_assign_lhs (use_nop_stmt);
+	      tree match_op[3];
+	      gimple *g;
+	      if (!gimple_nop_atomic_bit_test_and_p (and_expr,
+						     &match_op[0], NULL)
+		  || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (match_op[2])
+		  || !single_imm_use (match_op[2], &use_p, &g)
+		  || !is_gimple_assign (g))
+		return;
+	      mask = match_op[1];
+	      if (TREE_CODE (mask) == INTEGER_CST)
+		{
+		  ibit = tree_log2 (mask);
+		  gcc_assert (ibit >= 0);
+		}
+	      else
+		{
+		  g = SSA_NAME_DEF_STMT (mask);
+		  gcc_assert (is_gimple_assign (g));
+		  bit = gimple_assign_rhs2 (g);
+		}
+	      /* Convert
+		 _1 = __atomic_fetch_or_4 (ptr_6, mask, _3);
+		 _2 = (int) _1;
+		 _5 = _2 & mask;
+		 to
+		 _1 = __atomic_fetch_or_4 (ptr_6, mask, _3);
+		 _6 = _1 & mask;
+		 _5 = (int) _6;
+		 and convert
+		 _1 = ~mask_7;
+		 _2 = (unsigned int) _1;
+		 _3 = __atomic_fetch_and_4 (ptr_6, _2, 0);
+		 _4 = (int) _3;
+		 _5 = _4 & mask_7;
+		 to
+		 _1 = __atomic_fetch_and_* (ptr_6, ~mask_7, _3);
+		 _12 = _3 & mask_7;
+		 _5 = (int) _12;
+	       */
+	      replace_uses_by (use_lhs, lhs);
+	      tree use_nop_lhs = gimple_assign_lhs (use_nop_stmt);
+	      var = make_ssa_name (TREE_TYPE (use_nop_lhs));
+	      gimple_assign_set_lhs (use_nop_stmt, var);
+	      gsi = gsi_for_stmt (use_stmt);
+	      gsi_remove (&gsi, true);
+	      release_defs (use_stmt);
+	      gsi_remove (gsip, true);
+	      g = gimple_build_assign (use_nop_lhs, NOP_EXPR, var);
+	      gsi = gsi_for_stmt (use_nop_stmt);
+	      gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+	      use_stmt = use_nop_stmt;
+	    }
 	}
-      if (!is_gimple_assign (g)
-	  || gimple_assign_rhs_code (g) != LSHIFT_EXPR
-	  || !integer_onep (gimple_assign_rhs1 (g)))
+      else
 	return;
-      bit = gimple_assign_rhs2 (g);
+
+      if (!bit)
+	{
+	  if (ibit < 0)
+	    gcc_unreachable ();
+	  bit = build_int_cst (TREE_TYPE (lhs), ibit);
+	}
     }
-  else
+  else if (optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)))
+	   == CODE_FOR_nothing)
+    return;
+
+  tree use_lhs = gimple_assign_lhs (use_stmt);
+  if (!use_lhs)
     return;
 
-  if (gimple_assign_rhs1 (use_stmt) == lhs)
+  if (!bit)
     {
-      if (!operand_equal_p (gimple_assign_rhs2 (use_stmt), mask, 0))
+      if (TREE_CODE (mask) == INTEGER_CST)
+	{
+	  if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
+	    mask = const_unop (BIT_NOT_EXPR, TREE_TYPE (mask), mask);
+	  mask = fold_convert (TREE_TYPE (lhs), mask);
+	  int ibit = tree_log2 (mask);
+	  if (ibit < 0)
+	    return;
+	  bit = build_int_cst (TREE_TYPE (lhs), ibit);
+	}
+      else if (TREE_CODE (mask) == SSA_NAME)
+	{
+	  gimple *g = SSA_NAME_DEF_STMT (mask);
+	  if (fn == IFN_ATOMIC_BIT_TEST_AND_RESET)
+	    {
+	      if (!is_gimple_assign (g)
+		  || gimple_assign_rhs_code (g) != BIT_NOT_EXPR)
+		return;
+	      mask = gimple_assign_rhs1 (g);
+	      if (TREE_CODE (mask) != SSA_NAME)
+		return;
+	      g = SSA_NAME_DEF_STMT (mask);
+	    }
+	  if (!is_gimple_assign (g))
+	    return;
+	  rhs_code = gimple_assign_rhs_code (g);
+	  if (rhs_code != LSHIFT_EXPR)
+	    {
+	      if (rhs_code != NOP_EXPR)
+		return;
+
+	      /* Handle
+		 _1 = 1 << bit_4(D);
+		 mask_5 = (unsigned int) _1;
+		 _2 = __atomic_fetch_or_4 (v_7(D), mask_5, 0);
+		 _3 = _2 & mask_5;
+		 */
+	      tree nop_lhs = gimple_assign_lhs (g);
+	      tree nop_rhs = gimple_assign_rhs1 (g);
+	      if (TYPE_PRECISION (TREE_TYPE (nop_lhs))
+		  != TYPE_PRECISION (TREE_TYPE (nop_rhs)))
+		return;
+	      g = SSA_NAME_DEF_STMT (nop_rhs);
+	      if (!is_gimple_assign (g)
+		  || gimple_assign_rhs_code (g) != LSHIFT_EXPR)
+		return;
+	    }
+	  if (!integer_onep (gimple_assign_rhs1 (g)))
+	    return;
+	  bit = gimple_assign_rhs2 (g);
+	}
+      else
+	return;
+
+      if (gimple_assign_rhs1 (use_stmt) == lhs)
+	{
+	  if (!operand_equal_p (gimple_assign_rhs2 (use_stmt), mask, 0))
+	    return;
+	}
+      else if (gimple_assign_rhs2 (use_stmt) != lhs
+	       || !operand_equal_p (gimple_assign_rhs1 (use_stmt),
+				    mask, 0))
 	return;
     }
-  else if (gimple_assign_rhs2 (use_stmt) != lhs
-	   || !operand_equal_p (gimple_assign_rhs1 (use_stmt), mask, 0))
-    return;
 
   bool use_bool = true;
   bool has_debug_uses = false;
@@ -3434,28 +3810,27 @@ optimize_atomic_bit_test_and (gimple_stmt_iterator *gsip,
 	 of the specified bit after the atomic operation (makes only sense
 	 for xor, otherwise the bit content is compile time known),
 	 we need to invert the bit.  */
-      g = gimple_build_assign (make_ssa_name (TREE_TYPE (lhs)),
-			       BIT_XOR_EXPR, new_lhs,
-			       use_bool ? build_int_cst (TREE_TYPE (lhs), 1)
-					: mask);
-      new_lhs = gimple_assign_lhs (g);
+      tree mask_convert = mask;
+      gimple_seq stmts = NULL;
+      if (!use_bool)
+	mask_convert = gimple_convert (&stmts, TREE_TYPE (lhs), mask);
+      new_lhs = gimple_build (&stmts, BIT_XOR_EXPR, TREE_TYPE (lhs), new_lhs,
+			      use_bool ? build_int_cst (TREE_TYPE (lhs), 1)
+				       : mask_convert);
       if (throws)
 	{
-	  gsi_insert_on_edge_immediate (e, g);
-	  gsi = gsi_for_stmt (g);
+	  gsi_insert_seq_on_edge_immediate (e, stmts);
+	  gsi = gsi_for_stmt (gimple_seq_last (stmts));
 	}
       else
-	gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+	gsi_insert_seq_after (&gsi, stmts, GSI_NEW_STMT);
     }
   if (use_bool && has_debug_uses)
     {
       tree temp = NULL_TREE;
       if (!throws || after || single_pred_p (e->dest))
 	{
-	  temp = make_node (DEBUG_EXPR_DECL);
-	  DECL_ARTIFICIAL (temp) = 1;
-	  TREE_TYPE (temp) = TREE_TYPE (lhs);
-	  SET_DECL_MODE (temp, TYPE_MODE (TREE_TYPE (lhs)));
+	  temp = build_debug_expr_decl (TREE_TYPE (lhs));
 	  tree t = build2 (LSHIFT_EXPR, TREE_TYPE (lhs), new_lhs, bit);
 	  g = gimple_build_debug_bind (temp, t, g);
 	  if (throws && !after)
diff --git a/gcc/tree-ssa-dce.c b/gcc/tree-ssa-dce.c
index c4907af..dbf02c4 100644
--- a/gcc/tree-ssa-dce.c
+++ b/gcc/tree-ssa-dce.c
@@ -67,6 +67,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-scalar-evolution.h"
 #include "tree-ssa-propagate.h"
 #include "gimple-fold.h"
+#include "tree-ssa.h"
 
 static struct stmt_stats
 {
@@ -436,7 +437,7 @@ find_obviously_necessary_stmts (bool aggressive)
 
       for (auto loop : loops_list (cfun, 0))
 	/* For loops without an exit do not mark any condition.  */
-	if (loop->exits->next && !finite_loop_p (loop))
+	if (loop->exits->next->e && !finite_loop_p (loop))
 	  {
 	    if (dump_file)
 	      fprintf (dump_file, "cannot prove finiteness of loop %i\n",
@@ -1612,6 +1613,164 @@ tree_dce_done (bool aggressive)
   worklist.release ();
 }
 
+/* Sort PHI argument values for make_forwarders_with_degenerate_phis.  */
+
+static int
+sort_phi_args (const void *a_, const void *b_)
+{
+  auto *a = (const std::pair<edge, hashval_t> *) a_;
+  auto *b = (const std::pair<edge, hashval_t> *) b_;
+  hashval_t ha = a->second;
+  hashval_t hb = b->second;
+  if (ha < hb)
+    return -1;
+  else if (ha > hb)
+    return 1;
+  else
+    return 0;
+}
+
+/* Look for a non-virtual PHIs and make a forwarder block when all PHIs
+   have the same argument on a set of edges.  This is to not consider
+   control dependences of individual edges for same values but only for
+   the common set.  */
+
+static unsigned
+make_forwarders_with_degenerate_phis (function *fn)
+{
+  unsigned todo = 0;
+
+  basic_block bb;
+  FOR_EACH_BB_FN (bb, fn)
+    {
+      /* Only PHIs with three or more arguments have opportunities.  */
+      if (EDGE_COUNT (bb->preds) < 3)
+	continue;
+      /* Do not touch loop headers.  */
+      if (bb->loop_father->header == bb)
+	continue;
+
+      /* Take one PHI node as template to look for identical
+	 arguments.  Build a vector of candidates forming sets
+	 of argument edges with equal values.  Note optimality
+	 depends on the particular choice of the template PHI
+	 since equal arguments are unordered leaving other PHIs
+	 with more than one set of equal arguments within this
+	 argument range unsorted.  We'd have to break ties by
+	 looking at other PHI nodes.  */
+      gphi_iterator gsi = gsi_start_nonvirtual_phis (bb);
+      if (gsi_end_p (gsi))
+	continue;
+      gphi *phi = gsi.phi ();
+      auto_vec<std::pair<edge, hashval_t>, 8> args;
+      for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
+	{
+	  edge e = gimple_phi_arg_edge (phi, i);
+	  /* Skip abnormal edges since we cannot redirect them.  */
+	  if (e->flags & EDGE_ABNORMAL)
+	    continue;
+	  /* Skip loop exit edges when we are in loop-closed SSA form
+	     since the forwarder we'd create does not have a PHI node.  */
+	  if (loops_state_satisfies_p (LOOP_CLOSED_SSA)
+	      && loop_exit_edge_p (e->src->loop_father, e))
+	    continue;
+	  args.safe_push (std::make_pair (e, iterative_hash_expr
+					     (gimple_phi_arg_def (phi, i), 0)));
+	}
+      if (args.length () < 2)
+	continue;
+      args.qsort (sort_phi_args);
+
+      /* From the candidates vector now verify true candidates for
+	 forwarders and create them.  */
+      gphi *vphi = get_virtual_phi (bb);
+      unsigned start = 0;
+      while (start < args.length () - 1)
+	{
+	  unsigned i;
+	  for (i = start + 1; i < args.length (); ++i)
+	    if (!operand_equal_p (PHI_ARG_DEF_FROM_EDGE (phi, args[start].first),
+				  PHI_ARG_DEF_FROM_EDGE (phi, args[i].first)))
+	      break;
+	  /* args[start]..args[i-1] are equal.  */
+	  if (start != i - 1)
+	    {
+	      /* Check all PHI nodes for argument equality.  */
+	      bool equal = true;
+	      gphi_iterator gsi2 = gsi;
+	      gsi_next (&gsi2);
+	      for (; !gsi_end_p (gsi2); gsi_next (&gsi2))
+		{
+		  gphi *phi2 = gsi2.phi ();
+		  if (virtual_operand_p (gimple_phi_result (phi2)))
+		    continue;
+		  tree start_arg
+		    = PHI_ARG_DEF_FROM_EDGE (phi2, args[start].first);
+		  for (unsigned j = start + 1; j < i; ++j)
+		    {
+		      if (!operand_equal_p (start_arg,
+					    PHI_ARG_DEF_FROM_EDGE
+					      (phi2, args[j].first)))
+			{
+			  /* Another PHI might have a shorter set of
+			     equivalent args.  Go for that.  */
+			  i = j;
+			  if (j == start + 1)
+			    equal = false;
+			  break;
+			}
+		    }
+		  if (!equal)
+		    break;
+		}
+	      if (equal)
+		{
+		  /* If we are asked to forward all edges the block
+		     has all degenerate PHIs.  Do nothing in that case.  */
+		  if (start == 0
+		      && i == args.length ()
+		      && args.length () == gimple_phi_num_args (phi))
+		    break;
+		  /* Instead of using make_forwarder_block we are
+		     rolling our own variant knowing that the forwarder
+		     does not need PHI nodes apart from eventually
+		     a virtual one.  */
+		  auto_vec<tree, 8> vphi_args;
+		  if (vphi)
+		    {
+		      vphi_args.reserve_exact (i - start);
+		      for (unsigned j = start; j < i; ++j)
+			vphi_args.quick_push
+			  (PHI_ARG_DEF_FROM_EDGE (vphi, args[j].first));
+		    }
+		  free_dominance_info (fn, CDI_DOMINATORS);
+		  basic_block forwarder = split_edge (args[start].first);
+		  for (unsigned j = start + 1; j < i; ++j)
+		    {
+		      edge e = args[j].first;
+		      redirect_edge_and_branch_force (e, forwarder);
+		      redirect_edge_var_map_clear (e);
+		    }
+		  if (vphi)
+		    {
+		      tree def = copy_ssa_name (vphi_args[0]);
+		      gphi *vphi_copy = create_phi_node (def, forwarder);
+		      for (unsigned j = start; j < i; ++j)
+			add_phi_arg (vphi_copy, vphi_args[j - start],
+				     args[j].first, UNKNOWN_LOCATION);
+		      SET_PHI_ARG_DEF
+			(vphi, single_succ_edge (forwarder)->dest_idx, def);
+		    }
+		  todo |= TODO_cleanup_cfg;
+		}
+	    }
+	  /* Continue searching for more opportunities.  */
+	  start = i;
+	}
+    }
+  return todo;
+}
+
 /* Main routine to eliminate dead code.
 
    AGGRESSIVE controls the aggressiveness of the algorithm.
@@ -1630,8 +1789,7 @@ static unsigned int
 perform_tree_ssa_dce (bool aggressive)
 {
   bool something_changed = 0;
-
-  calculate_dominance_info (CDI_DOMINATORS);
+  unsigned todo = 0;
 
   /* Preheaders are needed for SCEV to work.
      Simple lateches and recorded exits improve chances that loop will
@@ -1644,6 +1802,11 @@ perform_tree_ssa_dce (bool aggressive)
 			   | LOOPS_HAVE_RECORDED_EXITS);
     }
 
+  if (aggressive)
+    todo |= make_forwarders_with_degenerate_phis (cfun);
+
+  calculate_dominance_info (CDI_DOMINATORS);
+
   tree_dce_init (aggressive);
 
   if (aggressive)
@@ -1701,9 +1864,9 @@ perform_tree_ssa_dce (bool aggressive)
       free_numbers_of_iterations_estimates (cfun);
       if (in_loop_pipeline)
 	scev_reset ();
-      return TODO_update_ssa | TODO_cleanup_cfg;
+      todo |= TODO_update_ssa | TODO_cleanup_cfg;
     }
-  return 0;
+  return todo;
 }
 
 /* Pass entry points.  */
@@ -1828,6 +1991,11 @@ simple_dce_from_worklist (bitmap worklist)
       if (gimple_has_side_effects (t))
 	continue;
 
+      /* Don't remove statements that are needed for non-call
+	 eh to work.  */
+      if (stmt_unremovable_because_of_non_call_eh_p (cfun, t))
+	continue;
+
       /* Add uses to the worklist.  */
       ssa_op_iter iter;
       use_operand_p use_p;
diff --git a/gcc/tree-ssa-dse.c b/gcc/tree-ssa-dse.c
index 27287fe..ce0083a 100644
--- a/gcc/tree-ssa-dse.c
+++ b/gcc/tree-ssa-dse.c
@@ -40,6 +40,9 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimplify.h"
 #include "tree-eh.h"
 #include "cfganal.h"
+#include "cgraph.h"
+#include "ipa-modref-tree.h"
+#include "ipa-modref.h"
 
 /* This file implements dead store elimination.
 
@@ -1027,6 +1030,91 @@ delete_dead_or_redundant_assignment (gimple_stmt_iterator *gsi, const char *type
   release_defs (stmt);
 }
 
+/* Try to prove, using modref summary, that all memory written to by a call is
+   dead and remove it.  Assume that if return value is written to memory
+   it is already proved to be dead.  */
+
+static bool
+dse_optimize_call (gimple_stmt_iterator *gsi, sbitmap live_bytes)
+{
+  gcall *stmt = dyn_cast <gcall *> (gsi_stmt (*gsi));
+
+  if (!stmt)
+    return false;
+
+  tree callee = gimple_call_fndecl (stmt);
+
+  if (!callee)
+    return false;
+
+  /* Pure/const functions are optimized by normal DCE
+     or handled as store above.  */
+  int flags = gimple_call_flags (stmt);
+  if ((flags & (ECF_PURE|ECF_CONST|ECF_NOVOPS))
+      && !(flags & (ECF_LOOPING_CONST_OR_PURE)))
+    return false;
+
+  cgraph_node *node = cgraph_node::get (callee);
+  if (!node)
+    return false;
+
+  if (stmt_could_throw_p (cfun, stmt)
+      && !cfun->can_delete_dead_exceptions)
+    return false;
+
+  /* If return value is used the call is not dead.  */
+  tree lhs = gimple_call_lhs (stmt);
+  if (lhs && TREE_CODE (lhs) == SSA_NAME)
+    {
+      imm_use_iterator ui;
+      gimple *use_stmt;
+      FOR_EACH_IMM_USE_STMT (use_stmt, ui, lhs)
+	if (!is_gimple_debug (use_stmt))
+	  return false;
+    }
+
+  /* Verify that there are no side-effects except for return value
+     and memory writes tracked by modref.  */
+  modref_summary *summary = get_modref_function_summary (node);
+  if (!summary || !summary->try_dse)
+    return false;
+
+  bool by_clobber_p = false;
+
+  /* Walk all memory writes and verify that they are dead.  */
+  for (auto base_node : summary->stores->bases)
+    for (auto ref_node : base_node->refs)
+      for (auto access_node : ref_node->accesses)
+	{
+	  tree arg = access_node.get_call_arg (stmt);
+
+	  if (!arg)
+	    return false;
+
+	  if (integer_zerop (arg) && flag_delete_null_pointer_checks)
+	    continue;
+
+	  ao_ref ref;
+
+	  if (!access_node.get_ao_ref (stmt, &ref))
+	    return false;
+	  ref.ref_alias_set = ref_node->ref;
+	  ref.base_alias_set = base_node->base;
+
+	  bool byte_tracking_enabled
+	      = setup_live_bytes_from_ref (&ref, live_bytes);
+	  enum dse_store_status store_status;
+
+	  store_status = dse_classify_store (&ref, stmt,
+					     byte_tracking_enabled,
+					     live_bytes, &by_clobber_p);
+	  if (store_status != DSE_STORE_DEAD)
+	    return false;
+	}
+  delete_dead_or_redundant_assignment (gsi, "dead", need_eh_cleanup);
+  return true;
+}
+
 /* Attempt to eliminate dead stores in the statement referenced by BSI.
 
    A dead store is a store into a memory location which will later be
@@ -1050,8 +1138,13 @@ dse_optimize_stmt (function *fun, gimple_stmt_iterator *gsi, sbitmap live_bytes)
     return;
 
   ao_ref ref;
+  /* If this is not a store we can still remove dead call using
+     modref summary.  */
   if (!initialize_ao_ref_for_dse (stmt, &ref))
-    return;
+    {
+      dse_optimize_call (gsi, live_bytes);
+      return;
+    }
 
   /* We know we have virtual definitions.  We can handle assignments and
      some builtin calls.  */
@@ -1162,6 +1255,9 @@ dse_optimize_stmt (function *fun, gimple_stmt_iterator *gsi, sbitmap live_bytes)
 	  || (stmt_could_throw_p (fun, stmt)
 	      && !fun->can_delete_dead_exceptions)))
     {
+      /* See if we can remove complete call.  */
+      if (dse_optimize_call (gsi, live_bytes))
+	return;
       /* Make sure we do not remove a return slot we cannot reconstruct
 	 later.  */
       if (gimple_call_return_slot_opt_p (as_a <gcall *>(stmt))
diff --git a/gcc/tree-ssa-forwprop.c b/gcc/tree-ssa-forwprop.c
index 5b30d4c..a830bab 100644
--- a/gcc/tree-ssa-forwprop.c
+++ b/gcc/tree-ssa-forwprop.c
@@ -2267,8 +2267,6 @@ simplify_permutation (gimple_stmt_iterator *gsi)
 	  if (!VECTOR_TYPE_P (tgt_type))
 	    return 0;
 	  tree op2_type = TREE_TYPE (op2);
-	  /* Should have folded this before.  */
-	  gcc_assert (op2_type != tgt_type);
 
 	  /* Figure out the shrunk factor.  */
 	  poly_uint64 tgt_units = TYPE_VECTOR_SUBPARTS (tgt_type);
diff --git a/gcc/tree-ssa-loop-ch.c b/gcc/tree-ssa-loop-ch.c
index ffb0aa8..566cc27 100644
--- a/gcc/tree-ssa-loop-ch.c
+++ b/gcc/tree-ssa-loop-ch.c
@@ -35,12 +35,48 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-ssa-sccvn.h"
 #include "tree-phinodes.h"
 #include "ssa-iterators.h"
+#include "value-range.h"
+#include "gimple-range.h"
+#include "gimple-range-path.h"
 
 /* Duplicates headers of loops if they are small enough, so that the statements
    in the loop body are always executed when the loop is entered.  This
    increases effectiveness of code motion optimizations, and reduces the need
    for loop preconditioning.  */
 
+/* Return true if the condition on the first iteration of the loop can
+   be statically determined.  */
+
+static bool
+entry_loop_condition_is_static (class loop *l, path_range_query *query)
+{
+  edge e = loop_preheader_edge (l);
+  gcond *last = safe_dyn_cast <gcond *> (last_stmt (e->dest));
+
+  if (!last
+      || !irange::supports_type_p (TREE_TYPE (gimple_cond_lhs (last))))
+    return false;
+
+  edge true_e, false_e;
+  extract_true_false_edges_from_block (e->dest, &true_e, &false_e);
+
+  /* If neither edge is the exit edge, this is not a case we'd like to
+     special-case.  */
+  if (!loop_exit_edge_p (l, true_e) && !loop_exit_edge_p (l, false_e))
+    return false;
+
+  tree desired_static_value;
+  if (loop_exit_edge_p (l, true_e))
+    desired_static_value = boolean_false_node;
+  else
+    desired_static_value = boolean_true_node;
+
+  int_range<2> r;
+  query->compute_ranges (e);
+  query->range_of_stmt (r, last);
+  return r == int_range<2> (desired_static_value, desired_static_value);
+}
+
 /* Check whether we should duplicate HEADER of LOOP.  At most *LIMIT
    instructions should be duplicated, limit is decreased by the actual
    amount.  */
@@ -53,20 +89,6 @@ should_duplicate_loop_header_p (basic_block header, class loop *loop,
 
   gcc_assert (!header->aux);
 
-  /* Loop header copying usually increases size of the code.  This used not to
-     be true, since quite often it is possible to verify that the condition is
-     satisfied in the first iteration and therefore to eliminate it.  Jump
-     threading handles these cases now.  */
-  if (optimize_loop_for_size_p (loop)
-      && !loop->force_vectorize)
-    {
-      if (dump_file && (dump_flags & TDF_DETAILS))
-	fprintf (dump_file,
-		 "  Not duplicating bb %i: optimizing for size.\n",
-		 header->index);
-      return false;
-    }
-
   gcc_assert (EDGE_COUNT (header->succs) > 0);
   if (single_succ_p (header))
     {
@@ -201,8 +223,6 @@ should_duplicate_loop_header_p (basic_block header, class loop *loop,
       return false;
     }
 
-  if (dump_file && (dump_flags & TDF_DETAILS))
-    fprintf (dump_file, "    Will duplicate bb %i\n", header->index); 
   return true;
 }
 
@@ -361,8 +381,10 @@ ch_base::copy_headers (function *fun)
   copied_bbs = XNEWVEC (basic_block, n_basic_blocks_for_fn (fun));
   bbs_size = n_basic_blocks_for_fn (fun);
 
+  auto_vec<loop_p> candidates;
   auto_vec<std::pair<edge, loop_p> > copied;
 
+  path_range_query *query = new path_range_query;
   for (auto loop : loops_list (cfun, 0))
     {
       int initial_limit = param_max_loop_header_insns;
@@ -381,6 +403,36 @@ ch_base::copy_headers (function *fun)
 	  || !process_loop_p (loop))
 	continue;
 
+      /* Avoid loop header copying when optimizing for size unless we can
+	 determine that the loop condition is static in the first
+	 iteration.  */
+      if (optimize_loop_for_size_p (loop)
+	  && !loop->force_vectorize
+	  && !entry_loop_condition_is_static (loop, query))
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    fprintf (dump_file,
+		     "  Not duplicating bb %i: optimizing for size.\n",
+		     header->index);
+	  continue;
+	}
+
+      if (should_duplicate_loop_header_p (header, loop, &remaining_limit))
+	candidates.safe_push (loop);
+    }
+  /* Do not use ranger after we change the IL and not have updated SSA.  */
+  delete query;
+
+  for (auto loop : candidates)
+    {
+      int initial_limit = param_max_loop_header_insns;
+      int remaining_limit = initial_limit;
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	fprintf (dump_file,
+		 "Copying headers of loop %i\n", loop->num);
+
+      header = loop->header;
+
       /* Iterate the header copying up to limit; this takes care of the cases
 	 like while (a && b) {...}, where we want to have both of the conditions
 	 copied.  TODO -- handle while (a || b) - like cases, by not requiring
@@ -391,6 +443,9 @@ ch_base::copy_headers (function *fun)
       n_bbs = 0;
       while (should_duplicate_loop_header_p (header, loop, &remaining_limit))
 	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    fprintf (dump_file, "    Will duplicate bb %i\n", header->index);
+
 	  /* Find a successor of header that is inside a loop; i.e. the new
 	     header after the condition is copied.  */
 	  if (flow_bb_inside_loop_p (loop, EDGE_SUCC (header, 0)->dest))
diff --git a/gcc/tree-ssa-loop-ivcanon.c b/gcc/tree-ssa-loop-ivcanon.c
index 8d8791f..be533b0 100644
--- a/gcc/tree-ssa-loop-ivcanon.c
+++ b/gcc/tree-ssa-loop-ivcanon.c
@@ -903,11 +903,10 @@ try_unroll_loop_completely (class loop *loop,
       if (may_be_zero)
 	bitmap_clear_bit (wont_exit, 1);
 
-      if (!gimple_duplicate_loop_to_header_edge (loop, loop_preheader_edge (loop),
-						 n_unroll, wont_exit,
-						 exit, &edges_to_remove,
-						 DLTHE_FLAG_UPDATE_FREQ
-						 | DLTHE_FLAG_COMPLETTE_PEEL))
+      if (!gimple_duplicate_loop_body_to_header_edge (
+	    loop, loop_preheader_edge (loop), n_unroll, wont_exit, exit,
+	    &edges_to_remove,
+	    DLTHE_FLAG_UPDATE_FREQ | DLTHE_FLAG_COMPLETTE_PEEL))
 	{
           free_original_copy_tables ();
 	  if (dump_file && (dump_flags & TDF_DETAILS))
@@ -1094,10 +1093,9 @@ try_peel_loop (class loop *loop,
     }
   if (may_be_zero)
     bitmap_clear_bit (wont_exit, 1);
-  if (!gimple_duplicate_loop_to_header_edge (loop, loop_preheader_edge (loop),
-					     npeel, wont_exit,
-					     exit, &edges_to_remove,
-					     DLTHE_FLAG_UPDATE_FREQ))
+  if (!gimple_duplicate_loop_body_to_header_edge (
+	loop, loop_preheader_edge (loop), npeel, wont_exit, exit,
+	&edges_to_remove, DLTHE_FLAG_UPDATE_FREQ))
     {
       free_original_copy_tables ();
       return false;
diff --git a/gcc/tree-ssa-loop-ivopts.c b/gcc/tree-ssa-loop-ivopts.c
index 4a498ab..5a7fd30 100644
--- a/gcc/tree-ssa-loop-ivopts.c
+++ b/gcc/tree-ssa-loop-ivopts.c
@@ -7742,9 +7742,8 @@ remove_unused_ivs (struct ivopts_data *data, bitmap toremove)
 	      comp = unshare_expr (comp);
 	      if (count > 1)
 		{
-		  tree vexpr = make_node (DEBUG_EXPR_DECL);
-		  DECL_ARTIFICIAL (vexpr) = 1;
-		  TREE_TYPE (vexpr) = TREE_TYPE (comp);
+		  tree vexpr = build_debug_expr_decl (TREE_TYPE (comp));
+		  /* FIXME: Is setting the mode really necessary? */
 		  if (SSA_NAME_VAR (def))
 		    SET_DECL_MODE (vexpr, DECL_MODE (SSA_NAME_VAR (def)));
 		  else
diff --git a/gcc/tree-ssa-loop-manip.c b/gcc/tree-ssa-loop-manip.c
index 2d576bf..1fb2450 100644
--- a/gcc/tree-ssa-loop-manip.c
+++ b/gcc/tree-ssa-loop-manip.c
@@ -911,7 +911,7 @@ copy_phi_node_args (unsigned first_new_block)
 }
 
 
-/* The same as cfgloopmanip.c:duplicate_loop_to_header_edge, but also
+/* The same as cfgloopmanip.c:duplicate_loop_body_to_header_edge, but also
    updates the PHI nodes at start of the copied region.  In order to
    achieve this, only loops whose exits all lead to the same location
    are handled.
@@ -921,10 +921,10 @@ copy_phi_node_args (unsigned first_new_block)
    after the loop has been duplicated.  */
 
 bool
-gimple_duplicate_loop_to_header_edge (class loop *loop, edge e,
-				    unsigned int ndupl, sbitmap wont_exit,
-				    edge orig, vec<edge> *to_remove,
-				    int flags)
+gimple_duplicate_loop_body_to_header_edge (class loop *loop, edge e,
+					   unsigned int ndupl,
+					   sbitmap wont_exit, edge orig,
+					   vec<edge> *to_remove, int flags)
 {
   unsigned first_new_block;
 
@@ -934,8 +934,8 @@ gimple_duplicate_loop_to_header_edge (class loop *loop, edge e,
     return false;
 
   first_new_block = last_basic_block_for_fn (cfun);
-  if (!duplicate_loop_to_header_edge (loop, e, ndupl, wont_exit,
-				      orig, to_remove, flags))
+  if (!duplicate_loop_body_to_header_edge (loop, e, ndupl, wont_exit, orig,
+					   to_remove, flags))
     return false;
 
   /* Readd the removed phi args for e.  */
@@ -1388,9 +1388,11 @@ tree_transform_and_unroll_loop (class loop *loop, unsigned factor,
   bitmap_clear_bit (wont_exit, factor - 1);
 
   auto_vec<edge> to_remove;
-  bool ok = gimple_duplicate_loop_to_header_edge
-	  (loop, loop_latch_edge (loop), factor - 1,
-	   wont_exit, new_exit, &to_remove, DLTHE_FLAG_UPDATE_FREQ);
+  bool ok
+    = gimple_duplicate_loop_body_to_header_edge (loop, loop_latch_edge (loop),
+						 factor - 1, wont_exit,
+						 new_exit, &to_remove,
+						 DLTHE_FLAG_UPDATE_FREQ);
   gcc_assert (ok);
 
   for (edge e : to_remove)
diff --git a/gcc/tree-ssa-loop-manip.h b/gcc/tree-ssa-loop-manip.h
index 5e2d7fa..4f604e1 100644
--- a/gcc/tree-ssa-loop-manip.h
+++ b/gcc/tree-ssa-loop-manip.h
@@ -42,10 +42,9 @@ extern basic_block ip_end_pos (class loop *);
 extern basic_block ip_normal_pos (class loop *);
 extern void standard_iv_increment_position (class loop *,
 					    gimple_stmt_iterator *, bool *);
-extern bool gimple_duplicate_loop_to_header_edge (class loop *, edge,
-						  unsigned int, sbitmap,
-						  edge, vec<edge> *,
-						  int);
+extern bool
+gimple_duplicate_loop_body_to_header_edge (class loop *, edge, unsigned int,
+					   sbitmap, edge, vec<edge> *, int);
 extern bool can_unroll_loop_p (class loop *loop, unsigned factor,
 			       class tree_niter_desc *niter);
 extern gcov_type niter_for_unrolled_loop (class loop *, unsigned);
diff --git a/gcc/tree-ssa-loop.c b/gcc/tree-ssa-loop.c
index 1bbf2f1..6b54e2b 100644
--- a/gcc/tree-ssa-loop.c
+++ b/gcc/tree-ssa-loop.c
@@ -370,57 +370,6 @@ make_pass_tree_loop_init (gcc::context *ctxt)
   return new pass_tree_loop_init (ctxt);
 }
 
-/* Loop autovectorization.  */
-
-namespace {
-
-const pass_data pass_data_vectorize =
-{
-  GIMPLE_PASS, /* type */
-  "vect", /* name */
-  OPTGROUP_LOOP | OPTGROUP_VEC, /* optinfo_flags */
-  TV_TREE_VECTORIZATION, /* tv_id */
-  ( PROP_cfg | PROP_ssa ), /* properties_required */
-  0, /* properties_provided */
-  0, /* properties_destroyed */
-  0, /* todo_flags_start */
-  0, /* todo_flags_finish */
-};
-
-class pass_vectorize : public gimple_opt_pass
-{
-public:
-  pass_vectorize (gcc::context *ctxt)
-    : gimple_opt_pass (pass_data_vectorize, ctxt)
-  {}
-
-  /* opt_pass methods: */
-  virtual bool gate (function *fun)
-    {
-      return flag_tree_loop_vectorize || fun->has_force_vectorize_loops;
-    }
-
-  virtual unsigned int execute (function *);
-
-}; // class pass_vectorize
-
-unsigned int
-pass_vectorize::execute (function *fun)
-{
-  if (number_of_loops (fun) <= 1)
-    return 0;
-
-  return vectorize_loops ();
-}
-
-} // anon namespace
-
-gimple_opt_pass *
-make_pass_vectorize (gcc::context *ctxt)
-{
-  return new pass_vectorize (ctxt);
-}
-
 /* Propagation of constants using scev.  */
 
 namespace {
diff --git a/gcc/tree-ssa-phiopt.c b/gcc/tree-ssa-phiopt.c
index 0e339c4..6b22f6b 100644
--- a/gcc/tree-ssa-phiopt.c
+++ b/gcc/tree-ssa-phiopt.c
@@ -220,7 +220,6 @@ tree_ssa_phiopt_worker (bool do_store_elim, bool do_hoist_loads, bool early_p)
 
       /* If either bb1's succ or bb2 or bb2's succ is non NULL.  */
       if (EDGE_COUNT (bb1->succs) == 0
-          || bb2 == NULL
 	  || EDGE_COUNT (bb2->succs) == 0)
         continue;
 
@@ -276,14 +275,14 @@ tree_ssa_phiopt_worker (bool do_store_elim, bool do_hoist_loads, bool early_p)
 	  || (e1->flags & EDGE_FALLTHRU) == 0)
         continue;
 
-      /* Also make sure that bb1 only have one predecessor and that it
-	 is bb.  */
-      if (!single_pred_p (bb1)
-          || single_pred (bb1) != bb)
-	continue;
-
       if (do_store_elim)
 	{
+	  /* Also make sure that bb1 only have one predecessor and that it
+	     is bb.  */
+	  if (!single_pred_p (bb1)
+	      || single_pred (bb1) != bb)
+	    continue;
+
 	  /* bb1 is the middle block, bb2 the join block, bb the split block,
 	     e1 the fallthrough edge from bb1 to bb2.  We can't do the
 	     optimization if the join block has more than two predecessors.  */
@@ -328,10 +327,11 @@ tree_ssa_phiopt_worker (bool do_store_elim, bool do_hoist_loads, bool early_p)
 	     node.  */
 	  gcc_assert (arg0 != NULL_TREE && arg1 != NULL_TREE);
 
-	  gphi *newphi = factor_out_conditional_conversion (e1, e2, phi,
-							    arg0, arg1,
-							    cond_stmt);
-	  if (newphi != NULL)
+	  gphi *newphi;
+	  if (single_pred_p (bb1)
+	      && (newphi = factor_out_conditional_conversion (e1, e2, phi,
+							      arg0, arg1,
+							      cond_stmt)))
 	    {
 	      phi = newphi;
 	      /* factor_out_conditional_conversion may create a new PHI in
@@ -350,12 +350,14 @@ tree_ssa_phiopt_worker (bool do_store_elim, bool do_hoist_loads, bool early_p)
 					       early_p))
 	    cfgchanged = true;
 	  else if (!early_p
+		   && single_pred_p (bb1)
 		   && cond_removal_in_builtin_zero_pattern (bb, bb1, e1, e2,
 							    phi, arg0, arg1))
 	    cfgchanged = true;
 	  else if (minmax_replacement (bb, bb1, e1, e2, phi, arg0, arg1))
 	    cfgchanged = true;
-	  else if (spaceship_replacement (bb, bb1, e1, e2, phi, arg0, arg1))
+	  else if (single_pred_p (bb1)
+		   && spaceship_replacement (bb, bb1, e1, e2, phi, arg0, arg1))
 	    cfgchanged = true;
 	}
     }
@@ -386,7 +388,6 @@ replace_phi_edge_with_variable (basic_block cond_block,
 				edge e, gphi *phi, tree new_tree)
 {
   basic_block bb = gimple_bb (phi);
-  basic_block block_to_remove;
   gimple_stmt_iterator gsi;
   tree phi_result = PHI_RESULT (phi);
 
@@ -422,28 +423,33 @@ replace_phi_edge_with_variable (basic_block cond_block,
   SET_USE (PHI_ARG_DEF_PTR (phi, e->dest_idx), new_tree);
 
   /* Remove the empty basic block.  */
+  edge edge_to_remove;
   if (EDGE_SUCC (cond_block, 0)->dest == bb)
+    edge_to_remove = EDGE_SUCC (cond_block, 1);
+  else
+    edge_to_remove = EDGE_SUCC (cond_block, 0);
+  if (EDGE_COUNT (edge_to_remove->dest->preds) == 1)
     {
-      EDGE_SUCC (cond_block, 0)->flags |= EDGE_FALLTHRU;
-      EDGE_SUCC (cond_block, 0)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
-      EDGE_SUCC (cond_block, 0)->probability = profile_probability::always ();
+      e->flags |= EDGE_FALLTHRU;
+      e->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
+      e->probability = profile_probability::always ();
+      delete_basic_block (edge_to_remove->dest);
 
-      block_to_remove = EDGE_SUCC (cond_block, 1)->dest;
+      /* Eliminate the COND_EXPR at the end of COND_BLOCK.  */
+      gsi = gsi_last_bb (cond_block);
+      gsi_remove (&gsi, true);
     }
   else
     {
-      EDGE_SUCC (cond_block, 1)->flags |= EDGE_FALLTHRU;
-      EDGE_SUCC (cond_block, 1)->flags
-	&= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
-      EDGE_SUCC (cond_block, 1)->probability = profile_probability::always ();
-
-      block_to_remove = EDGE_SUCC (cond_block, 0)->dest;
+      /* If there are other edges into the middle block make
+	 CFG cleanup deal with the edge removal to avoid
+	 updating dominators here in a non-trivial way.  */
+      gcond *cond = as_a <gcond *> (last_stmt (cond_block));
+      if (edge_to_remove->flags & EDGE_TRUE_VALUE)
+	gimple_cond_make_false (cond);
+      else
+	gimple_cond_make_true (cond);
     }
-  delete_basic_block (block_to_remove);
-
-  /* Eliminate the COND_EXPR at the end of COND_BLOCK.  */
-  gsi = gsi_last_bb (cond_block);
-  gsi_remove (&gsi, true);
 
   statistics_counter_event (cfun, "Replace PHI with variable", 1);
 
@@ -959,6 +965,9 @@ match_simplify_replacement (basic_block cond_bb, basic_block middle_bb,
      allow it and move it once the transformation is done. */
   if (!empty_block_p (middle_bb))
     {
+      if (!single_pred_p (middle_bb))
+	return false;
+
       stmt_to_move = last_and_only_stmt (middle_bb);
       if (!stmt_to_move)
 	return false;
@@ -1351,7 +1360,10 @@ value_replacement (basic_block cond_bb, basic_block middle_bb,
 	}
       else
 	{
-	  statistics_counter_event (cfun, "Replace PHI with variable/value_replacement", 1);
+	  if (!single_pred_p (middle_bb))
+	    return 0;
+	  statistics_counter_event (cfun, "Replace PHI with "
+				    "variable/value_replacement", 1);
 
 	  /* Replace the PHI arguments with arg. */
 	  SET_PHI_ARG_DEF (phi, e0->dest_idx, arg);
@@ -1367,7 +1379,6 @@ value_replacement (basic_block cond_bb, basic_block middle_bb,
             }
           return 1;
 	}
-
     }
 
   /* Now optimize (x != 0) ? x + y : y to just x + y.  */
@@ -2429,19 +2440,13 @@ spaceship_replacement (basic_block cond_bb, basic_block middle_bb,
 	     all floating point numbers should be comparable.  */
 	  gimple_stmt_iterator gsi = gsi_after_labels (gimple_bb (phi));
 	  tree type = TREE_TYPE (phires);
-	  tree temp1 = make_node (DEBUG_EXPR_DECL);
-	  DECL_ARTIFICIAL (temp1) = 1;
-	  TREE_TYPE (temp1) = type;
-	  SET_DECL_MODE (temp1, TYPE_MODE (type));
+	  tree temp1 = build_debug_expr_decl (type);
 	  tree t = build2 (one_cmp, boolean_type_node, lhs1, rhs2);
 	  t = build3 (COND_EXPR, type, t, build_one_cst (type),
 		      build_int_cst (type, -1));
 	  gimple *g = gimple_build_debug_bind (temp1, t, phi);
 	  gsi_insert_before (&gsi, g, GSI_SAME_STMT);
-	  tree temp2 = make_node (DEBUG_EXPR_DECL);
-	  DECL_ARTIFICIAL (temp2) = 1;
-	  TREE_TYPE (temp2) = type;
-	  SET_DECL_MODE (temp2, TYPE_MODE (type));
+	  tree temp2 = build_debug_expr_decl (type);
 	  t = build2 (EQ_EXPR, boolean_type_node, lhs1, rhs2);
 	  t = build3 (COND_EXPR, type, t, build_zero_cst (type), temp1);
 	  g = gimple_build_debug_bind (temp2, t, phi);
diff --git a/gcc/tree-ssa-pre.c b/gcc/tree-ssa-pre.c
index 1cc1aae..0669aaa 100644
--- a/gcc/tree-ssa-pre.c
+++ b/gcc/tree-ssa-pre.c
@@ -1391,6 +1391,7 @@ get_representative_for (const pre_expr e, basic_block b = NULL)
   vn_ssa_aux_t vn_info = VN_INFO (name);
   vn_info->value_id = value_id;
   vn_info->valnum = valnum ? valnum : name;
+  vn_info->visited = true;
   /* ???  For now mark this SSA name for release by VN.  */
   vn_info->needs_insertion = true;
   add_to_value (value_id, get_or_alloc_expr_for_name (name));
@@ -1508,10 +1509,6 @@ phi_translate_1 (bitmap_set_t dest,
 		  return constant;
 	      }
 
-	    /* vn_nary_* do not valueize operands.  */
-	    for (i = 0; i < newnary->length; ++i)
-	      if (TREE_CODE (newnary->op[i]) == SSA_NAME)
-		newnary->op[i] = VN_INFO (newnary->op[i])->valnum;
 	    tree result = vn_nary_op_lookup_pieces (newnary->length,
 						    newnary->opcode,
 						    newnary->type,
@@ -1588,6 +1585,21 @@ phi_translate_1 (bitmap_set_t dest,
 		newoperands.release ();
 		return NULL;
 	      }
+	    /* When we translate a MEM_REF across a backedge and we have
+	       restrict info that's not from our functions parameters
+	       we have to remap it since we now may deal with a different
+	       instance where the dependence info is no longer valid.
+	       See PR102970.  Note instead of keeping a remapping table
+	       per backedge we simply throw away restrict info.  */
+	    if ((newop.opcode == MEM_REF
+		 || newop.opcode == TARGET_MEM_REF)
+		&& newop.clique > 1
+		&& (e->flags & EDGE_DFS_BACK))
+	      {
+		newop.clique = 0;
+		newop.base = 0;
+		changed = true;
+	      }
 	    if (!changed)
 	      continue;
 	    if (!newoperands.exists ())
diff --git a/gcc/tree-ssa-reassoc.c b/gcc/tree-ssa-reassoc.c
index db9fb4e..6531622 100644
--- a/gcc/tree-ssa-reassoc.c
+++ b/gcc/tree-ssa-reassoc.c
@@ -1259,15 +1259,12 @@ make_new_ssa_for_def (gimple *stmt, enum tree_code opcode, tree op)
 	{
 	  if (new_debug_lhs == NULL_TREE)
 	    {
-	      new_debug_lhs = make_node (DEBUG_EXPR_DECL);
+	      new_debug_lhs = build_debug_expr_decl (TREE_TYPE (lhs));
 	      gdebug *def_temp
 		= gimple_build_debug_bind (new_debug_lhs,
 					   build2 (opcode, TREE_TYPE (lhs),
 						   new_lhs, op),
 					   stmt);
-	      DECL_ARTIFICIAL (new_debug_lhs) = 1;
-	      TREE_TYPE (new_debug_lhs) = TREE_TYPE (lhs);
-	      SET_DECL_MODE (new_debug_lhs, TYPE_MODE (TREE_TYPE (lhs)));
 	      gimple_set_uid (def_temp, gimple_uid (stmt));
 	      gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
 	      gsi_insert_after (&gsi, def_temp, GSI_SAME_STMT);
@@ -1518,7 +1515,8 @@ insert_stmt_after (gimple *stmt, gimple *insert_point)
       gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
       return;
     }
-  else if (gimple_code (insert_point) == GIMPLE_ASM)
+  else if (gimple_code (insert_point) == GIMPLE_ASM
+	   && gimple_asm_nlabels (as_a <gasm *> (insert_point)) != 0)
     /* We have no idea where to insert - it depends on where the
        uses will be placed.  */
     gcc_unreachable ();
diff --git a/gcc/tree-ssa-sccvn.c b/gcc/tree-ssa-sccvn.c
index ae0172a..149674e 100644
--- a/gcc/tree-ssa-sccvn.c
+++ b/gcc/tree-ssa-sccvn.c
@@ -427,7 +427,7 @@ static vn_nary_op_t vn_nary_op_insert_stmt (gimple *, tree);
 static unsigned int vn_nary_length_from_stmt (gimple *);
 static vn_nary_op_t alloc_vn_nary_op_noinit (unsigned int, obstack *);
 static vn_nary_op_t vn_nary_op_insert_into (vn_nary_op_t,
-					    vn_nary_op_table_type *, bool);
+					    vn_nary_op_table_type *);
 static void init_vn_nary_op_from_stmt (vn_nary_op_t, gassign *);
 static void init_vn_nary_op_from_pieces (vn_nary_op_t, unsigned int,
 					 enum tree_code, tree, tree *);
@@ -490,7 +490,7 @@ VN_INFO (tree name)
 					 boolean_type_node, ops);
 	    nary->predicated_values = 0;
 	    nary->u.result = boolean_true_node;
-	    vn_nary_op_insert_into (nary, valid_info->nary, true);
+	    vn_nary_op_insert_into (nary, valid_info->nary);
 	    gcc_assert (nary->unwind_to == NULL);
 	    /* Also do not link it into the undo chain.  */
 	    last_inserted_nary = nary->next;
@@ -500,7 +500,7 @@ VN_INFO (tree name)
 					 boolean_type_node, ops);
 	    nary->predicated_values = 0;
 	    nary->u.result = boolean_false_node;
-	    vn_nary_op_insert_into (nary, valid_info->nary, true);
+	    vn_nary_op_insert_into (nary, valid_info->nary);
 	    gcc_assert (nary->unwind_to == NULL);
 	    last_inserted_nary = nary->next;
 	    nary->next = (vn_nary_op_t)(void *)-1;
@@ -1640,13 +1640,12 @@ static void
 valueize_refs_1 (vec<vn_reference_op_s> *orig, bool *valueized_anything,
 		 bool with_avail = false)
 {
-  vn_reference_op_t vro;
-  unsigned int i;
-
   *valueized_anything = false;
 
-  FOR_EACH_VEC_ELT (*orig, i, vro)
+  for (unsigned i = 0; i < orig->length (); ++i)
     {
+re_valueize:
+      vn_reference_op_t vro = &(*orig)[i];
       if (vro->opcode == SSA_NAME
 	  || (vro->op0 && TREE_CODE (vro->op0) == SSA_NAME))
 	{
@@ -1694,7 +1693,11 @@ valueize_refs_1 (vec<vn_reference_op_s> *orig, bool *valueized_anything,
 	       && (*orig)[i - 1].opcode == MEM_REF)
 	{
 	  if (vn_reference_maybe_forwprop_address (orig, &i))
-	    *valueized_anything = true;
+	    {
+	      *valueized_anything = true;
+	      /* Re-valueize the current operand.  */
+	      goto re_valueize;
+	    }
 	}
       /* If it transforms a non-constant ARRAY_REF into a constant
 	 one, adjust the constant offset.  */
@@ -2440,7 +2443,7 @@ vn_nary_build_or_lookup_1 (gimple_match_op *res_op, bool insert,
 	  vno1->predicated_values = 0;
 	  vno1->u.result = result;
 	  init_vn_nary_op_from_stmt (vno1, as_a <gassign *> (new_stmt));
-	  vn_nary_op_insert_into (vno1, valid_info->nary, true);
+	  vn_nary_op_insert_into (vno1, valid_info->nary);
 	  /* Also do not link it into the undo chain.  */
 	  last_inserted_nary = vno1->next;
 	  vno1->next = (vn_nary_op_t)(void *)-1;
@@ -3855,10 +3858,6 @@ vn_nary_op_compute_hash (const vn_nary_op_t vno1)
   inchash::hash hstate;
   unsigned i;
 
-  for (i = 0; i < vno1->length; ++i)
-    if (TREE_CODE (vno1->op[i]) == SSA_NAME)
-      vno1->op[i] = SSA_VAL (vno1->op[i]);
-
   if (((vno1->length == 2
 	&& commutative_tree_code (vno1->opcode))
        || (vno1->length == 3
@@ -4000,6 +3999,10 @@ vn_nary_op_lookup_1 (vn_nary_op_t vno, vn_nary_op_t *vnresult)
   if (vnresult)
     *vnresult = NULL;
 
+  for (unsigned i = 0; i < vno->length; ++i)
+    if (TREE_CODE (vno->op[i]) == SSA_NAME)
+      vno->op[i] = SSA_VAL (vno->op[i]);
+
   vno->hashcode = vn_nary_op_compute_hash (vno);
   slot = valid_info->nary->find_slot_with_hash (vno, vno->hashcode, NO_INSERT);
   if (!slot)
@@ -4064,23 +4067,22 @@ alloc_vn_nary_op (unsigned int length, tree result, unsigned int value_id)
   return vno1;
 }
 
-/* Insert VNO into TABLE.  If COMPUTE_HASH is true, then compute
-   VNO->HASHCODE first.  */
+/* Insert VNO into TABLE.  */
 
 static vn_nary_op_t
-vn_nary_op_insert_into (vn_nary_op_t vno, vn_nary_op_table_type *table,
-			bool compute_hash)
+vn_nary_op_insert_into (vn_nary_op_t vno, vn_nary_op_table_type *table)
 {
   vn_nary_op_s **slot;
 
-  if (compute_hash)
-    {
-      vno->hashcode = vn_nary_op_compute_hash (vno);
-      gcc_assert (! vno->predicated_values
-		  || (! vno->u.values->next
-		      && vno->u.values->n == 1));
-    }
+  gcc_assert (! vno->predicated_values
+	      || (! vno->u.values->next
+		  && vno->u.values->n == 1));
+
+  for (unsigned i = 0; i < vno->length; ++i)
+    if (TREE_CODE (vno->op[i]) == SSA_NAME)
+      vno->op[i] = SSA_VAL (vno->op[i]);
 
+  vno->hashcode = vn_nary_op_compute_hash (vno);
   slot = table->find_slot_with_hash (vno, vno->hashcode, INSERT);
   vno->unwind_to = *slot;
   if (*slot)
@@ -4211,7 +4213,7 @@ vn_nary_op_insert_pieces (unsigned int length, enum tree_code code,
 {
   vn_nary_op_t vno1 = alloc_vn_nary_op (length, result, value_id);
   init_vn_nary_op_from_pieces (vno1, length, code, type, ops);
-  return vn_nary_op_insert_into (vno1, valid_info->nary, true);
+  return vn_nary_op_insert_into (vno1, valid_info->nary);
 }
 
 static vn_nary_op_t
@@ -4257,7 +4259,7 @@ vn_nary_op_insert_pieces_predicated (unsigned int length, enum tree_code code,
   vno1->u.values->result = result;
   vno1->u.values->n = 1;
   vno1->u.values->valid_dominated_by_p[0] = pred_e->dest->index;
-  return vn_nary_op_insert_into (vno1, valid_info->nary, true);
+  return vn_nary_op_insert_into (vno1, valid_info->nary);
 }
 
 static bool
@@ -4290,7 +4292,7 @@ vn_nary_op_insert_stmt (gimple *stmt, tree result)
     = alloc_vn_nary_op (vn_nary_length_from_stmt (stmt),
 			result, VN_INFO (result)->value_id);
   init_vn_nary_op_from_stmt (vno1, as_a <gassign *> (stmt));
-  return vn_nary_op_insert_into (vno1, valid_info->nary, true);
+  return vn_nary_op_insert_into (vno1, valid_info->nary);
 }
 
 /* Compute a hashcode for PHI operation VP1 and return it.  */
@@ -4441,11 +4443,15 @@ vn_phi_eq (const_vn_phi_t const vp1, const_vn_phi_t const vp2)
 	    if (inverted_p)
 	      std::swap (te2, fe2);
 
-	    /* ???  Handle VN_TOP specially.  */
+	    /* Since we do not know which edge will be executed we have
+	       to be careful when matching VN_TOP.  Be conservative and
+	       only match VN_TOP == VN_TOP for now, we could allow
+	       VN_TOP on the not prevailing PHI though.  See for example
+	       PR102920.  */
 	    if (! expressions_equal_p (vp1->phiargs[te1->dest_idx],
-				       vp2->phiargs[te2->dest_idx])
+				       vp2->phiargs[te2->dest_idx], false)
 		|| ! expressions_equal_p (vp1->phiargs[fe1->dest_idx],
-					  vp2->phiargs[fe2->dest_idx]))
+					  vp2->phiargs[fe2->dest_idx], false))
 	      return false;
 
 	    return true;
@@ -4470,7 +4476,7 @@ vn_phi_eq (const_vn_phi_t const vp1, const_vn_phi_t const vp2)
       tree phi2op = vp2->phiargs[i];
       if (phi1op == phi2op)
 	continue;
-      if (!expressions_equal_p (phi1op, phi2op))
+      if (!expressions_equal_p (phi1op, phi2op, false))
 	return false;
     }
 
@@ -4499,7 +4505,12 @@ vn_phi_lookup (gimple *phi, bool backedges_varying_p)
       tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
       if (TREE_CODE (def) == SSA_NAME
 	  && (!backedges_varying_p || !(e->flags & EDGE_DFS_BACK)))
-	def = SSA_VAL (def);
+	{
+	  if (ssa_undefined_value_p (def, false))
+	    def = VN_TOP;
+	  else
+	    def = SSA_VAL (def);
+	}
       vp1->phiargs[e->dest_idx] = def;
     }
   vp1->type = TREE_TYPE (gimple_phi_result (phi));
@@ -4543,7 +4554,12 @@ vn_phi_insert (gimple *phi, tree result, bool backedges_varying_p)
       tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
       if (TREE_CODE (def) == SSA_NAME
 	  && (!backedges_varying_p || !(e->flags & EDGE_DFS_BACK)))
-	def = SSA_VAL (def);
+	{
+	  if (ssa_undefined_value_p (def, false))
+	    def = VN_TOP;
+	  else
+	    def = SSA_VAL (def);
+	}
       vp1->phiargs[e->dest_idx] = def;
     }
   vp1->value_id = VN_INFO (result)->value_id;
@@ -5806,17 +5822,20 @@ get_next_constant_value_id (void)
 }
 
 
-/* Compare two expressions E1 and E2 and return true if they are equal.  */
+/* Compare two expressions E1 and E2 and return true if they are equal.
+   If match_vn_top_optimistically is true then VN_TOP is equal to anything,
+   otherwise VN_TOP only matches VN_TOP.  */
 
 bool
-expressions_equal_p (tree e1, tree e2)
+expressions_equal_p (tree e1, tree e2, bool match_vn_top_optimistically)
 {
   /* The obvious case.  */
   if (e1 == e2)
     return true;
 
   /* If either one is VN_TOP consider them equal.  */
-  if (e1 == VN_TOP || e2 == VN_TOP)
+  if (match_vn_top_optimistically
+      && (e1 == VN_TOP || e2 == VN_TOP))
     return true;
 
   /* SSA_NAME compare pointer equal.  */
diff --git a/gcc/tree-ssa-sccvn.h b/gcc/tree-ssa-sccvn.h
index 8a1b649..7d53ab5 100644
--- a/gcc/tree-ssa-sccvn.h
+++ b/gcc/tree-ssa-sccvn.h
@@ -22,7 +22,7 @@
 #define TREE_SSA_SCCVN_H
 
 /* In tree-ssa-sccvn.c  */
-bool expressions_equal_p (tree, tree);
+bool expressions_equal_p (tree, tree, bool = true);
 
 
 /* TOP of the VN lattice.  */
diff --git a/gcc/tree-ssa-strlen.c b/gcc/tree-ssa-strlen.c
index 8c39869..c0ec7d2 100644
--- a/gcc/tree-ssa-strlen.c
+++ b/gcc/tree-ssa-strlen.c
@@ -59,7 +59,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-ssa-loop.h"
 #include "tree-scalar-evolution.h"
 #include "vr-values.h"
-#include "gimple-ssa-evrp-analyze.h"
+#include "gimple-range.h"
 #include "tree-ssa.h"
 
 /* A vector indexed by SSA_NAME_VERSION.  0 means unknown, positive value
@@ -234,8 +234,7 @@ class strlen_pass : public dom_walker
 public:
   strlen_pass (cdi_direction direction)
     : dom_walker (direction),
-      evrp (false),
-      ptr_qry (&evrp, &var_cache),
+      ptr_qry (&m_ranger, &var_cache),
       var_cache (),
       m_cleanup_cfg (false)
   {
@@ -277,15 +276,18 @@ public:
 			   unsigned HOST_WIDE_INT len[2],
 			   unsigned HOST_WIDE_INT *psize);
   bool count_nonzero_bytes (tree expr_or_type,
+			    gimple *stmt,
 			    unsigned lenrange[3], bool *nulterm,
 			    bool *allnul, bool *allnonnul);
   bool count_nonzero_bytes (tree exp,
+			    gimple *stmt,
 			    unsigned HOST_WIDE_INT offset,
 			    unsigned HOST_WIDE_INT nbytes,
 			    unsigned lenrange[3], bool *nulterm,
 			    bool *allnul, bool *allnonnul,
 			    ssa_name_limit_t &snlim);
   bool count_nonzero_bytes_addr (tree exp,
+				 gimple *stmt,
 				 unsigned HOST_WIDE_INT offset,
 				 unsigned HOST_WIDE_INT nbytes,
 				 unsigned lenrange[3], bool *nulterm,
@@ -295,9 +297,7 @@ public:
 			unsigned HOST_WIDE_INT lenrng[2],
 			unsigned HOST_WIDE_INT *size, bool *nulterm);
 
-  /* EVRP analyzer used for printf argument range processing, and to
-     track strlen results across integer variable assignments.  */
-  evrp_range_analyzer evrp;
+  gimple_ranger m_ranger;
 
   /* A pointer_query object and its cache to store information about
      pointers and their targets in.  */
@@ -335,7 +335,8 @@ compare_nonzero_chars (strinfo *si, unsigned HOST_WIDE_INT off)
    Uses RVALS to determine length range.  */
 
 static int
-compare_nonzero_chars (strinfo *si, unsigned HOST_WIDE_INT off,
+compare_nonzero_chars (strinfo *si, gimple *stmt,
+		       unsigned HOST_WIDE_INT off,
 		       range_query *rvals)
 {
   if (!si->nonzero_chars)
@@ -348,7 +349,7 @@ compare_nonzero_chars (strinfo *si, unsigned HOST_WIDE_INT off,
     return -1;
 
   value_range vr;
-  if (!rvals->range_of_expr (vr, si->nonzero_chars, si->stmt))
+  if (!rvals->range_of_expr (vr, si->nonzero_chars, stmt))
     return -1;
   value_range_kind rng = vr.kind ();
   if (rng != VR_RANGE)
@@ -403,7 +404,8 @@ get_next_strinfo (strinfo *si)
    information.  */
 
 static int
-get_addr_stridx (tree exp, tree ptr, unsigned HOST_WIDE_INT *offset_out,
+get_addr_stridx (tree exp, gimple *stmt,
+		 tree ptr, unsigned HOST_WIDE_INT *offset_out,
 		 range_query *rvals = NULL)
 {
   HOST_WIDE_INT off;
@@ -442,7 +444,7 @@ get_addr_stridx (tree exp, tree ptr, unsigned HOST_WIDE_INT *offset_out,
       unsigned HOST_WIDE_INT rel_off
 	= (unsigned HOST_WIDE_INT) off - last->offset;
       strinfo *si = get_strinfo (last->idx);
-      if (si && compare_nonzero_chars (si, rel_off, rvals) >= 0)
+      if (si && compare_nonzero_chars (si, stmt, rel_off, rvals) >= 0)
 	{
 	  if (offset_out)
 	    {
@@ -464,7 +466,8 @@ get_addr_stridx (tree exp, tree ptr, unsigned HOST_WIDE_INT *offset_out,
    When nonnull, uses RVALS to determine range information.  */
 
 static int
-get_stridx (tree exp, wide_int offrng[2] = NULL, range_query *rvals = NULL)
+get_stridx (tree exp, gimple *stmt,
+	    wide_int offrng[2] = NULL, range_query *rvals = NULL)
 {
   if (offrng)
     offrng[0] = offrng[1] = wi::zero (TYPE_PRECISION (ptrdiff_type_node));
@@ -601,7 +604,7 @@ get_stridx (tree exp, wide_int offrng[2] = NULL, range_query *rvals = NULL)
 
   if (TREE_CODE (exp) == ADDR_EXPR)
     {
-      int idx = get_addr_stridx (TREE_OPERAND (exp, 0), exp, NULL);
+      int idx = get_addr_stridx (TREE_OPERAND (exp, 0), stmt, exp, NULL);
       if (idx != 0)
 	return idx;
     }
@@ -1095,7 +1098,7 @@ get_range_strlen_dynamic (tree src, gimple *stmt,
 			  c_strlen_data *pdata, bitmap *visited,
 			  range_query *rvals, unsigned *pssa_def_max)
 {
-  int idx = get_stridx (src);
+  int idx = get_stridx (src, stmt);
   if (!idx)
     {
       if (TREE_CODE (src) == SSA_NAME)
@@ -1833,7 +1836,7 @@ strlen_pass::adjust_last_stmt (strinfo *si, gimple *stmt, bool is_strcat)
       tree dst = gimple_call_arg (last.stmt, 0);
 
       access_ref aref;
-      tree size = compute_objsize (dst, 1, &aref, &ptr_qry);
+      tree size = compute_objsize (dst, stmt, 1, &aref, &ptr_qry);
       if (size && tree_int_cst_lt (size, len))
 	return;
     }
@@ -2035,7 +2038,7 @@ strlen_pass::maybe_warn_overflow (gimple *stmt, bool call_lhs, tree len,
   access_ref aref;
   /* The size of the destination region (which is smaller than
      the destination object for stores at a non-zero offset).  */
-  tree destsize = compute_objsize (dest, ostype, &aref, &ptr_qry);
+  tree destsize = compute_objsize (dest, stmt, ostype, &aref, &ptr_qry);
 
   if (!destsize)
     {
@@ -2200,7 +2203,7 @@ strlen_pass::handle_builtin_strlen ()
   tree src = gimple_call_arg (stmt, 0);
   tree bound = (DECL_FUNCTION_CODE (callee) == BUILT_IN_STRNLEN
 		? gimple_call_arg (stmt, 1) : NULL_TREE);
-  int idx = get_stridx (src);
+  int idx = get_stridx (src, stmt);
   if (idx || (bound && integer_zerop (bound)))
     {
       strinfo *si = NULL;
@@ -2380,7 +2383,7 @@ strlen_pass::handle_builtin_strchr ()
   if (!check_nul_terminated_array (NULL_TREE, src))
     return;
 
-  int idx = get_stridx (src);
+  int idx = get_stridx (src, stmt);
   if (idx)
     {
       strinfo *si = NULL;
@@ -2486,12 +2489,12 @@ strlen_pass::handle_builtin_strcpy (built_in_function bcode)
   src = gimple_call_arg (stmt, 1);
   dst = gimple_call_arg (stmt, 0);
   lhs = gimple_call_lhs (stmt);
-  idx = get_stridx (src);
+  idx = get_stridx (src, stmt);
   si = NULL;
   if (idx > 0)
     si = get_strinfo (idx);
 
-  didx = get_stridx (dst);
+  didx = get_stridx (dst, stmt);
   olddsi = NULL;
   oldlen = NULL_TREE;
   if (didx > 0)
@@ -2893,7 +2896,7 @@ maybe_diag_stxncpy_trunc (gimple_stmt_iterator gsi, tree src, tree cnt,
      when ssa_ver_to_stridx is empty.  That implies the caller isn't
      running under the control of this pass and ssa_ver_to_stridx hasn't
      been created yet.  */
-  int sidx = ssa_ver_to_stridx.length () ? get_stridx (src) : 0;
+  int sidx = ssa_ver_to_stridx.length () ? get_stridx (src, stmt) : 0;
   if (sidx < 0 && wi::gtu_p (cntrange[0], ~sidx))
     return false;
 
@@ -3115,7 +3118,7 @@ maybe_diag_stxncpy_trunc (gimple_stmt_iterator gsi, tree src, tree cnt,
     }
 
   access_ref aref;
-  if (tree dstsize = compute_objsize (dst, 1, &aref, ptr_qry))
+  if (tree dstsize = compute_objsize (dst, stmt, 1, &aref, ptr_qry))
     {
       /* The source length is unknown.  Try to determine the destination
 	 size and see if it matches the specified bound.  If not, bail.
@@ -3130,7 +3133,7 @@ maybe_diag_stxncpy_trunc (gimple_stmt_iterator gsi, tree src, tree cnt,
       /* Avoid warning for strncpy(a, b, N) calls where the following
 	 equalities hold:
 	   N == sizeof a && N == sizeof b */
-      if (tree srcsize = compute_objsize (src, 1, &aref, ptr_qry))
+      if (tree srcsize = compute_objsize (src, stmt, 1, &aref, ptr_qry))
 	if (wi::to_wide (srcsize) == cntrange[1])
 	  return false;
 
@@ -3167,7 +3170,7 @@ strlen_pass::handle_builtin_stxncpy_strncat (bool append_p)
      a lower bound).  */
   tree dstlenp1 = NULL_TREE, srclenp1 = NULL_TREE;;
 
-  int didx = get_stridx (dst);
+  int didx = get_stridx (dst, stmt);
   if (strinfo *sidst = didx > 0 ? get_strinfo (didx) : NULL)
     {
       /* Compute the size of the destination string including the nul
@@ -3193,7 +3196,7 @@ strlen_pass::handle_builtin_stxncpy_strncat (bool append_p)
       dst = sidst->ptr;
     }
 
-  int sidx = get_stridx (src);
+  int sidx = get_stridx (src, stmt);
   strinfo *sisrc = sidx > 0 ? get_strinfo (sidx) : NULL;
   if (sisrc)
     {
@@ -3302,7 +3305,7 @@ strlen_pass::handle_builtin_memcpy (built_in_function bcode)
   tree src = gimple_call_arg (stmt, 1);
   tree dst = gimple_call_arg (stmt, 0);
 
-  int didx = get_stridx (dst);
+  int didx = get_stridx (dst, stmt);
   strinfo *olddsi = NULL;
   if (didx > 0)
     olddsi = get_strinfo (didx);
@@ -3316,7 +3319,7 @@ strlen_pass::handle_builtin_memcpy (built_in_function bcode)
       adjust_last_stmt (olddsi, stmt, false);
     }
 
-  int idx = get_stridx (src);
+  int idx = get_stridx (src, stmt);
   if (idx == 0)
     return;
 
@@ -3491,7 +3494,7 @@ strlen_pass::handle_builtin_strcat (built_in_function bcode)
 
   tree lhs = gimple_call_lhs (stmt);
 
-  didx = get_stridx (dst);
+  didx = get_stridx (dst, stmt);
   if (didx < 0)
     return;
 
@@ -3501,7 +3504,7 @@ strlen_pass::handle_builtin_strcat (built_in_function bcode)
 
   srclen = NULL_TREE;
   si = NULL;
-  idx = get_stridx (src);
+  idx = get_stridx (src, stmt);
   if (idx < 0)
     srclen = build_int_cst (size_type_node, ~idx);
   else if (idx > 0)
@@ -3723,7 +3726,7 @@ strlen_pass::handle_alloc_call (built_in_function bcode)
   if (lhs == NULL_TREE)
     return;
 
-  gcc_assert (get_stridx (lhs) == 0);
+  gcc_assert (get_stridx (lhs, stmt) == 0);
   int idx = new_stridx (lhs);
   tree length = NULL_TREE;
   if (bcode == BUILT_IN_CALLOC)
@@ -3759,7 +3762,7 @@ strlen_pass::handle_builtin_memset (bool *zero_write)
   tree ptr = gimple_call_arg (memset_stmt, 0);
   /* Set to the non-constant offset added to PTR.  */
   wide_int offrng[2];
-  int idx1 = get_stridx (ptr, offrng, ptr_qry.rvals);
+  int idx1 = get_stridx (ptr, memset_stmt, offrng, ptr_qry.rvals);
   if (idx1 <= 0)
     return false;
   strinfo *si1 = get_strinfo (idx1);
@@ -4250,8 +4253,8 @@ strlen_pass::handle_builtin_string_cmp ()
 
   tree arg1 = gimple_call_arg (stmt, 0);
   tree arg2 = gimple_call_arg (stmt, 1);
-  int idx1 = get_stridx (arg1);
-  int idx2 = get_stridx (arg2);
+  int idx1 = get_stridx (arg1, stmt);
+  int idx2 = get_stridx (arg2, stmt);
 
   /* For strncmp set to the value of the third argument if known.  */
   HOST_WIDE_INT bound = -1;
@@ -4389,7 +4392,7 @@ strlen_pass::handle_pointer_plus ()
 {
   gimple *stmt = gsi_stmt (m_gsi);
   tree lhs = gimple_assign_lhs (stmt), off;
-  int idx = get_stridx (gimple_assign_rhs1 (stmt));
+  int idx = get_stridx (gimple_assign_rhs1 (stmt), stmt);
   strinfo *si, *zsi;
 
   if (idx == 0)
@@ -4482,7 +4485,7 @@ nonzero_bytes_for_type (tree type, unsigned lenrange[3],
    Returns true on success and false otherwise.  */
 
 bool
-strlen_pass::count_nonzero_bytes (tree exp,
+strlen_pass::count_nonzero_bytes (tree exp, gimple *stmt,
 				  unsigned HOST_WIDE_INT offset,
 				  unsigned HOST_WIDE_INT nbytes,
 				  unsigned lenrange[3], bool *nulterm,
@@ -4502,7 +4505,8 @@ strlen_pass::count_nonzero_bytes (tree exp,
 	     exact value is not known) recurse once to set the range
 	     for an arbitrary constant.  */
 	  exp = build_int_cst (type, 1);
-	  return count_nonzero_bytes (exp, offset, 1, lenrange,
+	  return count_nonzero_bytes (exp, stmt,
+				      offset, 1, lenrange,
 				      nulterm, allnul, allnonnul, snlim);
 	}
 
@@ -4529,7 +4533,8 @@ strlen_pass::count_nonzero_bytes (tree exp,
 	  for (unsigned i = 0; i != n; i++)
 	    {
 	      tree def = gimple_phi_arg_def (stmt, i);
-	      if (!count_nonzero_bytes (def, offset, nbytes, lenrange, nulterm,
+	      if (!count_nonzero_bytes (def, stmt,
+					offset, nbytes, lenrange, nulterm,
 					allnul, allnonnul, snlim))
 		return false;
 	    }
@@ -4586,7 +4591,8 @@ strlen_pass::count_nonzero_bytes (tree exp,
 	return false;
 
       /* Handle MEM_REF = SSA_NAME types of assignments.  */
-      return count_nonzero_bytes_addr (arg, offset, nbytes, lenrange, nulterm,
+      return count_nonzero_bytes_addr (arg, stmt,
+				       offset, nbytes, lenrange, nulterm,
 				       allnul, allnonnul, snlim);
     }
 
@@ -4698,14 +4704,14 @@ strlen_pass::count_nonzero_bytes (tree exp,
    bytes that are pointed to by EXP, which should be a pointer.  */
 
 bool
-strlen_pass::count_nonzero_bytes_addr (tree exp,
+strlen_pass::count_nonzero_bytes_addr (tree exp, gimple *stmt,
 				       unsigned HOST_WIDE_INT offset,
 				       unsigned HOST_WIDE_INT nbytes,
 				       unsigned lenrange[3], bool *nulterm,
 				       bool *allnul, bool *allnonnul,
 				       ssa_name_limit_t &snlim)
 {
-  int idx = get_stridx (exp);
+  int idx = get_stridx (exp, stmt);
   if (idx > 0)
     {
       strinfo *si = get_strinfo (idx);
@@ -4721,7 +4727,7 @@ strlen_pass::count_nonzero_bytes_addr (tree exp,
 	       && TREE_CODE (si->nonzero_chars) == SSA_NAME)
 	{
 	  value_range vr;
-	  ptr_qry.rvals->range_of_expr (vr, si->nonzero_chars, si->stmt);
+	  ptr_qry.rvals->range_of_expr (vr, si->nonzero_chars, stmt);
 	  if (vr.kind () != VR_RANGE)
 	    return false;
 
@@ -4767,7 +4773,8 @@ strlen_pass::count_nonzero_bytes_addr (tree exp,
     }
 
   if (TREE_CODE (exp) == ADDR_EXPR)
-    return count_nonzero_bytes (TREE_OPERAND (exp, 0), offset, nbytes,
+    return count_nonzero_bytes (TREE_OPERAND (exp, 0), stmt,
+				offset, nbytes,
 				lenrange, nulterm, allnul, allnonnul, snlim);
 
   if (TREE_CODE (exp) == SSA_NAME)
@@ -4786,7 +4793,8 @@ strlen_pass::count_nonzero_bytes_addr (tree exp,
 	  for (unsigned i = 0; i != n; i++)
 	    {
 	      tree def = gimple_phi_arg_def (stmt, i);
-	      if (!count_nonzero_bytes_addr (def, offset, nbytes, lenrange,
+	      if (!count_nonzero_bytes_addr (def, stmt,
+					     offset, nbytes, lenrange,
 					     nulterm, allnul, allnonnul,
 					     snlim))
 		return false;
@@ -4814,7 +4822,7 @@ strlen_pass::count_nonzero_bytes_addr (tree exp,
    (the results of strlen).  */
 
 bool
-strlen_pass::count_nonzero_bytes (tree expr_or_type,
+strlen_pass::count_nonzero_bytes (tree expr_or_type, gimple *stmt,
 				  unsigned lenrange[3], bool *nulterm,
 				  bool *allnul, bool *allnonnul)
 {
@@ -4833,7 +4841,8 @@ strlen_pass::count_nonzero_bytes (tree expr_or_type,
 
   ssa_name_limit_t snlim;
   tree expr = expr_or_type;
-  return count_nonzero_bytes (expr, 0, 0, lenrange, nulterm, allnul, allnonnul,
+  return count_nonzero_bytes (expr, stmt,
+			      0, 0, lenrange, nulterm, allnul, allnonnul,
 			      snlim);
 }
 
@@ -4885,18 +4894,19 @@ strlen_pass::handle_store (bool *zero_write)
 	     least OFFSET nonzero characters.  This is trivially true if
 	     OFFSET is zero.  */
 	  offset = tree_to_uhwi (mem_offset);
-	  idx = get_stridx (TREE_OPERAND (lhs, 0));
+	  idx = get_stridx (TREE_OPERAND (lhs, 0), stmt);
 	  if (idx > 0)
 	    si = get_strinfo (idx);
 	  if (offset == 0)
 	    ssaname = TREE_OPERAND (lhs, 0);
-	  else if (si == NULL || compare_nonzero_chars (si, offset, rvals) < 0)
+	  else if (si == NULL
+		   || compare_nonzero_chars (si, stmt, offset, rvals) < 0)
 	    {
 	      *zero_write = rhs ? initializer_zerop (rhs) : false;
 
 	      bool dummy;
 	      unsigned lenrange[] = { UINT_MAX, 0, 0 };
-	      if (count_nonzero_bytes (rhs ? rhs : storetype, lenrange,
+	      if (count_nonzero_bytes (rhs ? rhs : storetype, stmt, lenrange,
 				       &dummy, &dummy, &dummy))
 		maybe_warn_overflow (stmt, true, lenrange[2]);
 
@@ -4906,7 +4916,7 @@ strlen_pass::handle_store (bool *zero_write)
     }
   else
     {
-      idx = get_addr_stridx (lhs, NULL_TREE, &offset, rvals);
+      idx = get_addr_stridx (lhs, stmt, NULL_TREE, &offset, rvals);
       if (idx > 0)
 	si = get_strinfo (idx);
     }
@@ -4929,7 +4939,8 @@ strlen_pass::handle_store (bool *zero_write)
   bool full_string_p;
 
   const bool ranges_valid
-    = count_nonzero_bytes (rhs ? rhs : storetype, lenrange, &full_string_p,
+    = count_nonzero_bytes (rhs ? rhs : storetype, stmt,
+			   lenrange, &full_string_p,
 			   &storing_all_zeros_p, &storing_all_nonzero_p);
 
   if (ranges_valid)
@@ -4961,15 +4972,18 @@ strlen_pass::handle_store (bool *zero_write)
 	{
 	  /* The offset of the last stored byte.  */
 	  unsigned HOST_WIDE_INT endoff = offset + lenrange[2] - 1;
-	  store_before_nul[0] = compare_nonzero_chars (si, offset, rvals);
+	  store_before_nul[0]
+	    = compare_nonzero_chars (si, stmt, offset, rvals);
 	  if (endoff == offset)
 	    store_before_nul[1] = store_before_nul[0];
 	  else
-	    store_before_nul[1] = compare_nonzero_chars (si, endoff, rvals);
+	    store_before_nul[1]
+	      = compare_nonzero_chars (si, stmt, endoff, rvals);
 	}
       else
 	{
-	  store_before_nul[0] = compare_nonzero_chars (si, offset, rvals);
+	  store_before_nul[0]
+	    = compare_nonzero_chars (si, stmt, offset, rvals);
 	  store_before_nul[1] = store_before_nul[0];
 	  gcc_assert (offset == 0 || store_before_nul[0] >= 0);
 	}
@@ -5194,7 +5208,7 @@ fold_strstr_to_strncmp (tree rhs1, tree rhs2, gimple *stmt)
 	{
 	  tree arg1 = gimple_call_arg (call_stmt, 1);
 	  tree arg1_len = NULL_TREE;
-	  int idx = get_stridx (arg1);
+	  int idx = get_stridx (arg1, call_stmt);
 
 	  if (idx)
 	    {
@@ -5406,7 +5420,7 @@ strlen_pass::handle_integral_assign (bool *cleanup_eh)
       tree rhs1 = gimple_assign_rhs1 (stmt);
       if (code == MEM_REF)
 	{
-	  idx = get_stridx (TREE_OPERAND (rhs1, 0));
+	  idx = get_stridx (TREE_OPERAND (rhs1, 0), stmt);
 	  if (idx > 0)
 	    {
 	      strinfo *si = get_strinfo (idx);
@@ -5423,7 +5437,7 @@ strlen_pass::handle_integral_assign (bool *cleanup_eh)
 	    }
 	}
       if (idx <= 0)
-	idx = get_addr_stridx (rhs1, NULL_TREE, &coff);
+	idx = get_addr_stridx (rhs1, stmt, NULL_TREE, &coff);
       if (idx > 0)
 	{
 	  strinfo *si = get_strinfo (idx);
@@ -5485,7 +5499,8 @@ strlen_pass::handle_integral_assign (bool *cleanup_eh)
 	  unsigned lenrange[] = { UINT_MAX, 0, 0 };
 	  tree rhs = gimple_assign_rhs1 (stmt);
 	  const bool ranges_valid
-	    = count_nonzero_bytes (rhs, lenrange, &full_string_p,
+	    = count_nonzero_bytes (rhs, stmt,
+				   lenrange, &full_string_p,
 				   &storing_all_zeros_p,
 				   &storing_all_nonzero_p);
 	  if (ranges_valid)
@@ -5582,7 +5597,7 @@ strlen_pass::check_and_optimize_stmt (bool *cleanup_eh)
 	      || (gimple_assign_cast_p (stmt)
 		  && POINTER_TYPE_P (TREE_TYPE (gimple_assign_rhs1 (stmt)))))
 	    {
-	      int idx = get_stridx (gimple_assign_rhs1 (stmt));
+	      int idx = get_stridx (gimple_assign_rhs1 (stmt), stmt);
 	      ssa_ver_to_stridx[SSA_NAME_VERSION (lhs)] = idx;
 	    }
 	  else if (gimple_assign_rhs_code (stmt) == POINTER_PLUS_EXPR)
@@ -5672,8 +5687,6 @@ strlen_pass::~strlen_pass ()
 edge
 strlen_pass::before_dom_children (basic_block bb)
 {
-  evrp.enter (bb);
-
   basic_block dombb = get_immediate_dominator (CDI_DOMINATORS, bb);
 
   if (dombb == NULL)
@@ -5730,12 +5743,12 @@ strlen_pass::before_dom_children (basic_block bb)
       tree result = gimple_phi_result (phi);
       if (!virtual_operand_p (result) && POINTER_TYPE_P (TREE_TYPE (result)))
 	{
-	  int idx = get_stridx (gimple_phi_arg_def (phi, 0));
+	  int idx = get_stridx (gimple_phi_arg_def (phi, 0), phi);
 	  if (idx != 0)
 	    {
 	      unsigned int i, n = gimple_phi_num_args (phi);
 	      for (i = 1; i < n; i++)
-		if (idx != get_stridx (gimple_phi_arg_def (phi, i)))
+		if (idx != get_stridx (gimple_phi_arg_def (phi, i), phi))
 		  break;
 	      if (i == n)
 		ssa_ver_to_stridx[SSA_NAME_VERSION (result)] = idx;
@@ -5748,12 +5761,6 @@ strlen_pass::before_dom_children (basic_block bb)
   /* Attempt to optimize individual statements.  */
   for (m_gsi = gsi_start_bb (bb); !gsi_end_p (m_gsi); )
     {
-      gimple *stmt = gsi_stmt (m_gsi);
-
-      /* First record ranges generated by this statement so they
-	 can be used by printf argument processing.  */
-      evrp.record_ranges_from_stmt (stmt, false);
-
       /* Reset search depth preformance counter.  */
       ptr_qry.depth = 0;
 
@@ -5776,8 +5783,6 @@ strlen_pass::before_dom_children (basic_block bb)
 void
 strlen_pass::after_dom_children (basic_block bb)
 {
-  evrp.leave (bb);
-
   if (bb->aux)
     {
       stridx_to_strinfo = ((vec<strinfo *, va_heap, vl_embed> *) bb->aux);
diff --git a/gcc/tree-ssa-structalias.c b/gcc/tree-ssa-structalias.c
index 2e6513b..06eb22c 100644
--- a/gcc/tree-ssa-structalias.c
+++ b/gcc/tree-ssa-structalias.c
@@ -4060,40 +4060,117 @@ static void
 handle_call_arg (gcall *stmt, tree arg, vec<ce_s> *results, int flags,
 		 int callescape_id, bool writes_global_memory)
 {
+  int relevant_indirect_flags = EAF_NO_INDIRECT_CLOBBER | EAF_NO_INDIRECT_READ
+				| EAF_NO_INDIRECT_ESCAPE;
+  int relevant_flags = relevant_indirect_flags
+		       | EAF_NO_DIRECT_CLOBBER
+		       | EAF_NO_DIRECT_READ
+		       | EAF_NO_DIRECT_ESCAPE;
+  if (gimple_call_lhs (stmt))
+    {
+      relevant_flags |= EAF_NOT_RETURNED_DIRECTLY | EAF_NOT_RETURNED_INDIRECTLY;
+      relevant_indirect_flags |= EAF_NOT_RETURNED_INDIRECTLY;
+
+      /* If value is never read from it can not be returned indirectly
+	 (except through the escape solution).
+	 For all flags we get these implications right except for
+	 not_returned because we miss return functions in ipa-prop.  */
+	 
+      if (flags & EAF_NO_DIRECT_READ)
+	flags |= EAF_NOT_RETURNED_INDIRECTLY;
+    }
+
   /* If the argument is not used we can ignore it.
      Similarly argument is invisile for us if it not clobbered, does not
      escape, is not read and can not be returned.  */
-  if ((flags & EAF_UNUSED)
-      || ((flags & (EAF_NOCLOBBER | EAF_NOESCAPE | EAF_NOREAD
-		    | EAF_NOT_RETURNED))
-	  == (EAF_NOCLOBBER | EAF_NOESCAPE | EAF_NOREAD
-	      | EAF_NOT_RETURNED)))
+  if ((flags & EAF_UNUSED) || ((flags & relevant_flags) == relevant_flags))
     return;
 
+  /* Produce varinfo for direct accesses to ARG.  */
   varinfo_t tem = new_var_info (NULL_TREE, "callarg", true);
   tem->is_reg_var = true;
   make_constraint_to (tem->id, arg);
   make_any_offset_constraints (tem);
 
-  if (!(flags & EAF_DIRECT))
-    make_transitive_closure_constraints (tem);
+  bool callarg_transitive = false;
+
+  /* As an compile time optimization if we make no difference between
+     direct and indirect accesses make arg transitively closed.
+     This avoids the need to build indir arg and do everything twice.  */
+  if (((flags & EAF_NO_INDIRECT_CLOBBER) != 0)
+      == ((flags & EAF_NO_DIRECT_CLOBBER) != 0)
+      && (((flags & EAF_NO_INDIRECT_READ) != 0)
+	  == ((flags & EAF_NO_DIRECT_READ) != 0))
+      && (((flags & EAF_NO_INDIRECT_ESCAPE) != 0)
+	  == ((flags & EAF_NO_DIRECT_ESCAPE) != 0))
+      && (((flags & EAF_NOT_RETURNED_INDIRECTLY) != 0)
+	  == ((flags & EAF_NOT_RETURNED_DIRECTLY) != 0)))
+    {
+      make_transitive_closure_constraints (tem);
+      callarg_transitive = true;
+      gcc_checking_assert (!(flags & EAF_NO_DIRECT_READ));
+    }
 
-  if (!(flags & EAF_NOT_RETURNED))
+  /* If necessary, produce varinfo for indirect accesses to ARG.  */
+  varinfo_t indir_tem = NULL;
+  if (!callarg_transitive
+      && (flags & relevant_indirect_flags) != relevant_indirect_flags)
     {
-      struct constraint_expr cexpr;
-      cexpr.var = tem->id;
-      cexpr.type = SCALAR;
-      cexpr.offset = 0;
-      results->safe_push (cexpr);
+      struct constraint_expr lhs, rhs;
+      indir_tem = new_var_info (NULL_TREE, "indircallarg", true);
+      indir_tem->is_reg_var = true;
+
+      /* indir_term = *tem.  */
+      lhs.type = SCALAR;
+      lhs.var = indir_tem->id;
+      lhs.offset = 0;
+
+      rhs.type = DEREF;
+      rhs.var = tem->id;
+      rhs.offset = UNKNOWN_OFFSET;
+      process_constraint (new_constraint (lhs, rhs));
+
+      make_any_offset_constraints (indir_tem);
+
+      /* If we do not read indirectly there is no need for transitive closure.
+	 We know there is only one level of indirection.  */
+      if (!(flags & EAF_NO_INDIRECT_READ))
+	make_transitive_closure_constraints (indir_tem);
+      gcc_checking_assert (!(flags & EAF_NO_DIRECT_READ));
     }
 
-  if (!(flags & EAF_NOREAD))
+  if (gimple_call_lhs (stmt))
+    {
+      if (!(flags & EAF_NOT_RETURNED_DIRECTLY))
+	{
+	  struct constraint_expr cexpr;
+	  cexpr.var = tem->id;
+	  cexpr.type = SCALAR;
+	  cexpr.offset = 0;
+	  results->safe_push (cexpr);
+	}
+      if (!callarg_transitive & !(flags & EAF_NOT_RETURNED_INDIRECTLY))
+	{
+	  struct constraint_expr cexpr;
+	  cexpr.var = indir_tem->id;
+	  cexpr.type = SCALAR;
+	  cexpr.offset = 0;
+	  results->safe_push (cexpr);
+	}
+    }
+
+  if (!(flags & EAF_NO_DIRECT_READ))
     {
       varinfo_t uses = get_call_use_vi (stmt);
       make_copy_constraint (uses, tem->id);
+      if (!callarg_transitive & !(flags & EAF_NO_INDIRECT_READ))
+	make_copy_constraint (uses, indir_tem->id);
     }
+  else
+    /* To read indirectly we need to read directly.  */
+    gcc_checking_assert (flags & EAF_NO_INDIRECT_READ);
 
-  if (!(flags & EAF_NOCLOBBER))
+  if (!(flags & EAF_NO_DIRECT_CLOBBER))
     {
       struct constraint_expr lhs, rhs;
 
@@ -4110,8 +4187,25 @@ handle_call_arg (gcall *stmt, tree arg, vec<ce_s> *results, int flags,
       /* callclobbered = arg.  */
       make_copy_constraint (get_call_clobber_vi (stmt), tem->id);
     }
+  if (!callarg_transitive & !(flags & EAF_NO_INDIRECT_CLOBBER))
+    {
+      struct constraint_expr lhs, rhs;
+
+      /* *indir_arg = callescape.  */
+      lhs.type = DEREF;
+      lhs.var = indir_tem->id;
+      lhs.offset = 0;
 
-  if (!(flags & (EAF_NOESCAPE | EAF_NODIRECTESCAPE)))
+      rhs.type = SCALAR;
+      rhs.var = callescape_id;
+      rhs.offset = 0;
+      process_constraint (new_constraint (lhs, rhs));
+
+      /* callclobbered = indir_arg.  */
+      make_copy_constraint (get_call_clobber_vi (stmt), indir_tem->id);
+    }
+
+  if (!(flags & (EAF_NO_DIRECT_ESCAPE | EAF_NO_INDIRECT_ESCAPE)))
     {
       struct constraint_expr lhs, rhs;
 
@@ -4128,18 +4222,18 @@ handle_call_arg (gcall *stmt, tree arg, vec<ce_s> *results, int flags,
       if (writes_global_memory)
 	make_escape_constraint (arg);
     }
-  else if (!(flags & EAF_NOESCAPE))
+  else if (!callarg_transitive & !(flags & EAF_NO_INDIRECT_ESCAPE))
     {
       struct constraint_expr lhs, rhs;
 
-      /* callescape = *(arg + UNKNOWN);  */
+      /* callescape = *(indir_arg + UNKNOWN);  */
       lhs.var = callescape_id;
       lhs.offset = 0;
       lhs.type = SCALAR;
 
-      rhs.var = tem->id;
-      rhs.offset = UNKNOWN_OFFSET;
-      rhs.type = DEREF;
+      rhs.var = indir_tem->id;
+      rhs.offset = 0;
+      rhs.type = SCALAR;
       process_constraint (new_constraint (lhs, rhs));
 
       if (writes_global_memory)
@@ -4168,10 +4262,11 @@ determine_global_memory_access (gcall *stmt,
       && (summary = get_modref_function_summary (node)))
     {
       if (writes_global_memory && *writes_global_memory)
-	*writes_global_memory = summary->global_memory_written_p ();
+	*writes_global_memory = summary->global_memory_written;
       if (reads_global_memory && *reads_global_memory)
-	*reads_global_memory = summary->global_memory_read_p ();
+	*reads_global_memory = summary->global_memory_read;
       if (reads_global_memory && uses_global_memory
+	  && !summary->calls_interposable
 	  && !*reads_global_memory && node->binds_to_current_def_p ())
 	*uses_global_memory = false;
     }
@@ -4246,7 +4341,8 @@ handle_rhs_call (gcall *stmt, vec<ce_s> *results,
   /* The static chain escapes as well.  */
   if (gimple_call_chain (stmt))
     handle_call_arg (stmt, gimple_call_chain (stmt), results,
-		     implicit_eaf_flags,
+		     implicit_eaf_flags
+		     | gimple_call_static_chain_flags (stmt),
 		     callescape->id, writes_global_memory);
 
   /* And if we applied NRV the address of the return slot escapes as well.  */
@@ -4254,17 +4350,30 @@ handle_rhs_call (gcall *stmt, vec<ce_s> *results,
       && gimple_call_lhs (stmt) != NULL_TREE
       && TREE_ADDRESSABLE (TREE_TYPE (gimple_call_lhs (stmt))))
     {
-      auto_vec<ce_s> tmpc;
-      struct constraint_expr *c;
-      unsigned i;
+      int flags = gimple_call_retslot_flags (stmt);
+      const int relevant_flags = EAF_NO_DIRECT_ESCAPE
+				 | EAF_NOT_RETURNED_DIRECTLY;
 
-      get_constraint_for_address_of (gimple_call_lhs (stmt), &tmpc);
+      if (!(flags & EAF_UNUSED) && (flags & relevant_flags) != relevant_flags)
+	{
+	  auto_vec<ce_s> tmpc;
 
-      make_constraints_to (callescape->id, tmpc);
-      if (writes_global_memory)
-	make_constraints_to (escaped_id, tmpc);
-      FOR_EACH_VEC_ELT (tmpc, i, c)
-	results->safe_push (*c);
+	  get_constraint_for_address_of (gimple_call_lhs (stmt), &tmpc);
+
+	  if (!(flags & EAF_NO_DIRECT_ESCAPE))
+	    {
+	      make_constraints_to (callescape->id, tmpc);
+	      if (writes_global_memory)
+		make_constraints_to (escaped_id, tmpc);
+	    }
+	  if (!(flags & EAF_NOT_RETURNED_DIRECTLY))
+	    {
+	      struct constraint_expr *c;
+	      unsigned i;
+	      FOR_EACH_VEC_ELT (tmpc, i, c)
+		results->safe_push (*c);
+	    }
+	}
     }
 }
 
@@ -4888,7 +4997,7 @@ find_func_aliases_for_call (struct function *fn, gcall *t)
 	 reachable from their arguments, but they are not an escape
 	 point for reachable memory of their arguments.  */
       else if (flags & (ECF_PURE|ECF_LOOPING_CONST_OR_PURE))
-	handle_rhs_call (t, &rhsc, implicit_pure_eaf_flags, true, false);
+	handle_rhs_call (t, &rhsc, implicit_pure_eaf_flags, false, true);
       /* If the call is to a replaceable operator delete and results
 	 from a delete expression as opposed to a direct call to
 	 such operator, then the effects for PTA (in particular
@@ -7550,8 +7659,8 @@ compute_points_to_sets (void)
 		     always escaped.  */
 		  if (uses_global_memory)
 		    {
-		      pt->nonlocal = uses_global_memory;
-		      pt->escaped = uses_global_memory;
+		      pt->nonlocal = 1;
+		      pt->escaped = 1;
 		    }
 		}
 	      else if (uses_global_memory)
@@ -7561,6 +7670,8 @@ compute_points_to_sets (void)
 		  *pt = cfun->gimple_df->escaped;
 		  pt->nonlocal = 1;
 		}
+	      else
+		memset (pt, 0, sizeof (struct pt_solution));
 	    }
 
 	  pt = gimple_call_clobber_set (stmt);
@@ -7582,8 +7693,8 @@ compute_points_to_sets (void)
 		     always escaped.  */
 		  if (writes_global_memory)
 		    {
-		      pt->nonlocal = writes_global_memory;
-		      pt->escaped = writes_global_memory;
+		      pt->nonlocal = 1;
+		      pt->escaped = 1;
 		    }
 		}
 	      else if (writes_global_memory)
@@ -7593,6 +7704,8 @@ compute_points_to_sets (void)
 		  *pt = cfun->gimple_df->escaped;
 		  pt->nonlocal = 1;
 		}
+	      else
+		memset (pt, 0, sizeof (struct pt_solution));
 	    }
 	}
     }
diff --git a/gcc/tree-ssa-threadbackward.c b/gcc/tree-ssa-threadbackward.c
index 38913b0..71989c2 100644
--- a/gcc/tree-ssa-threadbackward.c
+++ b/gcc/tree-ssa-threadbackward.c
@@ -44,18 +44,16 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-cfgcleanup.h"
 #include "tree-pretty-print.h"
 #include "cfghooks.h"
+#include "dbgcnt.h"
 
 // Path registry for the backwards threader.  After all paths have been
 // registered with register_path(), thread_through_all_blocks() is called
 // to modify the CFG.
 
-class back_threader_registry
+class back_threader_registry : public back_jt_path_registry
 {
 public:
   bool register_path (const vec<basic_block> &, edge taken);
-  bool thread_through_all_blocks (bool may_peel_loop_headers);
-private:
-  back_jt_path_registry m_lowlevel_registry;
 };
 
 // Class to abstract the profitability code for the backwards threader.
@@ -72,18 +70,28 @@ private:
   const bool m_speed_p;
 };
 
+// Back threader flags.
+#define BT_NONE 0
+// Generate fast code at the expense of code size.
+#define BT_SPEED 1
+// Resolve unknown SSAs on entry to a threading path.  If set, use the
+// ranger.  If not, assume all ranges on entry to a path are VARYING.
+#define BT_RESOLVE 2
+
 class back_threader
 {
 public:
-  back_threader (bool speed_p, bool resolve);
-  void maybe_thread_block (basic_block bb);
-  bool thread_through_all_blocks (bool may_peel_loop_headers);
+  back_threader (function *fun, unsigned flags, bool first);
+  ~back_threader ();
+  unsigned thread_blocks ();
 private:
+  void maybe_thread_block (basic_block bb);
   void find_paths (basic_block bb, tree name);
+  bool debug_counter ();
   edge maybe_register_path ();
-  bool find_paths_to_names (basic_block bb, bitmap imports);
-  bool resolve_def (tree name, bitmap interesting, vec<tree> &worklist);
-  bool resolve_phi (gphi *phi, bitmap imports);
+  void maybe_register_path_dump (edge taken_edge);
+  void find_paths_to_names (basic_block bb, bitmap imports);
+  void resolve_phi (gphi *phi, bitmap imports);
   edge find_taken_edge (const vec<basic_block> &path);
   edge find_taken_edge_cond (const vec<basic_block> &path, gcond *);
   edge find_taken_edge_switch (const vec<basic_block> &path, gswitch *);
@@ -92,8 +100,7 @@ private:
 
   back_threader_registry m_registry;
   back_threader_profitability m_profit;
-  gimple_ranger m_ranger;
-  path_range_query m_solver;
+  path_range_query *m_solver;
 
   // Current path being analyzed.
   auto_vec<basic_block> m_path;
@@ -112,26 +119,108 @@ private:
   // Set to TRUE if unknown SSA names along a path should be resolved
   // with the ranger.  Otherwise, unknown SSA names are assumed to be
   // VARYING.  Setting to true is more precise but slower.
-  bool m_resolve;
+  function *m_fun;
+  unsigned m_flags;
+  // Set to TRUE for the first of each thread[12] pass or the first of
+  // each threadfull[12] pass.  This is used to differentiate between
+  // the different threading passes so we can set up debug counters.
+  bool m_first;
 };
 
 // Used to differentiate unreachable edges, so we may stop the search
 // in a the given direction.
 const edge back_threader::UNREACHABLE_EDGE = (edge) -1;
 
-back_threader::back_threader (bool speed_p, bool resolve)
-  : m_profit (speed_p),
-    m_solver (m_ranger, resolve)
+back_threader::back_threader (function *fun, unsigned flags, bool first)
+  : m_profit (flags & BT_SPEED),
+    m_first (first)
 {
+  if (flags & BT_SPEED)
+    loop_optimizer_init (LOOPS_HAVE_PREHEADERS | LOOPS_HAVE_SIMPLE_LATCHES);
+  else
+    loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
+
+  m_fun = fun;
+  m_flags = flags;
+  m_solver = new path_range_query (flags & BT_RESOLVE);
   m_last_stmt = NULL;
-  m_resolve = resolve;
 }
 
-// Register the current path for jump threading if it's profitable to
-// do so.
+back_threader::~back_threader ()
+{
+  delete m_solver;
+
+  loop_optimizer_finalize ();
+}
+
+// A wrapper for the various debug counters for the threading passes.
+// Returns TRUE if it's OK to register the current threading
+// candidate.
+
+bool
+back_threader::debug_counter ()
+{
+  // The ethread pass is mostly harmless ;-).
+  if ((m_flags & BT_SPEED) == 0)
+    return true;
+
+  if (m_flags & BT_RESOLVE)
+    {
+      if (m_first && !dbg_cnt (back_threadfull1))
+	return false;
+
+      if (!m_first && !dbg_cnt (back_threadfull2))
+	return false;
+    }
+  else
+    {
+      if (m_first && !dbg_cnt (back_thread1))
+	return false;
+
+      if (!m_first && !dbg_cnt (back_thread2))
+	return false;
+    }
+  return true;
+}
+
+static void
+dump_path (FILE *dump_file, const vec<basic_block> &path)
+{
+  for (unsigned i = path.length (); i > 0; --i)
+    {
+      basic_block bb = path[i - 1];
+      fprintf (dump_file, "%d", bb->index);
+      if (i > 1)
+	fprintf (dump_file, "->");
+    }
+}
+
+// Dump details of an attempt to register a path.
+
+void
+back_threader::maybe_register_path_dump (edge taken)
+{
+  if (m_path.is_empty ())
+    return;
+
+  fprintf (dump_file, "path: ");
+  dump_path (dump_file, m_path);
+  fprintf (dump_file, "->");
+
+  if (taken == UNREACHABLE_EDGE)
+    fprintf (dump_file, "xx REJECTED (unreachable)\n");
+  else if (taken)
+    fprintf (dump_file, "%d SUCCESS\n", taken->dest->index);
+  else
+    fprintf (dump_file, "xx REJECTED\n");
+}
+
+// If an outgoing edge can be determined out of the current path,
+// register it for jump threading and return the taken edge.
 //
-// Return the known taken edge out of the path, even if the path was
-// not registered, or NULL if the taken edge could not be determined.
+// Return NULL if it is unprofitable to thread this path, or the
+// outgoing edge is unknown.  Return UNREACHABLE_EDGE if the path is
+// unreachable.
 
 edge
 back_threader::maybe_register_path ()
@@ -140,22 +229,32 @@ back_threader::maybe_register_path ()
 
   if (taken_edge && taken_edge != UNREACHABLE_EDGE)
     {
-      // Avoid circular paths.
       if (m_visited_bbs.contains (taken_edge->dest))
-	return UNREACHABLE_EDGE;
-
-      bool irreducible = false;
-      bool profitable
-	= m_profit.profitable_path_p (m_path, m_name, taken_edge, &irreducible);
-
-      if (profitable)
 	{
-	  m_registry.register_path (m_path, taken_edge);
+	  // Avoid circular paths by indicating there is nothing to
+	  // see in this direction.
+	  taken_edge = UNREACHABLE_EDGE;
+	}
+      else
+	{
+	  bool irreducible = false;
+	  if (m_profit.profitable_path_p (m_path, m_name, taken_edge,
+					  &irreducible)
+	      && debug_counter ())
+	    {
+	      m_registry.register_path (m_path, taken_edge);
 
-	  if (irreducible)
-	    vect_free_loop_info_assumptions (m_path[0]->loop_father);
+	      if (irreducible)
+		vect_free_loop_info_assumptions (m_path[0]->loop_father);
+	    }
+	  else
+	    taken_edge = NULL;
 	}
     }
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    maybe_register_path_dump (taken_edge);
+
   return taken_edge;
 }
 
@@ -189,8 +288,8 @@ back_threader::find_taken_edge_switch (const vec<basic_block> &path,
   tree name = gimple_switch_index (sw);
   int_range_max r;
 
-  m_solver.compute_ranges (path, m_imports);
-  m_solver.range_of_expr (r, name, sw);
+  m_solver->compute_ranges (path, m_imports);
+  m_solver->range_of_expr (r, name, sw);
 
   if (r.undefined_p ())
     return UNREACHABLE_EDGE;
@@ -198,11 +297,11 @@ back_threader::find_taken_edge_switch (const vec<basic_block> &path,
   if (r.varying_p ())
     return NULL;
 
-  tree val;
-  if (r.singleton_p (&val))
-    return ::find_taken_edge (gimple_bb (sw), val);
+  tree label = find_case_label_range (sw, &r);
+  if (!label)
+    return NULL;
 
-  return NULL;
+  return find_edge (gimple_bb (sw), label_to_block (cfun, CASE_LABEL (label)));
 }
 
 // Same as find_taken_edge, but for paths ending in a GIMPLE_COND.
@@ -213,10 +312,10 @@ back_threader::find_taken_edge_cond (const vec<basic_block> &path,
 {
   int_range_max r;
 
-  m_solver.compute_ranges (path, m_imports);
-  m_solver.range_of_stmt (r, cond);
+  m_solver->compute_ranges (path, m_imports);
+  m_solver->range_of_stmt (r, cond);
 
-  if (m_solver.unreachable_path_p ())
+  if (m_solver->unreachable_path_p ())
     return UNREACHABLE_EDGE;
 
   int_range<2> true_range (boolean_true_node, boolean_true_node);
@@ -247,149 +346,81 @@ populate_worklist (vec<tree> &worklist, bitmap bits)
     }
 }
 
-// If taking any of the incoming edges to a PHI causes the final
-// conditional of the current path to be constant, register the
-// path(s), and return TRUE.
+// Find jump threading paths that go through a PHI.
 
-bool
+void
 back_threader::resolve_phi (gphi *phi, bitmap interesting)
 {
   if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_phi_result (phi)))
-    return true;
+    return;
 
-  bool done = false;
   for (size_t i = 0; i < gimple_phi_num_args (phi); ++i)
     {
       edge e = gimple_phi_arg_edge (phi, i);
 
       // This is like path_crosses_loops in profitable_path_p but more
-      // restrictive, since profitable_path_p allows threading the
-      // first block because it would be redirected anyhow.
-      //
-      // If we loosened the restriction and used profitable_path_p()
-      // here instead, we would peel off the first iterations of loops
-      // in places like tree-ssa/pr14341.c.
+      // restrictive to avoid peeling off loop iterations (see
+      // tree-ssa/pr14341.c for an example).
       bool profitable_p = m_path[0]->loop_father == e->src->loop_father;
       if (!profitable_p)
 	{
 	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    fprintf (dump_file,
-		     "  FAIL: path through PHI in bb%d (incoming bb:%d) crosses loop\n",
-		     e->dest->index, e->src->index);
+	    {
+	      fprintf (dump_file,
+		       "  FAIL: path through PHI in bb%d (incoming bb:%d) crosses loop\n",
+		       e->dest->index, e->src->index);
+	      fprintf (dump_file, "path: %d->", e->src->index);
+	      dump_path (dump_file, m_path);
+	      fprintf (dump_file, "->xx REJECTED\n");
+	    }
 	  continue;
 	}
 
-      // FIXME: We currently stop looking if we find a threadable path
-      // through a PHI.  This is pessimistic, as there can be multiple
-      // paths that can resolve the path.  For example:
-      //
-      // x_5 = PHI <10(4), 20(5), ...>
-      // if (x_5 > 5)
-
       tree arg = gimple_phi_arg_def (phi, i);
-      if (TREE_CODE (arg) == SSA_NAME)
-	{
-	  unsigned v = SSA_NAME_VERSION (arg);
-
-	  // Avoid loops as in: x_5 = PHI <x_5(2), ...>.
-	  if (bitmap_bit_p (interesting, v))
-	    continue;
+      unsigned v = 0;
 
+      if (TREE_CODE (arg) == SSA_NAME
+	  && !bitmap_bit_p (interesting, SSA_NAME_VERSION (arg)))
+	{
+	  // Record that ARG is interesting when searching down this path.
+	  v = SSA_NAME_VERSION (arg);
+	  gcc_checking_assert (v != 0);
 	  bitmap_set_bit (interesting, v);
 	  bitmap_set_bit (m_imports, v);
-
-	  // When resolving unknowns, see if the incoming SSA can be
-	  // resolved on entry without having to keep looking back.
-	  bool keep_looking = true;
-	  if (m_resolve)
-	    {
-	      m_path.safe_push (e->src);
-	      if (maybe_register_path ())
-		{
-		  keep_looking = false;
-		  m_visited_bbs.add (e->src);
-		}
-	      m_path.pop ();
-	    }
-	  if (keep_looking)
-	    done |= find_paths_to_names (e->src, interesting);
-
-	  bitmap_clear_bit (interesting, v);
-	}
-      else if (TREE_CODE (arg) == INTEGER_CST)
-	{
-	  m_path.safe_push (e->src);
-	  edge taken_edge = maybe_register_path ();
-	  if (taken_edge && taken_edge != UNREACHABLE_EDGE)
-	    done = true;
-	  m_path.pop ();
 	}
-    }
-  return done;
-}
 
-// If the definition of NAME causes the final conditional of the
-// current path to be constant, register the path, and return TRUE.
+      find_paths_to_names (e->src, interesting);
 
-bool
-back_threader::resolve_def (tree name, bitmap interesting, vec<tree> &worklist)
-{
-  gimple *def_stmt = SSA_NAME_DEF_STMT (name);
-
-  // Handle PHIs.
-  if (is_a<gphi *> (def_stmt)
-      && resolve_phi (as_a<gphi *> (def_stmt), interesting))
-    return true;
-
-  // Defer copies of SSAs by adding the source to the worklist.
-  if (gimple_assign_single_p (def_stmt)
-      && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
-    {
-      tree rhs = gimple_assign_rhs1 (def_stmt);
-      bitmap_set_bit (m_imports, SSA_NAME_VERSION (rhs));
-      bitmap_set_bit (interesting, SSA_NAME_VERSION (rhs));
-      worklist.safe_push (rhs);
+      if (v)
+	bitmap_clear_bit (interesting, v);
     }
-  return false;
 }
 
 // Find jump threading paths to any of the SSA names in the
 // INTERESTING bitmap, and register any such paths.
 //
-// Return TRUE if no further processing past this block is necessary.
-// This is because we've either registered a path, or because there is
-// nothing of interesting beyond this block.
-//
 // BB is the current path being processed.
 
-bool
+void
 back_threader::find_paths_to_names (basic_block bb, bitmap interesting)
 {
   if (m_visited_bbs.add (bb))
-    return true;
+    return;
 
   m_path.safe_push (bb);
 
+  // Try to resolve the path without looking back.
   if (m_path.length () > 1
-      && !m_profit.profitable_path_p (m_path, m_name, NULL))
+      && (!m_profit.profitable_path_p (m_path, m_name, NULL)
+	  || maybe_register_path ()))
     {
       m_path.pop ();
       m_visited_bbs.remove (bb);
-      return false;
-    }
-
-  // Try to resolve the path with nothing but ranger knowledge.
-  if (m_resolve && m_path.length () > 1 && maybe_register_path ())
-    {
-      m_path.pop ();
-      m_visited_bbs.remove (bb);
-      return true;
+      return;
     }
 
   auto_bitmap processed;
-  unsigned i;
   bool done = false;
-
   // We use a worklist instead of iterating through the bitmap,
   // because we may add new items in-flight.
   auto_vec<tree> worklist (bitmap_count_bits (interesting));
@@ -398,52 +429,37 @@ back_threader::find_paths_to_names (basic_block bb, bitmap interesting)
     {
       tree name = worklist.pop ();
       unsigned i = SSA_NAME_VERSION (name);
-      basic_block def_bb = gimple_bb (SSA_NAME_DEF_STMT (name));
-
-      // Process any names defined in this block.
-      if (def_bb == bb)
-	{
-	  bitmap_set_bit (processed, i);
+      gimple *def_stmt = SSA_NAME_DEF_STMT (name);
+      basic_block def_bb = gimple_bb (def_stmt);
 
-	  if (resolve_def (name, interesting, worklist))
-	    {
-	      done = true;
-	      goto leave_bb;
-	    }
-	}
-      // Examine blocks that define or export an interesting SSA,
-      // since they may compute a range which resolve this path.
-      if ((def_bb == bb
-	   || bitmap_bit_p (m_ranger.gori ().exports (bb), i))
-	  && m_path.length () > 1)
+      // Process any PHIs defined in this block.
+      if (def_bb == bb
+	  && bitmap_set_bit (processed, i)
+	  && gimple_code (def_stmt) == GIMPLE_PHI)
 	{
-	  if (maybe_register_path ())
-	    {
-	      done = true;
-	      goto leave_bb;
-	    }
+	  resolve_phi (as_a<gphi *> (def_stmt), interesting);
+	  done = true;
+	  break;
 	}
     }
-
   // If there are interesting names not yet processed, keep looking.
-  bitmap_and_compl_into (interesting, processed);
-  if (!bitmap_empty_p (interesting))
+  if (!done)
     {
-      edge_iterator iter;
-      edge e;
-      FOR_EACH_EDGE (e, iter, bb->preds)
-	if ((e->flags & EDGE_ABNORMAL) == 0)
-	  done |= find_paths_to_names (e->src, interesting);
+      bitmap_and_compl_into (interesting, processed);
+      if (!bitmap_empty_p (interesting))
+	{
+	  edge_iterator iter;
+	  edge e;
+	  FOR_EACH_EDGE (e, iter, bb->preds)
+	    if ((e->flags & EDGE_ABNORMAL) == 0)
+	      find_paths_to_names (e->src, interesting);
+	}
     }
 
- leave_bb:
-  bitmap_iterator bi;
-  EXECUTE_IF_SET_IN_BITMAP (processed, 0, i, bi)
-    bitmap_set_bit (interesting, i);
-
+  // Reset things to their original state.
+  bitmap_ior_into (interesting, processed);
   m_path.pop ();
   m_visited_bbs.remove (bb);
-  return done;
 }
 
 // Search backwards from BB looking for paths where the final
@@ -466,10 +482,9 @@ back_threader::find_paths (basic_block bb, tree name)
       m_visited_bbs.empty ();
       m_path.truncate (0);
       m_name = name;
-      bitmap_clear (m_imports);
+      m_solver->compute_imports (m_imports, bb);
 
       auto_bitmap interesting;
-      bitmap_copy (m_imports, m_ranger.gori ().imports (bb));
       bitmap_copy (interesting, m_imports);
       find_paths_to_names (bb, interesting);
     }
@@ -519,38 +534,17 @@ back_threader::maybe_thread_block (basic_block bb)
   find_paths (bb, name);
 }
 
-// Perform the actual jump threading for the all queued paths.
-
-bool
-back_threader::thread_through_all_blocks (bool may_peel_loop_headers)
-{
-  return m_registry.thread_through_all_blocks (may_peel_loop_headers);
-}
-
-// Dump a sequence of BBs through the CFG.
-
-DEBUG_FUNCTION void
-dump_path (FILE *dump_file, const vec<basic_block> &path)
-{
-  for (size_t i = 0; i < path.length (); ++i)
-    {
-      fprintf (dump_file, "BB%d", path[i]->index);
-      if (i + 1 < path.length ())
-	fprintf (dump_file, " <- ");
-    }
-  fprintf (dump_file, "\n");
-}
-
 DEBUG_FUNCTION void
 debug (const vec <basic_block> &path)
 {
   dump_path (stderr, path);
+  fputc ('\n', stderr);
 }
 
 void
 back_threader::dump (FILE *out)
 {
-  m_solver.dump (out);
+  m_solver->dump (out);
   fprintf (out, "\nCandidates for pre-computation:\n");
   fprintf (out, "===================================\n");
 
@@ -571,12 +565,6 @@ back_threader::debug ()
   dump (stderr);
 }
 
-bool
-back_threader_registry::thread_through_all_blocks (bool may_peel_loop_headers)
-{
-  return m_lowlevel_registry.thread_through_all_blocks (may_peel_loop_headers);
-}
-
 /* Examine jump threading path PATH and return TRUE if it is profitable to
    thread it, otherwise return FALSE.
 
@@ -624,7 +612,6 @@ back_threader_profitability::profitable_path_p (const vec<basic_block> &m_path,
   int n_insns = 0;
   gimple_stmt_iterator gsi;
   loop_p loop = m_path[0]->loop_father;
-  bool path_crosses_loops = false;
   bool threaded_through_latch = false;
   bool multiway_branch_in_path = false;
   bool threaded_multiway_branch = false;
@@ -643,30 +630,15 @@ back_threader_profitability::profitable_path_p (const vec<basic_block> &m_path,
 
       if (dump_file && (dump_flags & TDF_DETAILS))
 	fprintf (dump_file, " bb:%i", bb->index);
-      /* Remember, blocks in the path are stored in opposite order
-	 in the PATH array.  The last entry in the array represents
-	 the block with an outgoing edge that we will redirect to the
-	 jump threading path.  Thus we don't care about that block's
-	 loop father, nor how many statements are in that block because
-	 it will not be copied or whether or not it ends in a multiway
-	 branch.  */
+      /* Remember, blocks in the path are stored in opposite order in
+	 the PATH array.  The last entry in the array represents the
+	 block with an outgoing edge that we will redirect to the jump
+	 threading path.  Thus we don't care how many statements are
+	 in that block because it will not be copied or whether or not
+	 it ends in a multiway branch.  */
       if (j < m_path.length () - 1)
 	{
 	  int orig_n_insns = n_insns;
-	  if (bb->loop_father != loop)
-	    {
-	      path_crosses_loops = true;
-
-	      // Dump rest of blocks.
-	      if (dump_file && (dump_flags & TDF_DETAILS))
-		for (j++; j < m_path.length (); j++)
-		  {
-		    bb = m_path[j];
-		    fprintf (dump_file, " bb:%i", bb->index);
-		  }
-	      break;
-	    }
-
 	  /* PHIs in the path will create degenerate PHIS in the
 	     copied path which will then get propagated away, so
 	     looking at just the duplicate path the PHIs would
@@ -721,8 +693,6 @@ back_threader_profitability::profitable_path_p (const vec<basic_block> &m_path,
 		return false;
 	      /* Do not count empty statements and labels.  */
 	      if (gimple_code (stmt) != GIMPLE_NOP
-		  && !(gimple_code (stmt) == GIMPLE_ASSIGN
-		       && gimple_assign_rhs_code (stmt) == ASSERT_EXPR)
 		  && !is_gimple_debug (stmt))
 		n_insns += estimate_num_insns (stmt, &eni_size_weights);
 	    }
@@ -787,14 +757,6 @@ back_threader_profitability::profitable_path_p (const vec<basic_block> &m_path,
 	*creates_irreducible_loop = true;
     }
 
-  if (path_crosses_loops)
-    {
-      if (dump_file && (dump_flags & TDF_DETAILS))
-	fprintf (dump_file, "  FAIL: Jump-thread path not considered: "
-		 "the path crosses loops.\n");
-      return false;
-    }
-
   /* Threading is profitable if the path duplicated is hot but also
      in a case we separate cold path from hot path and permit optimization
      of the hot path later.  Be on the agressive side here. In some testcases,
@@ -903,8 +865,7 @@ bool
 back_threader_registry::register_path (const vec<basic_block> &m_path,
 				       edge taken_edge)
 {
-  vec<jump_thread_edge *> *jump_thread_path
-    = m_lowlevel_registry.allocate_thread_path ();
+  vec<jump_thread_edge *> *jump_thread_path = allocate_thread_path ();
 
   // The generic copier ignores the edge type.  We can build the
   // thread edges with any type.
@@ -915,54 +876,32 @@ back_threader_registry::register_path (const vec<basic_block> &m_path,
 
       edge e = find_edge (bb1, bb2);
       gcc_assert (e);
-      m_lowlevel_registry.push_edge (jump_thread_path, e, EDGE_COPY_SRC_BLOCK);
+      push_edge (jump_thread_path, e, EDGE_COPY_SRC_BLOCK);
     }
 
-  m_lowlevel_registry.push_edge (jump_thread_path,
-				 taken_edge, EDGE_NO_COPY_SRC_BLOCK);
-  m_lowlevel_registry.register_jump_thread (jump_thread_path);
+  push_edge (jump_thread_path, taken_edge, EDGE_NO_COPY_SRC_BLOCK);
+  register_jump_thread (jump_thread_path);
   return true;
 }
 
-// Try to thread blocks in FUN.  RESOLVE is TRUE when fully resolving
-// unknown SSAs.  SPEED is TRUE when optimizing for speed.
+// Thread all suitable paths in the current function.
 //
-// Return TRUE if any jump thread paths were registered.
+// Return TODO_flags.
 
-static bool
-try_thread_blocks (function *fun, bool resolve, bool speed)
+unsigned int
+back_threader::thread_blocks ()
 {
-  back_threader threader (speed, resolve);
   basic_block bb;
-  FOR_EACH_BB_FN (bb, fun)
-    {
-      if (EDGE_COUNT (bb->succs) > 1)
-	threader.maybe_thread_block (bb);
-    }
-  return threader.thread_through_all_blocks (/*peel_loop_headers=*/true);
-}
+  FOR_EACH_BB_FN (bb, m_fun)
+    if (EDGE_COUNT (bb->succs) > 1)
+      maybe_thread_block (bb);
 
-static unsigned int
-do_early_thread_jumps (function *fun, bool resolve)
-{
-  loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
-
-  try_thread_blocks (fun, resolve, /*speed=*/false);
-
-  loop_optimizer_finalize ();
-  return 0;
-}
+  bool changed = m_registry.thread_through_all_blocks (true);
 
-static unsigned int
-do_thread_jumps (function *fun, bool resolve)
-{
-  loop_optimizer_init (LOOPS_HAVE_PREHEADERS | LOOPS_HAVE_SIMPLE_LATCHES);
-
-  bool changed = try_thread_blocks (fun, resolve, /*speed=*/true);
+  if (m_flags & BT_SPEED)
+    return changed ? TODO_cleanup_cfg : 0;
 
-  loop_optimizer_finalize ();
-
-  return changed ? TODO_cleanup_cfg : 0;
+  return false;
 }
 
 namespace {
@@ -996,7 +935,7 @@ const pass_data pass_data_thread_jumps =
 const pass_data pass_data_thread_jumps_full =
 {
   GIMPLE_PASS,
-  "thread-full",
+  "threadfull",
   OPTGROUP_NONE,
   TV_TREE_SSA_THREAD_JUMPS,
   ( PROP_cfg | PROP_ssa ),
@@ -1018,14 +957,21 @@ public:
   {
     return new pass_early_thread_jumps (m_ctxt);
   }
+  void set_pass_param (unsigned int, bool param) override
+  {
+    m_first = param;
+  }
   bool gate (function *) override
   {
     return flag_thread_jumps;
   }
   unsigned int execute (function *fun) override
   {
-    return do_early_thread_jumps (fun, /*resolve=*/false);
+    back_threader threader (fun, BT_NONE, m_first);
+    return threader.thread_blocks ();
   }
+private:
+  bool m_first;
 };
 
 // Jump threading pass without resolving of unknown SSAs.
@@ -1039,14 +985,21 @@ public:
   {
     return new pass_thread_jumps (m_ctxt);
   }
+  void set_pass_param (unsigned int, bool param) override
+  {
+    m_first = param;
+  }
   bool gate (function *) override
   {
     return flag_thread_jumps && flag_expensive_optimizations;
   }
   unsigned int execute (function *fun) override
   {
-    return do_thread_jumps (fun, /*resolve=*/false);
+    back_threader threader (fun, BT_SPEED, m_first);
+    return threader.thread_blocks ();
   }
+private:
+  bool m_first;
 };
 
 // Jump threading pass that fully resolves unknown SSAs.
@@ -1060,14 +1013,21 @@ public:
   {
     return new pass_thread_jumps_full (m_ctxt);
   }
+  void set_pass_param (unsigned int, bool param) override
+  {
+    m_first = param;
+  }
   bool gate (function *) override
   {
     return flag_thread_jumps && flag_expensive_optimizations;
   }
   unsigned int execute (function *fun) override
   {
-    return do_thread_jumps (fun, /*resolve=*/true);
+    back_threader threader (fun, BT_SPEED | BT_RESOLVE, m_first);
+    return threader.thread_blocks ();
   }
+private:
+  bool m_first;
 };
 
 } // namespace {
diff --git a/gcc/tree-ssa-threadedge.c b/gcc/tree-ssa-threadedge.c
index a63a976..891c62a 100644
--- a/gcc/tree-ssa-threadedge.c
+++ b/gcc/tree-ssa-threadedge.c
@@ -37,7 +37,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "cfganal.h"
 #include "alloc-pool.h"
 #include "vr-values.h"
-#include "gimple-ssa-evrp-analyze.h"
 #include "gimple-range.h"
 #include "gimple-range-path.h"
 
@@ -1334,25 +1333,19 @@ jt_state::register_equivs_stmt (gimple *stmt, basic_block bb,
      to expose more context sensitive equivalences which in turn may
      allow us to simplify the condition at the end of the loop.
 
-     Handle simple copy operations as well as implied copies from
-     ASSERT_EXPRs.  */
+     Handle simple copy operations.  */
   tree cached_lhs = NULL;
   if (gimple_assign_single_p (stmt)
       && TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME)
     cached_lhs = gimple_assign_rhs1 (stmt);
-  else if (gimple_assign_single_p (stmt)
-	   && TREE_CODE (gimple_assign_rhs1 (stmt)) == ASSERT_EXPR)
-    cached_lhs = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
   else
     {
-      /* A statement that is not a trivial copy or ASSERT_EXPR.
+      /* A statement that is not a trivial copy.
 	 Try to fold the new expression.  Inserting the
 	 expression into the hash table is unlikely to help.  */
       /* ???  The DOM callback below can be changed to setting
 	 the mprts_hook around the call to thread_across_edge,
-	 avoiding the use substitution.  The VRP hook should be
-	 changed to properly valueize operands itself using
-	 SSA_NAME_VALUE in addition to its own lattice.  */
+	 avoiding the use substitution.  */
       cached_lhs = gimple_fold_stmt_to_constant_1 (stmt,
 						   threadedge_valueize);
       if (NUM_SSA_OPERANDS (stmt, SSA_OP_ALL_USES) != 0
diff --git a/gcc/tree-ssa-threadupdate.c b/gcc/tree-ssa-threadupdate.c
index 8e6f043..8aac733 100644
--- a/gcc/tree-ssa-threadupdate.c
+++ b/gcc/tree-ssa-threadupdate.c
@@ -253,6 +253,9 @@ dump_jump_thread_path (FILE *dump_file,
 	default:
 	  gcc_unreachable ();
 	}
+
+      if ((path[i]->e->flags & EDGE_DFS_BACK) != 0)
+	fprintf (dump_file, " (back)");
     }
   fprintf (dump_file, "; \n");
 }
diff --git a/gcc/tree-ssa-uninit.c b/gcc/tree-ssa-uninit.c
index d67534f..1df0bcc 100644
--- a/gcc/tree-ssa-uninit.c
+++ b/gcc/tree-ssa-uninit.c
@@ -744,7 +744,8 @@ maybe_warn_pass_by_reference (gcall *stmt, wlimits &wlims)
 	wlims.always_executed = false;
 
       /* Ignore args we are not going to read from.  */
-      if (gimple_call_arg_flags (stmt, argno - 1) & (EAF_UNUSED | EAF_NOREAD))
+      if (gimple_call_arg_flags (stmt, argno - 1)
+	  & (EAF_UNUSED | EAF_NO_DIRECT_READ))
 	continue;
 
       tree arg = gimple_call_arg (stmt, argno - 1);
diff --git a/gcc/tree-ssa.c b/gcc/tree-ssa.c
index fde13de..1565e21 100644
--- a/gcc/tree-ssa.c
+++ b/gcc/tree-ssa.c
@@ -434,14 +434,13 @@ insert_debug_temp_for_var_def (gimple_stmt_iterator *gsi, tree var)
       else
 	{
 	  gdebug *def_temp;
-	  tree vexpr = make_node (DEBUG_EXPR_DECL);
+	  tree vexpr = build_debug_expr_decl (TREE_TYPE (value));
 
 	  def_temp = gimple_build_debug_bind (vexpr,
 					      unshare_expr (value),
 					      def_stmt);
 
-	  DECL_ARTIFICIAL (vexpr) = 1;
-	  TREE_TYPE (vexpr) = TREE_TYPE (value);
+	  /* FIXME: Is setting the mode really necessary? */
 	  if (DECL_P (value))
 	    SET_DECL_MODE (vexpr, DECL_MODE (value));
 	  else
@@ -649,11 +648,9 @@ verify_vssa (basic_block bb, tree current_vdef, sbitmap visited)
 {
   bool err = false;
 
-  if (bitmap_bit_p (visited, bb->index))
+  if (!bitmap_set_bit (visited, bb->index))
     return false;
 
-  bitmap_set_bit (visited, bb->index);
-
   /* Pick up the single virtual PHI def.  */
   gphi *phi = NULL;
   for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);
diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
index 556ae97..2ea8e98 100644
--- a/gcc/tree-vect-data-refs.c
+++ b/gcc/tree-vect-data-refs.c
@@ -887,10 +887,11 @@ vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance)
   return res;
 }
 
-/* Return the misalignment of DR_INFO accessed in VECTYPE.  */
+/* Return the misalignment of DR_INFO accessed in VECTYPE with OFFSET
+   applied.  */
 
 int
-dr_misalignment (dr_vec_info *dr_info, tree vectype)
+dr_misalignment (dr_vec_info *dr_info, tree vectype, poly_int64 offset)
 {
   HOST_WIDE_INT diff = 0;
   /* Alignment is only analyzed for the first element of a DR group,
@@ -919,13 +920,9 @@ dr_misalignment (dr_vec_info *dr_info, tree vectype)
 		targetm.vectorize.preferred_vector_alignment (vectype)))
     return DR_MISALIGNMENT_UNKNOWN;
 
-  /* If this is a backward running DR then first access in the larger
-     vectype actually is N-1 elements before the address in the DR.
-     Adjust misalign accordingly.  */
-  poly_int64 misalignment = misalign + diff;
-  if (tree_int_cst_sgn (DR_STEP (dr_info->dr)) < 0)
-    misalignment += ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
-		     * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
+  /* Apply the offset from the DR group start and the externally supplied
+     offset which can for example result from a negative stride access.  */
+  poly_int64 misalignment = misalign + diff + offset;
 
   /* vect_compute_data_ref_alignment will have ensured that target_alignment
      is constant and otherwise set misalign to DR_MISALIGNMENT_UNKNOWN.  */
@@ -1549,8 +1546,15 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
       int misalignment;
       unsigned HOST_WIDE_INT alignment;
 
+      bool negative = tree_int_cst_compare (DR_STEP (dr_info->dr),
+					    size_zero_node) < 0;
+      poly_int64 off = 0;
+      if (negative)
+	off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
+	       * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
+
       if (npeel == 0)
-	misalignment = dr_misalignment (dr_info, vectype);
+	misalignment = dr_misalignment (dr_info, vectype, off);
       else if (dr_info == dr0_info
 	       || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info))
 	misalignment = 0;
@@ -1560,7 +1564,7 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
 	misalignment = DR_MISALIGNMENT_UNKNOWN;
       else
 	{
-	  misalignment = dr_misalignment (dr_info, vectype);
+	  misalignment = dr_misalignment (dr_info, vectype, off);
 	  misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
 	  misalignment &= alignment - 1;
 	}
@@ -1960,8 +1964,11 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
 	       */
 	      unsigned int target_align =
 		DR_TARGET_ALIGNMENT (dr_info).to_constant ();
-	      unsigned int dr_size = vect_get_scalar_dr_size (dr_info);
-	      unsigned int mis = dr_misalignment (dr_info, vectype);
+	      unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (dr_info);
+	      poly_int64 off = 0;
+	      if (negative)
+		off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
+	      unsigned int mis = dr_misalignment (dr_info, vectype, off);
 	      mis = negative ? mis : -mis;
 	      if (mis != 0)
 		npeel_tmp = (mis & (target_align - 1)) / dr_size;
@@ -1994,9 +2001,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
 		 prune all entries from the peeling hashtable which cause
 		 DRs to be not supported.  */
 	      bool supportable_if_not_aligned
-		= targetm.vectorize.support_vector_misalignment
-		    (TYPE_MODE (vectype), TREE_TYPE (DR_REF (dr_info->dr)),
-		     DR_MISALIGNMENT_UNKNOWN, false);
+		= vect_supportable_dr_alignment
+		    (loop_vinfo, dr_info, vectype, DR_MISALIGNMENT_UNKNOWN);
 	      while (known_le (npeel_tmp, nscalars))
                 {
                   vect_peeling_hash_insert (&peeling_htab, loop_vinfo,
@@ -2239,8 +2245,13 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
                  updating DR_MISALIGNMENT values.  The peeling factor is the
                  vectorization factor minus the misalignment as an element
                  count.  */
+	      tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+	      poly_int64 off = 0;
+	      if (negative)
+		off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
+		       * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
 	      unsigned int mis
-		= dr_misalignment (dr0_info, STMT_VINFO_VECTYPE (stmt_info));
+		= dr_misalignment (dr0_info, vectype, off);
 	      mis = negative ? mis : -mis;
 	      /* If known_alignment_for_access_p then we have set
 	         DR_MISALIGNMENT which is only done if we know it at compiler
@@ -2384,19 +2395,27 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
       FOR_EACH_VEC_ELT (datarefs, i, dr)
         {
 	  dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
-	  stmt_vec_info stmt_info = dr_info->stmt;
-	  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-	  int misalignment;
-	  if ((misalignment = dr_misalignment (dr_info, vectype)) == 0
-	      || !vect_relevant_for_alignment_p (dr_info))
+	  if (!vect_relevant_for_alignment_p (dr_info))
 	    continue;
 
+	  stmt_vec_info stmt_info = dr_info->stmt;
 	  if (STMT_VINFO_STRIDED_P (stmt_info))
 	    {
 	      do_versioning = false;
 	      break;
 	    }
 
+	  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+	  bool negative = tree_int_cst_compare (DR_STEP (dr),
+						size_zero_node) < 0;
+	  poly_int64 off = 0;
+	  if (negative)
+	    off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
+		   * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
+	  int misalignment;
+	  if ((misalignment = dr_misalignment (dr_info, vectype, off)) == 0)
+	    continue;
+
 	  enum dr_alignment_support supportable_dr_alignment
 	    = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
 					     misalignment);
@@ -4753,9 +4772,6 @@ vect_duplicate_ssa_name_ptr_info (tree name, dr_vec_info *dr_info)
 	    is as follows:
 	    if LOOP=i_loop:	&in		(relative to i_loop)
 	    if LOOP=j_loop: 	&in+i*2B	(relative to j_loop)
-   BYTE_OFFSET: Optional, defaulted to NULL.  If supplied, it is added to the
-	    initial address.  Unlike OFFSET, which is number of elements to
-	    be added, BYTE_OFFSET is measured in bytes.
 
    Output:
    1. Return an SSA_NAME whose value is the address of the memory location of
@@ -4768,8 +4784,7 @@ vect_duplicate_ssa_name_ptr_info (tree name, dr_vec_info *dr_info)
 tree
 vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
 				      gimple_seq *new_stmt_list,
-				      tree offset,
-				      tree byte_offset)
+				      tree offset)
 {
   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
   struct data_reference *dr = dr_info->dr;
@@ -4778,7 +4793,6 @@ vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
   tree dest;
   gimple_seq seq = NULL;
   tree vect_ptr_type;
-  tree step = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr)));
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
 
@@ -4802,17 +4816,10 @@ vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
 
   if (offset)
     {
-      offset = fold_build2 (MULT_EXPR, sizetype,
-			    fold_convert (sizetype, offset), step);
+      offset = fold_convert (sizetype, offset);
       base_offset = fold_build2 (PLUS_EXPR, sizetype,
 				 base_offset, offset);
     }
-  if (byte_offset)
-    {
-      byte_offset = fold_convert (sizetype, byte_offset);
-      base_offset = fold_build2 (PLUS_EXPR, sizetype,
-				 base_offset, byte_offset);
-    }
 
   /* base + base_offset */
   if (loop_vinfo)
@@ -4861,15 +4868,11 @@ vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
    2. AGGR_TYPE: the type of the reference, which should be either a vector
         or an array.
    3. AT_LOOP: the loop where the vector memref is to be created.
-   4. OFFSET (optional): an offset to be added to the initial address accessed
-	by the data-ref in STMT_INFO.
+   4. OFFSET (optional): a byte offset to be added to the initial address
+	accessed by the data-ref in STMT_INFO.
    5. BSI: location where the new stmts are to be placed if there is no loop
    6. ONLY_INIT: indicate if ap is to be updated in the loop, or remain
         pointing to the initial address.
-   7. BYTE_OFFSET (optional, defaults to NULL): a byte offset to be added
-	to the initial address accessed by the data-ref in STMT_INFO.  This is
-	similar to OFFSET, but OFFSET is counted in elements, while BYTE_OFFSET
-	in bytes.
    8. IV_STEP (optional, defaults to NULL): the amount that should be added
 	to the IV during each iteration of the loop.  NULL says to move
 	by one copy of AGGR_TYPE up or down, depending on the step of the
@@ -4886,7 +4889,7 @@ vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
       if OFFSET is not supplied:
          initial_address = &a[init];
       if OFFSET is supplied:
-         initial_address = &a[init + OFFSET];
+	 initial_address = &a[init] + OFFSET;
       if BYTE_OFFSET is supplied:
 	 initial_address = &a[init] + BYTE_OFFSET;
 
@@ -4904,7 +4907,7 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
 			  tree aggr_type, class loop *at_loop, tree offset,
 			  tree *initial_address, gimple_stmt_iterator *gsi,
 			  gimple **ptr_incr, bool only_init,
-			  tree byte_offset, tree iv_step)
+			  tree iv_step)
 {
   const char *base_name;
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
@@ -5032,11 +5035,11 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
   /* (2) Calculate the initial address of the aggregate-pointer, and set
      the aggregate-pointer to point to it before the loop.  */
 
-  /* Create: (&(base[init_val+offset]+byte_offset) in the loop preheader.  */
+  /* Create: (&(base[init_val]+offset) in the loop preheader.  */
 
   new_temp = vect_create_addr_base_for_vector_ref (vinfo,
 						   stmt_info, &new_stmt_list,
-						   offset, byte_offset);
+						   offset);
   if (new_stmt_list)
     {
       if (pe)
diff --git a/gcc/tree-vect-loop-manip.c b/gcc/tree-vect-loop-manip.c
index 4988c93..f788deb 100644
--- a/gcc/tree-vect-loop-manip.c
+++ b/gcc/tree-vect-loop-manip.c
@@ -1625,7 +1625,9 @@ get_misalign_in_elems (gimple **seq, loop_vec_info loop_vinfo)
   bool negative = tree_int_cst_compare (DR_STEP (dr_info->dr),
 					size_zero_node) < 0;
   tree offset = (negative
-		 ? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1)
+		 ? size_int ((-TYPE_VECTOR_SUBPARTS (vectype) + 1)
+			     * TREE_INT_CST_LOW
+				 (TYPE_SIZE_UNIT (TREE_TYPE (vectype))))
 		 : size_zero_node);
   tree start_addr = vect_create_addr_base_for_vector_ref (loop_vinfo,
 							  stmt_info, seq,
@@ -1820,7 +1822,8 @@ vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters,
   FOR_EACH_VEC_ELT (datarefs, i, dr)
     {
       dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
-      if (!STMT_VINFO_GATHER_SCATTER_P (dr_info->stmt))
+      if (!STMT_VINFO_GATHER_SCATTER_P (dr_info->stmt)
+	  && !STMT_VINFO_SIMD_LANE_ACCESS_P (dr_info->stmt))
 	vect_update_init_of_dr (dr_info, niters, code);
     }
 }
@@ -3226,7 +3229,9 @@ vect_create_cond_for_align_checks (loop_vec_info loop_vinfo,
       bool negative = tree_int_cst_compare
 	(DR_STEP (STMT_VINFO_DATA_REF (stmt_info)), size_zero_node) < 0;
       tree offset = negative
-	? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1) : size_zero_node;
+	? size_int ((-TYPE_VECTOR_SUBPARTS (vectype) + 1)
+		    * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))))
+	: size_zero_node;
 
       /* create: addr_tmp = (int)(address_of_first_vector) */
       addr_base =
@@ -3557,12 +3562,29 @@ vect_loop_versioning (loop_vec_info loop_vinfo,
 			 "applying loop versioning to outer loop %d\n",
 			 loop_to_version->num);
 
+      unsigned orig_pe_idx = loop_preheader_edge (loop)->dest_idx;
+
       initialize_original_copy_tables ();
       nloop = loop_version (loop_to_version, cond_expr, &condition_bb,
 			    prob, prob.invert (), prob, prob.invert (), true);
       gcc_assert (nloop);
       nloop = get_loop_copy (loop);
 
+      /* For cycle vectorization with SLP we rely on the PHI arguments
+	 appearing in the same order as the SLP node operands which for the
+	 loop PHI nodes means the preheader edge dest index needs to remain
+	 the same for the analyzed loop which also becomes the vectorized one.
+	 Make it so in case the state after versioning differs by redirecting
+	 the first edge into the header to the same destination which moves
+	 it last.  */
+      if (loop_preheader_edge (loop)->dest_idx != orig_pe_idx)
+	{
+	  edge e = EDGE_PRED (loop->header, 0);
+	  ssa_redirect_edge (e, e->dest);
+	  flush_pending_stmts (e);
+	}
+      gcc_assert (loop_preheader_edge (loop)->dest_idx == orig_pe_idx);
+
       /* Kill off IFN_LOOP_VECTORIZED_CALL in the copy, nobody will
          reap those otherwise;  they also refer to the original
 	 loops.  */
diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index 961c162..73efdb9 100644
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -814,13 +814,15 @@ bb_in_loop_p (const_basic_block bb, const void *data)
    stmt_vec_info structs for all the stmts in LOOP_IN.  */
 
 _loop_vec_info::_loop_vec_info (class loop *loop_in, vec_info_shared *shared)
-  : vec_info (vec_info::loop, init_cost (loop_in, false), shared),
+  : vec_info (vec_info::loop, shared),
     loop (loop_in),
     bbs (XCNEWVEC (basic_block, loop->num_nodes)),
     num_itersm1 (NULL_TREE),
     num_iters (NULL_TREE),
     num_iters_unchanged (NULL_TREE),
     num_iters_assumptions (NULL_TREE),
+    vector_costs (nullptr),
+    scalar_costs (nullptr),
     th (0),
     versioning_threshold (0),
     vectorization_factor (0),
@@ -838,9 +840,6 @@ _loop_vec_info::_loop_vec_info (class loop *loop_in, vec_info_shared *shared)
     ivexpr_map (NULL),
     scan_map (NULL),
     slp_unrolling_factor (1),
-    single_scalar_iteration_cost (0),
-    vec_outside_cost (0),
-    vec_inside_cost (0),
     inner_loop_cost_factor (param_vect_inner_loop_cost_factor),
     vectorizable (false),
     can_use_partial_vectors_p (param_vect_partial_vector_usage != 0),
@@ -932,6 +931,8 @@ _loop_vec_info::~_loop_vec_info ()
   delete ivexpr_map;
   delete scan_map;
   epilogue_vinfos.release ();
+  delete scalar_costs;
+  delete vector_costs;
 
   /* When we release an epiloge vinfo that we do not intend to use
      avoid clearing AUX of the main loop which should continue to
@@ -1292,24 +1293,19 @@ vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo)
     }
 
   /* Now accumulate cost.  */
-  void *target_cost_data = init_cost (loop, true);
+  loop_vinfo->scalar_costs = init_cost (loop_vinfo, true);
   stmt_info_for_cost *si;
   int j;
   FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
 		    j, si)
-    (void) add_stmt_cost (loop_vinfo, target_cost_data, si->count,
+    (void) add_stmt_cost (loop_vinfo->scalar_costs, si->count,
 			  si->kind, si->stmt_info, si->vectype,
 			  si->misalign, si->where);
-  unsigned prologue_cost = 0, body_cost = 0, epilogue_cost = 0;
-  finish_cost (target_cost_data, &prologue_cost, &body_cost,
-	       &epilogue_cost);
-  destroy_cost_data (target_cost_data);
-  LOOP_VINFO_SINGLE_SCALAR_ITERATION_COST (loop_vinfo)
-    = prologue_cost + body_cost + epilogue_cost;
+  loop_vinfo->scalar_costs->finish_cost (nullptr);
 }
 
 
-/* Function vect_analyze_loop_form_1.
+/* Function vect_analyze_loop_form.
 
    Verify that certain CFG restrictions hold, including:
    - the loop has a pre-header
@@ -1319,9 +1315,7 @@ vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo)
      niter could be analyzed under some assumptions.  */
 
 opt_result
-vect_analyze_loop_form_1 (class loop *loop, gcond **loop_cond,
-			  tree *assumptions, tree *number_of_iterationsm1,
-			  tree *number_of_iterations, gcond **inner_loop_cond)
+vect_analyze_loop_form (class loop *loop, vect_loop_form_info *info)
 {
   DUMP_VECT_SCOPE ("vect_analyze_loop_form");
 
@@ -1329,6 +1323,7 @@ vect_analyze_loop_form_1 (class loop *loop, gcond **loop_cond,
      vs. an outer (nested) loop.
      (FORNOW. May want to relax some of these restrictions in the future).  */
 
+  info->inner_loop_cond = NULL;
   if (!loop->inner)
     {
       /* Inner-most loop.  We currently require that the number of BBs is
@@ -1393,11 +1388,8 @@ vect_analyze_loop_form_1 (class loop *loop, gcond **loop_cond,
 				       " unsupported outerloop form.\n");
 
       /* Analyze the inner-loop.  */
-      tree inner_niterm1, inner_niter, inner_assumptions;
-      opt_result res
-	= vect_analyze_loop_form_1 (loop->inner, inner_loop_cond,
-				    &inner_assumptions, &inner_niterm1,
-				    &inner_niter, NULL);
+      vect_loop_form_info inner;
+      opt_result res = vect_analyze_loop_form (loop->inner, &inner);
       if (!res)
 	{
 	  if (dump_enabled_p ())
@@ -1408,11 +1400,11 @@ vect_analyze_loop_form_1 (class loop *loop, gcond **loop_cond,
 
       /* Don't support analyzing niter under assumptions for inner
 	 loop.  */
-      if (!integer_onep (inner_assumptions))
+      if (!integer_onep (inner.assumptions))
 	return opt_result::failure_at (vect_location,
 				       "not vectorized: Bad inner loop.\n");
 
-      if (!expr_invariant_in_loop_p (loop, inner_niter))
+      if (!expr_invariant_in_loop_p (loop, inner.number_of_iterations))
 	return opt_result::failure_at (vect_location,
 				       "not vectorized: inner-loop count not"
 				       " invariant.\n");
@@ -1420,6 +1412,7 @@ vect_analyze_loop_form_1 (class loop *loop, gcond **loop_cond,
       if (dump_enabled_p ())
         dump_printf_loc (MSG_NOTE, vect_location,
 			 "Considering outer-loop vectorization.\n");
+      info->inner_loop_cond = inner.loop_cond;
     }
 
   if (!single_exit (loop))
@@ -1446,78 +1439,65 @@ vect_analyze_loop_form_1 (class loop *loop, gcond **loop_cond,
 				   "not vectorized:"
 				   " abnormal loop exit edge.\n");
 
-  *loop_cond = vect_get_loop_niters (loop, assumptions, number_of_iterations,
-				     number_of_iterationsm1);
-  if (!*loop_cond)
+  info->loop_cond
+    = vect_get_loop_niters (loop, &info->assumptions,
+			    &info->number_of_iterations,
+			    &info->number_of_iterationsm1);
+  if (!info->loop_cond)
     return opt_result::failure_at
       (vect_location,
        "not vectorized: complicated exit condition.\n");
 
-  if (integer_zerop (*assumptions)
-      || !*number_of_iterations
-      || chrec_contains_undetermined (*number_of_iterations))
+  if (integer_zerop (info->assumptions)
+      || !info->number_of_iterations
+      || chrec_contains_undetermined (info->number_of_iterations))
     return opt_result::failure_at
-      (*loop_cond,
+      (info->loop_cond,
        "not vectorized: number of iterations cannot be computed.\n");
 
-  if (integer_zerop (*number_of_iterations))
+  if (integer_zerop (info->number_of_iterations))
     return opt_result::failure_at
-      (*loop_cond,
+      (info->loop_cond,
        "not vectorized: number of iterations = 0.\n");
 
+  if (!(tree_fits_shwi_p (info->number_of_iterations)
+	&& tree_to_shwi (info->number_of_iterations) > 0))
+    {
+      if (dump_enabled_p ())
+	{
+	  dump_printf_loc (MSG_NOTE, vect_location,
+			   "Symbolic number of iterations is ");
+	  dump_generic_expr (MSG_NOTE, TDF_DETAILS, info->number_of_iterations);
+	  dump_printf (MSG_NOTE, "\n");
+	}
+    }
+
   return opt_result::success ();
 }
 
-/* Analyze LOOP form and return a loop_vec_info if it is of suitable form.  */
+/* Create a loop_vec_info for LOOP with SHARED and the
+   vect_analyze_loop_form result.  */
 
-opt_loop_vec_info
-vect_analyze_loop_form (class loop *loop, vec_info_shared *shared)
+loop_vec_info
+vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
+			const vect_loop_form_info *info,
+			loop_vec_info main_loop_info)
 {
-  tree assumptions, number_of_iterations, number_of_iterationsm1;
-  gcond *loop_cond, *inner_loop_cond = NULL;
-
-  opt_result res
-    = vect_analyze_loop_form_1 (loop, &loop_cond,
-				&assumptions, &number_of_iterationsm1,
-				&number_of_iterations, &inner_loop_cond);
-  if (!res)
-    return opt_loop_vec_info::propagate_failure (res);
-
   loop_vec_info loop_vinfo = new _loop_vec_info (loop, shared);
-  LOOP_VINFO_NITERSM1 (loop_vinfo) = number_of_iterationsm1;
-  LOOP_VINFO_NITERS (loop_vinfo) = number_of_iterations;
-  LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = number_of_iterations;
-  if (!integer_onep (assumptions))
-    {
-      /* We consider to vectorize this loop by versioning it under
-	 some assumptions.  In order to do this, we need to clear
-	 existing information computed by scev and niter analyzer.  */
-      scev_reset_htab ();
-      free_numbers_of_iterations_estimates (loop);
-      /* Also set flag for this loop so that following scev and niter
-	 analysis are done under the assumptions.  */
-      loop_constraint_set (loop, LOOP_C_FINITE);
-      /* Also record the assumptions for versioning.  */
-      LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo) = assumptions;
-    }
-
-  if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
-    {
-      if (dump_enabled_p ())
-        {
-          dump_printf_loc (MSG_NOTE, vect_location,
-			   "Symbolic number of iterations is ");
-	  dump_generic_expr (MSG_NOTE, TDF_DETAILS, number_of_iterations);
-          dump_printf (MSG_NOTE, "\n");
-        }
-    }
-
-  stmt_vec_info loop_cond_info = loop_vinfo->lookup_stmt (loop_cond);
+  LOOP_VINFO_NITERSM1 (loop_vinfo) = info->number_of_iterationsm1;
+  LOOP_VINFO_NITERS (loop_vinfo) = info->number_of_iterations;
+  LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = info->number_of_iterations;
+  LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo) = main_loop_info;
+  /* Also record the assumptions for versioning.  */
+  if (!integer_onep (info->assumptions) && !main_loop_info)
+    LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo) = info->assumptions;
+
+  stmt_vec_info loop_cond_info = loop_vinfo->lookup_stmt (info->loop_cond);
   STMT_VINFO_TYPE (loop_cond_info) = loop_exit_ctrl_vec_info_type;
-  if (inner_loop_cond)
+  if (info->inner_loop_cond)
     {
       stmt_vec_info inner_loop_cond_info
-	= loop_vinfo->lookup_stmt (inner_loop_cond);
+	= loop_vinfo->lookup_stmt (info->inner_loop_cond);
       STMT_VINFO_TYPE (inner_loop_cond_info) = loop_exit_ctrl_vec_info_type;
       /* If we have an estimate on the number of iterations of the inner
 	 loop use that to limit the scale for costing, otherwise use
@@ -1528,9 +1508,7 @@ vect_analyze_loop_form (class loop *loop, vec_info_shared *shared)
 	  = wi::smin (nit, param_vect_inner_loop_cost_factor).to_uhwi ();
     }
 
-  gcc_assert (!loop->aux);
-  loop->aux = loop_vinfo;
-  return opt_loop_vec_info::success (loop_vinfo);
+  return loop_vinfo;
 }
 
 
@@ -1783,7 +1761,7 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo)
         }
     } /* bbs */
 
-  add_stmt_costs (loop_vinfo, loop_vinfo->target_cost_data, &cost_vec);
+  add_stmt_costs (loop_vinfo->vector_costs, &cost_vec);
 
   /* All operations in the loop are either irrelevant (deal with loop
      control, or dead), or only used outside the loop and can be moved
@@ -2217,7 +2195,7 @@ vect_determine_partial_vectors_and_peeling (loop_vec_info loop_vinfo,
    for it.  The different analyses will record information in the
    loop_vec_info struct.  */
 static opt_result
-vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal, unsigned *n_stmts)
+vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal)
 {
   opt_result ok = opt_result::success ();
   int res;
@@ -2252,7 +2230,7 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal, unsigned *n_stmts)
       opt_result res
 	= vect_get_datarefs_in_loop (loop, LOOP_VINFO_BBS (loop_vinfo),
 				     &LOOP_VINFO_DATAREFS (loop_vinfo),
-				     n_stmts);
+				     &LOOP_VINFO_N_STMTS (loop_vinfo));
       if (!res)
 	{
 	  if (dump_enabled_p ())
@@ -2349,7 +2327,7 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal, unsigned *n_stmts)
   poly_uint64 saved_vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
 
   /* Check the SLP opportunities in the loop, analyze and build SLP trees.  */
-  ok = vect_analyze_slp (loop_vinfo, *n_stmts);
+  ok = vect_analyze_slp (loop_vinfo, LOOP_VINFO_N_STMTS (loop_vinfo));
   if (!ok)
     return ok;
 
@@ -2393,6 +2371,8 @@ start_over:
 		   LOOP_VINFO_INT_NITERS (loop_vinfo));
     }
 
+  loop_vinfo->vector_costs = init_cost (loop_vinfo, false);
+
   /* Analyze the alignment of the data-refs in the loop.
      Fail if a data reference is found that cannot be vectorized.  */
 
@@ -2647,6 +2627,7 @@ start_over:
 			LOOP_VINFO_VECT_FACTOR (loop_vinfo)));
 
   /* Ok to vectorize!  */
+  LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1;
   return opt_result::success ();
 
 again:
@@ -2757,9 +2738,8 @@ again:
   LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).release ();
   LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).release ();
   /* Reset target cost data.  */
-  destroy_cost_data (LOOP_VINFO_TARGET_COST_DATA (loop_vinfo));
-  LOOP_VINFO_TARGET_COST_DATA (loop_vinfo)
-    = init_cost (LOOP_VINFO_LOOP (loop_vinfo), false);
+  delete loop_vinfo->vector_costs;
+  loop_vinfo->vector_costs = nullptr;
   /* Reset accumulated rgroup information.  */
   release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo));
   release_vec_loop_controls (&LOOP_VINFO_LENS (loop_vinfo));
@@ -2800,84 +2780,12 @@ vect_better_loop_vinfo_p (loop_vec_info new_loop_vinfo,
 	return new_simdlen_p;
     }
 
-  /* Limit the VFs to what is likely to be the maximum number of iterations,
-     to handle cases in which at least one loop_vinfo is fully-masked.  */
-  HOST_WIDE_INT estimated_max_niter;
-  loop_vec_info main_loop = LOOP_VINFO_ORIG_LOOP_INFO (old_loop_vinfo);
-  unsigned HOST_WIDE_INT main_vf;
-  if (main_loop
-      && LOOP_VINFO_NITERS_KNOWN_P (main_loop)
-      && LOOP_VINFO_VECT_FACTOR (main_loop).is_constant (&main_vf))
-    estimated_max_niter = LOOP_VINFO_INT_NITERS (main_loop) % main_vf;
-  else
-    estimated_max_niter = likely_max_stmt_executions_int (loop);
-  if (estimated_max_niter != -1)
-    {
-      if (known_le (estimated_max_niter, new_vf))
-	new_vf = estimated_max_niter;
-      if (known_le (estimated_max_niter, old_vf))
-	old_vf = estimated_max_niter;
-    }
-
-  /* Check whether the (fractional) cost per scalar iteration is lower
-     or higher: new_inside_cost / new_vf vs. old_inside_cost / old_vf.  */
-  poly_int64 rel_new = new_loop_vinfo->vec_inside_cost * old_vf;
-  poly_int64 rel_old = old_loop_vinfo->vec_inside_cost * new_vf;
-
-  HOST_WIDE_INT est_rel_new_min
-    = estimated_poly_value (rel_new, POLY_VALUE_MIN);
-  HOST_WIDE_INT est_rel_new_max
-    = estimated_poly_value (rel_new, POLY_VALUE_MAX);
-
-  HOST_WIDE_INT est_rel_old_min
-    = estimated_poly_value (rel_old, POLY_VALUE_MIN);
-  HOST_WIDE_INT est_rel_old_max
-    = estimated_poly_value (rel_old, POLY_VALUE_MAX);
-
-  /* Check first if we can make out an unambigous total order from the minimum
-     and maximum estimates.  */
-  if (est_rel_new_min < est_rel_old_min
-      && est_rel_new_max < est_rel_old_max)
-    return true;
-  else if (est_rel_old_min < est_rel_new_min
-	   && est_rel_old_max < est_rel_new_max)
-    return false;
-  /* When old_loop_vinfo uses a variable vectorization factor,
-     we know that it has a lower cost for at least one runtime VF.
-     However, we don't know how likely that VF is.
-
-     One option would be to compare the costs for the estimated VFs.
-     The problem is that that can put too much pressure on the cost
-     model.  E.g. if the estimated VF is also the lowest possible VF,
-     and if old_loop_vinfo is 1 unit worse than new_loop_vinfo
-     for the estimated VF, we'd then choose new_loop_vinfo even
-     though (a) new_loop_vinfo might not actually be better than
-     old_loop_vinfo for that VF and (b) it would be significantly
-     worse at larger VFs.
-
-     Here we go for a hacky compromise: pick new_loop_vinfo if it is
-     no more expensive than old_loop_vinfo even after doubling the
-     estimated old_loop_vinfo VF.  For all but trivial loops, this
-     ensures that we only pick new_loop_vinfo if it is significantly
-     better than old_loop_vinfo at the estimated VF.  */
-
-  if (est_rel_old_min != est_rel_new_min
-      || est_rel_old_max != est_rel_new_max)
-    {
-      HOST_WIDE_INT est_rel_new_likely
-	= estimated_poly_value (rel_new, POLY_VALUE_LIKELY);
-      HOST_WIDE_INT est_rel_old_likely
-	= estimated_poly_value (rel_old, POLY_VALUE_LIKELY);
-
-      return est_rel_new_likely * 2 <= est_rel_old_likely;
-    }
-
-  /* If there's nothing to choose between the loop bodies, see whether
-     there's a difference in the prologue and epilogue costs.  */
-  if (new_loop_vinfo->vec_outside_cost != old_loop_vinfo->vec_outside_cost)
-    return new_loop_vinfo->vec_outside_cost < old_loop_vinfo->vec_outside_cost;
+  const auto *old_costs = old_loop_vinfo->vector_costs;
+  const auto *new_costs = new_loop_vinfo->vector_costs;
+  if (loop_vec_info main_loop = LOOP_VINFO_ORIG_LOOP_INFO (old_loop_vinfo))
+    return new_costs->better_epilogue_loop_than_p (old_costs, main_loop);
 
-  return false;
+  return new_costs->better_main_loop_than_p (old_costs);
 }
 
 /* Decide whether to replace OLD_LOOP_VINFO with NEW_LOOP_VINFO.  Return
@@ -2898,43 +2806,78 @@ vect_joust_loop_vinfos (loop_vec_info new_loop_vinfo,
   return true;
 }
 
-/* If LOOP_VINFO is already a main loop, return it unmodified.  Otherwise
-   try to reanalyze it as a main loop.  Return the loop_vinfo on success
-   and null on failure.  */
-
-static loop_vec_info
-vect_reanalyze_as_main_loop (loop_vec_info loop_vinfo, unsigned int *n_stmts)
+/* Analyze LOOP with VECTOR_MODES[MODE_I] and as epilogue if MAIN_LOOP_VINFO is
+   not NULL.  Set AUTODETECTED_VECTOR_MODE if VOIDmode and advance
+   MODE_I to the next mode useful to analyze.
+   Return the loop_vinfo on success and wrapped null on failure.  */
+
+static opt_loop_vec_info
+vect_analyze_loop_1 (class loop *loop, vec_info_shared *shared,
+		     const vect_loop_form_info *loop_form_info,
+		     loop_vec_info main_loop_vinfo,
+		     const vector_modes &vector_modes, unsigned &mode_i,
+		     machine_mode &autodetected_vector_mode,
+		     bool &fatal)
 {
-  if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo))
-    return loop_vinfo;
+  loop_vec_info loop_vinfo
+    = vect_create_loop_vinfo (loop, shared, loop_form_info, main_loop_vinfo);
+
+  machine_mode vector_mode = vector_modes[mode_i];
+  loop_vinfo->vector_mode = vector_mode;
 
+  /* Run the main analysis.  */
+  opt_result res = vect_analyze_loop_2 (loop_vinfo, fatal);
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location,
-		     "***** Reanalyzing as a main loop with vector mode %s\n",
+		     "***** Analysis %s with vector mode %s\n",
+		     res ? "succeeded" : " failed",
 		     GET_MODE_NAME (loop_vinfo->vector_mode));
 
-  struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-  vec_info_shared *shared = loop_vinfo->shared;
-  opt_loop_vec_info main_loop_vinfo = vect_analyze_loop_form (loop, shared);
-  gcc_assert (main_loop_vinfo);
+  /* Remember the autodetected vector mode.  */
+  if (vector_mode == VOIDmode)
+    autodetected_vector_mode = loop_vinfo->vector_mode;
 
-  main_loop_vinfo->vector_mode = loop_vinfo->vector_mode;
-
-  bool fatal = false;
-  bool res = vect_analyze_loop_2 (main_loop_vinfo, fatal, n_stmts);
-  loop->aux = NULL;
-  if (!res)
+  /* Advance mode_i, first skipping modes that would result in the
+     same analysis result.  */
+  while (mode_i + 1 < vector_modes.length ()
+	 && vect_chooses_same_modes_p (loop_vinfo,
+				       vector_modes[mode_i + 1]))
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_NOTE, vect_location,
-			 "***** Failed to analyze main loop with vector"
-			 " mode %s\n",
-			 GET_MODE_NAME (loop_vinfo->vector_mode));
-      delete main_loop_vinfo;
-      return NULL;
+			 "***** The result for vector mode %s would"
+			 " be the same\n",
+			 GET_MODE_NAME (vector_modes[mode_i + 1]));
+      mode_i += 1;
+    }
+  if (mode_i + 1 < vector_modes.length ()
+      && VECTOR_MODE_P (autodetected_vector_mode)
+      && (related_vector_mode (vector_modes[mode_i + 1],
+			       GET_MODE_INNER (autodetected_vector_mode))
+	  == autodetected_vector_mode)
+      && (related_vector_mode (autodetected_vector_mode,
+			       GET_MODE_INNER (vector_modes[mode_i + 1]))
+	  == vector_modes[mode_i + 1]))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_NOTE, vect_location,
+			 "***** Skipping vector mode %s, which would"
+			 " repeat the analysis for %s\n",
+			 GET_MODE_NAME (vector_modes[mode_i + 1]),
+			 GET_MODE_NAME (autodetected_vector_mode));
+      mode_i += 1;
     }
-  LOOP_VINFO_VECTORIZABLE_P (main_loop_vinfo) = 1;
-  return main_loop_vinfo;
+  mode_i++;
+
+  if (!res)
+    {
+      delete loop_vinfo;
+      if (fatal)
+	gcc_checking_assert (main_loop_vinfo == NULL);
+      return opt_loop_vec_info::propagate_failure (res);
+    }
+
+  return opt_loop_vec_info::success (loop_vinfo);
 }
 
 /* Function vect_analyze_loop.
@@ -2945,14 +2888,6 @@ vect_reanalyze_as_main_loop (loop_vec_info loop_vinfo, unsigned int *n_stmts)
 opt_loop_vec_info
 vect_analyze_loop (class loop *loop, vec_info_shared *shared)
 {
-  auto_vector_modes vector_modes;
-
-  /* Autodetect first vector size we try.  */
-  unsigned int autovec_flags
-    = targetm.vectorize.autovectorize_vector_modes (&vector_modes,
-						    loop->simdlen != 0);
-  unsigned int mode_i = 0;
-
   DUMP_VECT_SCOPE ("analyze_loop_nest");
 
   if (loop_outer (loop)
@@ -2967,99 +2902,60 @@ vect_analyze_loop (class loop *loop, vec_info_shared *shared)
        "not vectorized: loop nest containing two or more consecutive inner"
        " loops cannot be vectorized\n");
 
-  unsigned n_stmts = 0;
-  machine_mode autodetected_vector_mode = VOIDmode;
-  opt_loop_vec_info first_loop_vinfo = opt_loop_vec_info::success (NULL);
-  machine_mode next_vector_mode = VOIDmode;
-  poly_uint64 lowest_th = 0;
-  unsigned vectorized_loops = 0;
+  /* Analyze the loop form.  */
+  vect_loop_form_info loop_form_info;
+  opt_result res = vect_analyze_loop_form (loop, &loop_form_info);
+  if (!res)
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "bad loop form.\n");
+      return opt_loop_vec_info::propagate_failure (res);
+    }
+  if (!integer_onep (loop_form_info.assumptions))
+    {
+      /* We consider to vectorize this loop by versioning it under
+	 some assumptions.  In order to do this, we need to clear
+	 existing information computed by scev and niter analyzer.  */
+      scev_reset_htab ();
+      free_numbers_of_iterations_estimates (loop);
+      /* Also set flag for this loop so that following scev and niter
+	 analysis are done under the assumptions.  */
+      loop_constraint_set (loop, LOOP_C_FINITE);
+    }
+
+  auto_vector_modes vector_modes;
+  /* Autodetect first vector size we try.  */
+  vector_modes.safe_push (VOIDmode);
+  unsigned int autovec_flags
+    = targetm.vectorize.autovectorize_vector_modes (&vector_modes,
+						    loop->simdlen != 0);
   bool pick_lowest_cost_p = ((autovec_flags & VECT_COMPARE_COSTS)
 			     && !unlimited_cost_model (loop));
-
-  bool vect_epilogues = false;
-  opt_result res = opt_result::success ();
+  machine_mode autodetected_vector_mode = VOIDmode;
+  opt_loop_vec_info first_loop_vinfo = opt_loop_vec_info::success (NULL);
+  unsigned int mode_i = 0;
+  unsigned int first_loop_i = 0;
+  unsigned int first_loop_next_i = 0;
   unsigned HOST_WIDE_INT simdlen = loop->simdlen;
+
+  /* First determine the main loop vectorization mode, either the first
+     one that works, starting with auto-detecting the vector mode and then
+     following the targets order of preference, or the one with the
+     lowest cost if pick_lowest_cost_p.  */
   while (1)
     {
-      /* Check the CFG characteristics of the loop (nesting, entry/exit).  */
-      opt_loop_vec_info loop_vinfo = vect_analyze_loop_form (loop, shared);
-      if (!loop_vinfo)
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "bad loop form.\n");
-	  gcc_checking_assert (first_loop_vinfo == NULL);
-	  return loop_vinfo;
-	}
-      loop_vinfo->vector_mode = next_vector_mode;
-
-      bool fatal = false;
-
-      /* When pick_lowest_cost_p is true, we should in principle iterate
-	 over all the loop_vec_infos that LOOP_VINFO could replace and
-	 try to vectorize LOOP_VINFO under the same conditions.
-	 E.g. when trying to replace an epilogue loop, we should vectorize
-	 LOOP_VINFO as an epilogue loop with the same VF limit.  When trying
-	 to replace the main loop, we should vectorize LOOP_VINFO as a main
-	 loop too.
-
-	 However, autovectorize_vector_modes is usually sorted as follows:
-
-	 - Modes that naturally produce lower VFs usually follow modes that
-	   naturally produce higher VFs.
-
-	 - When modes naturally produce the same VF, maskable modes
-	   usually follow unmaskable ones, so that the maskable mode
-	   can be used to vectorize the epilogue of the unmaskable mode.
-
-	 This order is preferred because it leads to the maximum
-	 epilogue vectorization opportunities.  Targets should only use
-	 a different order if they want to make wide modes available while
-	 disparaging them relative to earlier, smaller modes.  The assumption
-	 in that case is that the wider modes are more expensive in some
-	 way that isn't reflected directly in the costs.
-
-	 There should therefore be few interesting cases in which
-	 LOOP_VINFO fails when treated as an epilogue loop, succeeds when
-	 treated as a standalone loop, and ends up being genuinely cheaper
-	 than FIRST_LOOP_VINFO.  */
-      if (vect_epilogues)
-	LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo) = first_loop_vinfo;
-
-      res = vect_analyze_loop_2 (loop_vinfo, fatal, &n_stmts);
-      if (mode_i == 0)
-	autodetected_vector_mode = loop_vinfo->vector_mode;
-      if (dump_enabled_p ())
-	{
-	  if (res)
-	    dump_printf_loc (MSG_NOTE, vect_location,
-			     "***** Analysis succeeded with vector mode %s\n",
-			     GET_MODE_NAME (loop_vinfo->vector_mode));
-	  else
-	    dump_printf_loc (MSG_NOTE, vect_location,
-			     "***** Analysis failed with vector mode %s\n",
-			     GET_MODE_NAME (loop_vinfo->vector_mode));
-	}
-
-      loop->aux = NULL;
-
-      if (!fatal)
-	while (mode_i < vector_modes.length ()
-	       && vect_chooses_same_modes_p (loop_vinfo, vector_modes[mode_i]))
-	  {
-	    if (dump_enabled_p ())
-	      dump_printf_loc (MSG_NOTE, vect_location,
-			       "***** The result for vector mode %s would"
-			       " be the same\n",
-			       GET_MODE_NAME (vector_modes[mode_i]));
-	    mode_i += 1;
-	  }
+      unsigned int loop_vinfo_i = mode_i;
+      bool fatal;
+      opt_loop_vec_info loop_vinfo
+	= vect_analyze_loop_1 (loop, shared, &loop_form_info,
+			       NULL, vector_modes, mode_i,
+			       autodetected_vector_mode, fatal);
+      if (fatal)
+	break;
 
-      if (res)
+      if (loop_vinfo)
 	{
-	  LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1;
-	  vectorized_loops++;
-
 	  /* Once we hit the desired simdlen for the first time,
 	     discard any previous attempts.  */
 	  if (simdlen
@@ -3067,10 +2963,105 @@ vect_analyze_loop (class loop *loop, vec_info_shared *shared)
 	    {
 	      delete first_loop_vinfo;
 	      first_loop_vinfo = opt_loop_vec_info::success (NULL);
-	      LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo) = NULL;
 	      simdlen = 0;
 	    }
-	  else if (pick_lowest_cost_p && first_loop_vinfo)
+	  else if (pick_lowest_cost_p
+		   && first_loop_vinfo
+		   && vect_joust_loop_vinfos (loop_vinfo, first_loop_vinfo))
+	    {
+	      /* Pick loop_vinfo over first_loop_vinfo.  */
+	      delete first_loop_vinfo;
+	      first_loop_vinfo = opt_loop_vec_info::success (NULL);
+	    }
+	  if (first_loop_vinfo == NULL)
+	    {
+	      first_loop_vinfo = loop_vinfo;
+	      first_loop_i = loop_vinfo_i;
+	      first_loop_next_i = mode_i;
+	    }
+	  else
+	    {
+	      delete loop_vinfo;
+	      loop_vinfo = opt_loop_vec_info::success (NULL);
+	    }
+
+	  /* Commit to first_loop_vinfo if we have no reason to try
+	     alternatives.  */
+	  if (!simdlen && !pick_lowest_cost_p)
+	    break;
+	}
+      if (mode_i == vector_modes.length ()
+	  || autodetected_vector_mode == VOIDmode)
+	break;
+
+      /* Try the next biggest vector size.  */
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_NOTE, vect_location,
+			 "***** Re-trying analysis with vector mode %s\n",
+			 GET_MODE_NAME (vector_modes[mode_i]));
+    }
+  if (!first_loop_vinfo)
+    return opt_loop_vec_info::propagate_failure (res);
+
+  if (dump_enabled_p ())
+    dump_printf_loc (MSG_NOTE, vect_location,
+		     "***** Choosing vector mode %s\n",
+		     GET_MODE_NAME (first_loop_vinfo->vector_mode));
+
+  /* Only vectorize epilogues if PARAM_VECT_EPILOGUES_NOMASK is
+     enabled, SIMDUID is not set, it is the innermost loop and we have
+     either already found the loop's SIMDLEN or there was no SIMDLEN to
+     begin with.
+     TODO: Enable epilogue vectorization for loops with SIMDUID set.  */
+  bool vect_epilogues = (!simdlen
+			 && loop->inner == NULL
+			 && param_vect_epilogues_nomask
+			 && LOOP_VINFO_PEELING_FOR_NITER (first_loop_vinfo)
+			 && !loop->simduid);
+  if (!vect_epilogues)
+    return first_loop_vinfo;
+
+  /* Now analyze first_loop_vinfo for epilogue vectorization.  */
+  poly_uint64 lowest_th = LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo);
+
+  /* Handle the case that the original loop can use partial
+     vectorization, but want to only adopt it for the epilogue.
+     The retry should be in the same mode as original.  */
+  if (LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P (first_loop_vinfo))
+    {
+      gcc_assert (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (first_loop_vinfo)
+		  && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (first_loop_vinfo));
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_NOTE, vect_location,
+			 "***** Re-trying analysis with same vector mode"
+			 " %s for epilogue with partial vectors.\n",
+			 GET_MODE_NAME (first_loop_vinfo->vector_mode));
+      mode_i = first_loop_i;
+    }
+  else
+    {
+      mode_i = first_loop_next_i;
+      if (mode_i == vector_modes.length ())
+	return first_loop_vinfo;
+    }
+
+  /* ???  If first_loop_vinfo was using VOIDmode then we probably
+     want to instead search for the corresponding mode in vector_modes[].  */
+
+  while (1)
+    {
+      bool fatal;
+      opt_loop_vec_info loop_vinfo
+	= vect_analyze_loop_1 (loop, shared, &loop_form_info,
+			       first_loop_vinfo,
+			       vector_modes, mode_i,
+			       autodetected_vector_mode, fatal);
+      if (fatal)
+	break;
+
+      if (loop_vinfo)
+	{
+	  if (pick_lowest_cost_p)
 	    {
 	      /* Keep trying to roll back vectorization attempts while the
 		 loop_vec_infos they produced were worse than this one.  */
@@ -3081,48 +3072,9 @@ vect_analyze_loop (class loop *loop, vec_info_shared *shared)
 		  gcc_assert (vect_epilogues);
 		  delete vinfos.pop ();
 		}
-	      if (vinfos.is_empty ()
-		  && vect_joust_loop_vinfos (loop_vinfo, first_loop_vinfo))
-		{
-		  loop_vec_info main_loop_vinfo
-		    = vect_reanalyze_as_main_loop (loop_vinfo, &n_stmts);
-		  if (main_loop_vinfo == loop_vinfo)
-		    {
-		      delete first_loop_vinfo;
-		      first_loop_vinfo = opt_loop_vec_info::success (NULL);
-		    }
-		  else if (main_loop_vinfo
-			   && vect_joust_loop_vinfos (main_loop_vinfo,
-						      first_loop_vinfo))
-		    {
-		      delete first_loop_vinfo;
-		      first_loop_vinfo = opt_loop_vec_info::success (NULL);
-		      delete loop_vinfo;
-		      loop_vinfo
-			= opt_loop_vec_info::success (main_loop_vinfo);
-		    }
-		  else
-		    {
-		      if (dump_enabled_p ())
-			dump_printf_loc (MSG_NOTE, vect_location,
-					 "***** No longer preferring vector"
-					 " mode %s after reanalyzing the loop"
-					 " as a main loop\n",
-					 GET_MODE_NAME
-					   (main_loop_vinfo->vector_mode));
-		      delete main_loop_vinfo;
-		    }
-		}
-	    }
-
-	  if (first_loop_vinfo == NULL)
-	    {
-	      first_loop_vinfo = loop_vinfo;
-	      lowest_th = LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo);
 	    }
-	  else if (vect_epilogues
-		   /* For now only allow one epilogue loop.  */
-		   && first_loop_vinfo->epilogue_vinfos.is_empty ())
+	  /* For now only allow one epilogue loop.  */
+	  if (first_loop_vinfo->epilogue_vinfos.is_empty ())
 	    {
 	      first_loop_vinfo->epilogue_vinfos.safe_push (loop_vinfo);
 	      poly_uint64 th = LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo);
@@ -3139,96 +3091,34 @@ vect_analyze_loop (class loop *loop, vec_info_shared *shared)
 	      loop_vinfo = opt_loop_vec_info::success (NULL);
 	    }
 
-	  /* Only vectorize epilogues if PARAM_VECT_EPILOGUES_NOMASK is
-	     enabled, SIMDUID is not set, it is the innermost loop and we have
-	     either already found the loop's SIMDLEN or there was no SIMDLEN to
-	     begin with.
-	     TODO: Enable epilogue vectorization for loops with SIMDUID set.  */
-	  vect_epilogues = (!simdlen
-			    && loop->inner == NULL
-			    && param_vect_epilogues_nomask
-			    && LOOP_VINFO_PEELING_FOR_NITER (first_loop_vinfo)
-			    && !loop->simduid
-			    /* For now only allow one epilogue loop, but allow
-			       pick_lowest_cost_p to replace it.  */
-			    && (first_loop_vinfo->epilogue_vinfos.is_empty ()
-				|| pick_lowest_cost_p));
-
-	  /* Commit to first_loop_vinfo if we have no reason to try
-	     alternatives.  */
-	  if (!simdlen && !vect_epilogues && !pick_lowest_cost_p)
+	  /* For now only allow one epilogue loop, but allow
+	     pick_lowest_cost_p to replace it, so commit to the
+	     first epilogue if we have no reason to try alternatives.  */
+	  if (!pick_lowest_cost_p)
 	    break;
 	}
-      else
-	{
-	  delete loop_vinfo;
-	  loop_vinfo = opt_loop_vec_info::success (NULL);
-	  if (fatal)
-	    {
-	      gcc_checking_assert (first_loop_vinfo == NULL);
-	      break;
-	    }
-	}
 
-      /* Handle the case that the original loop can use partial
-	 vectorization, but want to only adopt it for the epilogue.
-	 The retry should be in the same mode as original.  */
-      if (vect_epilogues
-	  && loop_vinfo
-	  && LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P (loop_vinfo))
-	{
-	  gcc_assert (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
-		      && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo));
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_NOTE, vect_location,
-			     "***** Re-trying analysis with same vector mode"
-			     " %s for epilogue with partial vectors.\n",
-			     GET_MODE_NAME (loop_vinfo->vector_mode));
-	  continue;
-	}
-
-      if (mode_i < vector_modes.length ()
-	  && VECTOR_MODE_P (autodetected_vector_mode)
-	  && (related_vector_mode (vector_modes[mode_i],
-				   GET_MODE_INNER (autodetected_vector_mode))
-	      == autodetected_vector_mode)
-	  && (related_vector_mode (autodetected_vector_mode,
-				   GET_MODE_INNER (vector_modes[mode_i]))
-	      == vector_modes[mode_i]))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_NOTE, vect_location,
-			     "***** Skipping vector mode %s, which would"
-			     " repeat the analysis for %s\n",
-			     GET_MODE_NAME (vector_modes[mode_i]),
-			     GET_MODE_NAME (autodetected_vector_mode));
-	  mode_i += 1;
-	}
-
-      if (mode_i == vector_modes.length ()
-	  || autodetected_vector_mode == VOIDmode)
+      if (mode_i == vector_modes.length ())
 	break;
 
       /* Try the next biggest vector size.  */
-      next_vector_mode = vector_modes[mode_i++];
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_NOTE, vect_location,
-			 "***** Re-trying analysis with vector mode %s\n",
-			 GET_MODE_NAME (next_vector_mode));
+			 "***** Re-trying epilogue analysis with vector "
+			 "mode %s\n", GET_MODE_NAME (vector_modes[mode_i]));
     }
 
-  if (first_loop_vinfo)
+  if (!first_loop_vinfo->epilogue_vinfos.is_empty ())
     {
-      loop->aux = (loop_vec_info) first_loop_vinfo;
+      LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo) = lowest_th;
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_NOTE, vect_location,
-			 "***** Choosing vector mode %s\n",
-			 GET_MODE_NAME (first_loop_vinfo->vector_mode));
-      LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo) = lowest_th;
-      return first_loop_vinfo;
+			 "***** Choosing epilogue vector mode %s\n",
+			 GET_MODE_NAME
+			   (first_loop_vinfo->epilogue_vinfos[0]->vector_mode));
     }
 
-  return opt_loop_vec_info::propagate_failure (res);
+  return first_loop_vinfo;
 }
 
 /* Return true if there is an in-order reduction function for CODE, storing
@@ -3306,7 +3196,7 @@ reduction_fn_for_scalar_code (enum tree_code code, internal_fn *reduc_fn)
    of the scalar elements.  If the reduction has just a single initial value
    then INITIAL_VALUE is that value, otherwise it is null.  */
 
-static tree
+tree
 neutral_op_for_reduction (tree scalar_type, tree_code code, tree initial_value)
 {
   switch (code)
@@ -3677,6 +3567,17 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info,
       return def_stmt_info;
     }
 
+  /* When the inner loop of a double reduction ends up with more than
+     one loop-closed PHI we have failed to classify alternate such
+     PHIs as double reduction, leading to wrong code.  See PR103237.  */
+  if (inner_loop_of_double_reduc && lcphis.length () != 1)
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "unhandle double reduction\n");
+      return NULL;
+    }
+
   /* If this isn't a nested cycle or if the nested cycle reduction value
      is used ouside of the inner loop we cannot handle uses of the reduction
      value.  */
@@ -3895,7 +3796,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
   int scalar_outside_cost = 0;
   int assumed_vf = vect_vf_for_cost (loop_vinfo);
   int npeel = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
-  void *target_cost_data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo);
+  vector_costs *target_cost_data = loop_vinfo->vector_costs;
 
   /* Cost model disabled.  */
   if (unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
@@ -3912,7 +3813,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
     {
       /*  FIXME: Make cost depend on complexity of individual check.  */
       unsigned len = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ();
-      (void) add_stmt_cost (loop_vinfo, target_cost_data, len, vector_stmt,
+      (void) add_stmt_cost (target_cost_data, len, vector_stmt,
 			    NULL, NULL_TREE, 0, vect_prologue);
       if (dump_enabled_p ())
 	dump_printf (MSG_NOTE,
@@ -3925,12 +3826,12 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
     {
       /*  FIXME: Make cost depend on complexity of individual check.  */
       unsigned len = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo).length ();
-      (void) add_stmt_cost (loop_vinfo, target_cost_data, len, vector_stmt,
+      (void) add_stmt_cost (target_cost_data, len, vector_stmt,
 			    NULL, NULL_TREE, 0, vect_prologue);
       len = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).length ();
       if (len)
 	/* Count LEN - 1 ANDs and LEN comparisons.  */
-	(void) add_stmt_cost (loop_vinfo, target_cost_data, len * 2 - 1,
+	(void) add_stmt_cost (target_cost_data, len * 2 - 1,
 			      scalar_stmt, NULL, NULL_TREE, 0, vect_prologue);
       len = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).length ();
       if (len)
@@ -3941,7 +3842,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
 	  for (unsigned int i = 0; i < len; ++i)
 	    if (!LOOP_VINFO_LOWER_BOUNDS (loop_vinfo)[i].unsigned_p)
 	      nstmts += 1;
-	  (void) add_stmt_cost (loop_vinfo, target_cost_data, nstmts,
+	  (void) add_stmt_cost (target_cost_data, nstmts,
 				scalar_stmt, NULL, NULL_TREE, 0, vect_prologue);
 	}
       if (dump_enabled_p ())
@@ -3954,7 +3855,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
   if (LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo))
     {
       /*  FIXME: Make cost depend on complexity of individual check.  */
-      (void) add_stmt_cost (loop_vinfo, target_cost_data, 1, vector_stmt,
+      (void) add_stmt_cost (target_cost_data, 1, vector_stmt,
 			    NULL, NULL_TREE, 0, vect_prologue);
       if (dump_enabled_p ())
 	dump_printf (MSG_NOTE,
@@ -3963,7 +3864,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
     }
 
   if (LOOP_REQUIRES_VERSIONING (loop_vinfo))
-    (void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
+    (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
 			  NULL, NULL_TREE, 0, vect_prologue);
 
   /* Count statements in scalar loop.  Using this as scalar cost for a single
@@ -3974,8 +3875,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
      TODO: Consider assigning different costs to different scalar
      statements.  */
 
-  scalar_single_iter_cost
-    = LOOP_VINFO_SINGLE_SCALAR_ITERATION_COST (loop_vinfo);
+  scalar_single_iter_cost = loop_vinfo->scalar_costs->total_cost ();
 
   /* Add additional cost for the peeled instructions in prologue and epilogue
      loop.  (For fully-masked loops there will be no peeling.)
@@ -4051,7 +3951,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
   if (peel_iters_prologue)
     FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
       {
-	(void) add_stmt_cost (loop_vinfo, target_cost_data,
+	(void) add_stmt_cost (target_cost_data,
 			      si->count * peel_iters_prologue, si->kind,
 			      si->stmt_info, si->vectype, si->misalign,
 			      vect_prologue);
@@ -4061,7 +3961,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
   if (peel_iters_epilogue)
     FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo), j, si)
       {
-	(void) add_stmt_cost (loop_vinfo, target_cost_data,
+	(void) add_stmt_cost (target_cost_data,
 			      si->count * peel_iters_epilogue, si->kind,
 			      si->stmt_info, si->vectype, si->misalign,
 			      vect_epilogue);
@@ -4070,20 +3970,20 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
   /* Add possible cond_branch_taken/cond_branch_not_taken cost.  */
 
   if (prologue_need_br_taken_cost)
-    (void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
+    (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
 			  NULL, NULL_TREE, 0, vect_prologue);
 
   if (prologue_need_br_not_taken_cost)
-    (void) add_stmt_cost (loop_vinfo, target_cost_data, 1,
+    (void) add_stmt_cost (target_cost_data, 1,
 			  cond_branch_not_taken, NULL, NULL_TREE, 0,
 			  vect_prologue);
 
   if (epilogue_need_br_taken_cost)
-    (void) add_stmt_cost (loop_vinfo, target_cost_data, 1, cond_branch_taken,
+    (void) add_stmt_cost (target_cost_data, 1, cond_branch_taken,
 			  NULL, NULL_TREE, 0, vect_epilogue);
 
   if (epilogue_need_br_not_taken_cost)
-    (void) add_stmt_cost (loop_vinfo, target_cost_data, 1,
+    (void) add_stmt_cost (target_cost_data, 1,
 			  cond_branch_not_taken, NULL, NULL_TREE, 0,
 			  vect_epilogue);
 
@@ -4111,9 +4011,9 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
 	 simpler and safer to use the worst-case cost; if this ends up
 	 being the tie-breaker between vectorizing or not, then it's
 	 probably better not to vectorize.  */
-      (void) add_stmt_cost (loop_vinfo, target_cost_data, num_masks,
+      (void) add_stmt_cost (target_cost_data, num_masks,
 			    vector_stmt, NULL, NULL_TREE, 0, vect_prologue);
-      (void) add_stmt_cost (loop_vinfo, target_cost_data, num_masks - 1,
+      (void) add_stmt_cost (target_cost_data, num_masks - 1,
 			    vector_stmt, NULL, NULL_TREE, 0, vect_body);
     }
   else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
@@ -4163,9 +4063,9 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
 	      body_stmts += 3 * num_vectors;
 	  }
 
-      (void) add_stmt_cost (loop_vinfo, target_cost_data, prologue_stmts,
+      (void) add_stmt_cost (target_cost_data, prologue_stmts,
 			    scalar_stmt, NULL, NULL_TREE, 0, vect_prologue);
-      (void) add_stmt_cost (loop_vinfo, target_cost_data, body_stmts,
+      (void) add_stmt_cost (target_cost_data, body_stmts,
 			    scalar_stmt, NULL, NULL_TREE, 0, vect_body);
     }
 
@@ -4241,15 +4141,11 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
     }
 
   /* Complete the target-specific cost calculations.  */
-  finish_cost (LOOP_VINFO_TARGET_COST_DATA (loop_vinfo), &vec_prologue_cost,
-	       &vec_inside_cost, &vec_epilogue_cost);
+  finish_cost (loop_vinfo->vector_costs, loop_vinfo->scalar_costs,
+	       &vec_prologue_cost, &vec_inside_cost, &vec_epilogue_cost);
 
   vec_outside_cost = (int)(vec_prologue_cost + vec_epilogue_cost);
 
-  /* Stash the costs so that we can compare two loop_vec_infos.  */
-  loop_vinfo->vec_inside_cost = vec_inside_cost;
-  loop_vinfo->vec_outside_cost = vec_outside_cost;
-
   if (dump_enabled_p ())
     {
       dump_printf_loc (MSG_NOTE, vect_location, "Cost model analysis: \n");
diff --git a/gcc/tree-vect-slp-patterns.c b/gcc/tree-vect-slp-patterns.c
index b8d09b7..53fbe51 100644
--- a/gcc/tree-vect-slp-patterns.c
+++ b/gcc/tree-vect-slp-patterns.c
@@ -306,24 +306,6 @@ vect_match_expression_p (slp_tree node, tree_code code)
   return true;
 }
 
-/* Checks to see if the expression represented by NODE is a call to the internal
-   function FN.  */
-
-static inline bool
-vect_match_call_p (slp_tree node, internal_fn fn)
-{
-  if (!node
-      || !SLP_TREE_REPRESENTATIVE (node))
-    return false;
-
-  gimple* expr = STMT_VINFO_STMT (SLP_TREE_REPRESENTATIVE (node));
-  if (!expr
-      || !gimple_call_internal_p (expr, fn))
-    return false;
-
-   return true;
-}
-
 /* Check if the given lane permute in PERMUTES matches an alternating sequence
    of {even odd even odd ...}.  This to account for unrolled loops.  Further
    mode there resulting permute must be linear.   */
@@ -389,6 +371,16 @@ vect_detect_pair_op (slp_tree node1, slp_tree node2, lane_permutation_t &lanes,
 
   if (result != CMPLX_NONE && ops != NULL)
     {
+      if (two_operands)
+	{
+	  auto l0node = SLP_TREE_CHILDREN (node1);
+	  auto l1node = SLP_TREE_CHILDREN (node2);
+
+	  /* Check if the tree is connected as we expect it.  */
+	  if (!((l0node[0] == l1node[0] && l0node[1] == l1node[1])
+	      || (l0node[0] == l1node[1] && l0node[1] == l1node[0])))
+	    return CMPLX_NONE;
+	}
       ops->safe_push (node1);
       ops->safe_push (node2);
     }
@@ -717,27 +709,6 @@ complex_add_pattern::recognize (slp_tree_to_load_perm_map_t *perm_cache,
  * complex_mul_pattern
  ******************************************************************************/
 
-/* Helper function of that looks for a match in the CHILDth child of NODE.  The
-   child used is stored in RES.
-
-   If the match is successful then ARGS will contain the operands matched
-   and the complex_operation_t type is returned.  If match is not successful
-   then CMPLX_NONE is returned and ARGS is left unmodified.  */
-
-static inline complex_operation_t
-vect_match_call_complex_mla (slp_tree node, unsigned child,
-			     vec<slp_tree> *args = NULL, slp_tree *res = NULL)
-{
-  gcc_assert (child < SLP_TREE_CHILDREN (node).length ());
-
-  slp_tree data = SLP_TREE_CHILDREN (node)[child];
-
-  if (res)
-    *res = data;
-
-  return vect_detect_pair_op (data, false, args);
-}
-
 /* Check to see if either of the trees in ARGS are a NEGATE_EXPR.  If the first
    child (args[0]) is a NEGATE_EXPR then NEG_FIRST_P is set to TRUE.
 
@@ -945,9 +916,10 @@ class complex_mul_pattern : public complex_pattern
 
 };
 
-/* Pattern matcher for trying to match complex multiply pattern in SLP tree
-   If the operation matches then IFN is set to the operation it matched
-   and the arguments to the two replacement statements are put in m_ops.
+/* Pattern matcher for trying to match complex multiply and complex multiply
+   and accumulate pattern in SLP tree.  If the operation matches then IFN
+   is set to the operation it matched and the arguments to the two
+   replacement statements are put in m_ops.
 
    If no match is found then IFN is set to IFN_LAST and m_ops is unchanged.
 
@@ -972,19 +944,42 @@ complex_mul_pattern::matches (complex_operation_t op,
   if (op != MINUS_PLUS)
     return IFN_LAST;
 
-  slp_tree root = *node;
-  /* First two nodes must be a multiply.  */
-  auto_vec<slp_tree> muls;
-  if (vect_match_call_complex_mla (root, 0) != MULT_MULT
-      || vect_match_call_complex_mla (root, 1, &muls) != MULT_MULT)
+  auto childs = *ops;
+  auto l0node = SLP_TREE_CHILDREN (childs[0]);
+
+  bool mul0 = vect_match_expression_p (l0node[0], MULT_EXPR);
+  bool mul1 = vect_match_expression_p (l0node[1], MULT_EXPR);
+  if (!mul0 && !mul1)
     return IFN_LAST;
 
   /* Now operand2+4 may lead to another expression.  */
   auto_vec<slp_tree> left_op, right_op;
-  left_op.safe_splice (SLP_TREE_CHILDREN (muls[0]));
-  right_op.safe_splice (SLP_TREE_CHILDREN (muls[1]));
+  slp_tree add0 = NULL;
+
+  /* Check if we may be a multiply add.  */
+  if (!mul0
+      && vect_match_expression_p (l0node[0], PLUS_EXPR))
+    {
+      auto vals = SLP_TREE_CHILDREN (l0node[0]);
+      /* Check if it's a multiply, otherwise no idea what this is.  */
+      if (!vect_match_expression_p (vals[1], MULT_EXPR))
+	return IFN_LAST;
+
+      /* Check if the ADD is linear, otherwise it's not valid complex FMA.  */
+      if (linear_loads_p (perm_cache, vals[0]) != PERM_EVENODD)
+	return IFN_LAST;
 
-  if (linear_loads_p (perm_cache, left_op[1]) == PERM_ODDEVEN)
+      left_op.safe_splice (SLP_TREE_CHILDREN (vals[1]));
+      add0 = vals[0];
+    }
+  else
+    left_op.safe_splice (SLP_TREE_CHILDREN (l0node[0]));
+
+  right_op.safe_splice (SLP_TREE_CHILDREN (l0node[1]));
+
+  if (left_op.length () != 2
+      || right_op.length () != 2
+      || linear_loads_p (perm_cache, left_op[1]) == PERM_ODDEVEN)
     return IFN_LAST;
 
   bool neg_first = false;
@@ -998,23 +993,32 @@ complex_mul_pattern::matches (complex_operation_t op,
       if (!vect_validate_multiplication (perm_cache, left_op, PERM_EVENEVEN)
 	  || vect_normalize_conj_loc (left_op))
 	return IFN_LAST;
-      ifn = IFN_COMPLEX_MUL;
+      if (!mul0)
+	ifn = IFN_COMPLEX_FMA;
+      else
+	ifn = IFN_COMPLEX_MUL;
     }
-  else if (is_neg)
+  else
     {
       if (!vect_validate_multiplication (perm_cache, left_op, right_op,
 					 neg_first, &conj_first_operand,
 					 false))
 	return IFN_LAST;
 
-      ifn = IFN_COMPLEX_MUL_CONJ;
+      if(!mul0)
+	ifn = IFN_COMPLEX_FMA_CONJ;
+      else
+	ifn = IFN_COMPLEX_MUL_CONJ;
     }
 
   if (!vect_pattern_validate_optab (ifn, *node))
     return IFN_LAST;
 
   ops->truncate (0);
-  ops->create (3);
+  ops->create (add0 ? 4 : 3);
+
+  if (add0)
+    ops->quick_push (add0);
 
   complex_perm_kinds_t kind = linear_loads_p (perm_cache, left_op[0]);
   if (kind == PERM_EVENODD)
@@ -1070,171 +1074,56 @@ complex_mul_pattern::build (vec_info *vinfo)
 {
   slp_tree node;
   unsigned i;
-  slp_tree newnode
-    = vect_build_combine_node (this->m_ops[0], this->m_ops[1], *this->m_node);
-  SLP_TREE_REF_COUNT (this->m_ops[2])++;
-
-  FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (*this->m_node), i, node)
-    vect_free_slp_tree (node);
-
-  /* First re-arrange the children.  */
-  SLP_TREE_CHILDREN (*this->m_node).reserve_exact (2);
-  SLP_TREE_CHILDREN (*this->m_node)[0] = this->m_ops[2];
-  SLP_TREE_CHILDREN (*this->m_node)[1] = newnode;
+  switch (this->m_ifn)
+  {
+    case IFN_COMPLEX_MUL:
+    case IFN_COMPLEX_MUL_CONJ:
+      {
+	slp_tree newnode
+	  = vect_build_combine_node (this->m_ops[0], this->m_ops[1],
+				     *this->m_node);
+	SLP_TREE_REF_COUNT (this->m_ops[2])++;
+
+	FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (*this->m_node), i, node)
+	  vect_free_slp_tree (node);
+
+	/* First re-arrange the children.  */
+	SLP_TREE_CHILDREN (*this->m_node).reserve_exact (2);
+	SLP_TREE_CHILDREN (*this->m_node)[0] = this->m_ops[2];
+	SLP_TREE_CHILDREN (*this->m_node)[1] = newnode;
+	break;
+      }
+    case IFN_COMPLEX_FMA:
+    case IFN_COMPLEX_FMA_CONJ:
+      {
+	SLP_TREE_REF_COUNT (this->m_ops[0])++;
+	slp_tree newnode
+	  = vect_build_combine_node (this->m_ops[1], this->m_ops[2],
+				     *this->m_node);
+	SLP_TREE_REF_COUNT (this->m_ops[3])++;
+
+	FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (*this->m_node), i, node)
+	  vect_free_slp_tree (node);
+
+	/* First re-arrange the children.  */
+	SLP_TREE_CHILDREN (*this->m_node).safe_grow (3);
+	SLP_TREE_CHILDREN (*this->m_node)[0] = this->m_ops[0];
+	SLP_TREE_CHILDREN (*this->m_node)[1] = this->m_ops[3];
+	SLP_TREE_CHILDREN (*this->m_node)[2] = newnode;
+
+	/* Tell the builder to expect an extra argument.  */
+	this->m_num_args++;
+	break;
+      }
+    default:
+      gcc_unreachable ();
+  }
 
   /* And then rewrite the node itself.  */
   complex_pattern::build (vinfo);
 }
 
 /*******************************************************************************
- * complex_fma_pattern class
- ******************************************************************************/
-
-class complex_fma_pattern : public complex_pattern
-{
-  protected:
-    complex_fma_pattern (slp_tree *node, vec<slp_tree> *m_ops, internal_fn ifn)
-      : complex_pattern (node, m_ops, ifn)
-    {
-      this->m_num_args = 3;
-    }
-
-  public:
-    void build (vec_info *);
-    static internal_fn
-    matches (complex_operation_t op, slp_tree_to_load_perm_map_t *, slp_tree *,
-	     vec<slp_tree> *);
-
-    static vect_pattern*
-    recognize (slp_tree_to_load_perm_map_t *, slp_tree *);
-
-    static vect_pattern*
-    mkInstance (slp_tree *node, vec<slp_tree> *m_ops, internal_fn ifn)
-    {
-      return new complex_fma_pattern (node, m_ops, ifn);
-    }
-};
-
-/* Pattern matcher for trying to match complex multiply and accumulate
-   and multiply and subtract patterns in SLP tree.
-   If the operation matches then IFN is set to the operation it matched and
-   the arguments to the two replacement statements are put in m_ops.
-
-   If no match is found then IFN is set to IFN_LAST and m_ops is unchanged.
-
-   This function matches the patterns shaped as:
-
-   double ax = (b[i+1] * a[i]) + (b[i] * a[i]);
-   double bx = (a[i+1] * b[i]) - (a[i+1] * b[i+1]);
-
-   c[i] = c[i] - ax;
-   c[i+1] = c[i+1] + bx;
-
-   If a match occurred then TRUE is returned, else FALSE.  The match is
-   performed after COMPLEX_MUL which would have done the majority of the work.
-   This function merely matches an ADD with a COMPLEX_MUL IFN.  The initial
-   match is expected to be in OP1 and the initial match operands in args0.  */
-
-internal_fn
-complex_fma_pattern::matches (complex_operation_t op,
-			      slp_tree_to_load_perm_map_t * /* perm_cache */,
-			      slp_tree *ref_node, vec<slp_tree> *ops)
-{
-  internal_fn ifn = IFN_LAST;
-
-  /* Find the two components.  We match Complex MUL first which reduces the
-     amount of work this pattern has to do.  After that we just match the
-     head node and we're done.:
-
-     * FMA: + +.
-
-     We need to ignore the two_operands nodes that may also match.
-     For that we can check if they have any scalar statements and also
-     check that it's not a permute node as we're looking for a normal
-     PLUS_EXPR operation.  */
-  if (op != CMPLX_NONE)
-    return IFN_LAST;
-
-  /* Find the two components.  We match Complex MUL first which reduces the
-     amount of work this pattern has to do.  After that we just match the
-     head node and we're done.:
-
-   * FMA: + + on a non-two_operands node.  */
-  slp_tree vnode = *ref_node;
-  if (SLP_TREE_LANE_PERMUTATION (vnode).exists ()
-      || !SLP_TREE_CHILDREN (vnode).exists ()
-      || !vect_match_expression_p (vnode, PLUS_EXPR))
-    return IFN_LAST;
-
-  slp_tree node = SLP_TREE_CHILDREN (vnode)[1];
-
-  if (vect_match_call_p (node, IFN_COMPLEX_MUL))
-    ifn = IFN_COMPLEX_FMA;
-  else if (vect_match_call_p (node, IFN_COMPLEX_MUL_CONJ))
-    ifn = IFN_COMPLEX_FMA_CONJ;
-  else
-    return IFN_LAST;
-
-  if (!vect_pattern_validate_optab (ifn, vnode))
-    return IFN_LAST;
-
-  ops->truncate (0);
-  ops->create (3);
-
-  if (ifn == IFN_COMPLEX_FMA)
-    {
-      ops->quick_push (SLP_TREE_CHILDREN (vnode)[0]);
-      ops->quick_push (SLP_TREE_CHILDREN (node)[1]);
-      ops->quick_push (SLP_TREE_CHILDREN (node)[0]);
-    }
-  else
-    {
-      ops->quick_push (SLP_TREE_CHILDREN (vnode)[0]);
-      ops->quick_push (SLP_TREE_CHILDREN (node)[0]);
-      ops->quick_push (SLP_TREE_CHILDREN (node)[1]);
-    }
-
-  return ifn;
-}
-
-/* Attempt to recognize a complex mul pattern.  */
-
-vect_pattern*
-complex_fma_pattern::recognize (slp_tree_to_load_perm_map_t *perm_cache,
-				slp_tree *node)
-{
-  auto_vec<slp_tree> ops;
-  complex_operation_t op
-    = vect_detect_pair_op (*node, true, &ops);
-  internal_fn ifn
-    = complex_fma_pattern::matches (op, perm_cache, node, &ops);
-  if (ifn == IFN_LAST)
-    return NULL;
-
-  return new complex_fma_pattern (node, &ops, ifn);
-}
-
-/* Perform a replacement of the detected complex mul pattern with the new
-   instruction sequences.  */
-
-void
-complex_fma_pattern::build (vec_info *vinfo)
-{
-  slp_tree node = SLP_TREE_CHILDREN (*this->m_node)[1];
-
-  SLP_TREE_CHILDREN (*this->m_node).release ();
-  SLP_TREE_CHILDREN (*this->m_node).create (3);
-  SLP_TREE_CHILDREN (*this->m_node).safe_splice (this->m_ops);
-
-  SLP_TREE_REF_COUNT (this->m_ops[1])++;
-  SLP_TREE_REF_COUNT (this->m_ops[2])++;
-
-  vect_free_slp_tree (node);
-
-  complex_pattern::build (vinfo);
-}
-
-/*******************************************************************************
  * complex_fms_pattern class
  ******************************************************************************/
 
@@ -1264,10 +1153,10 @@ class complex_fms_pattern : public complex_pattern
 };
 
 
-/* Pattern matcher for trying to match complex multiply and accumulate
-   and multiply and subtract patterns in SLP tree.
-   If the operation matches then IFN is set to the operation it matched and
-   the arguments to the two replacement statements are put in m_ops.
+/* Pattern matcher for trying to match complex multiply and subtract pattern
+   in SLP tree.  If the operation matches then IFN is set to the operation
+   it matched and the arguments to the two replacement statements are put in
+   m_ops.
 
    If no match is found then IFN is set to IFN_LAST and m_ops is unchanged.
 
@@ -1289,38 +1178,37 @@ complex_fms_pattern::matches (complex_operation_t op,
 {
   internal_fn ifn = IFN_LAST;
 
-  /* Find the two components.  We match Complex MUL first which reduces the
-     amount of work this pattern has to do.  After that we just match the
-     head node and we're done.:
-
-     * FMS: - +.  */
-  slp_tree child = NULL;
-
   /* We need to ignore the two_operands nodes that may also match,
      for that we can check if they have any scalar statements and also
      check that it's not a permute node as we're looking for a normal
-     PLUS_EXPR operation.  */
-  if (op != PLUS_MINUS)
+     MINUS_EXPR operation.  */
+  if (op != CMPLX_NONE)
     return IFN_LAST;
 
-  child = SLP_TREE_CHILDREN ((*ops)[1])[1];
-  if (vect_detect_pair_op (child) != MINUS_PLUS)
+  slp_tree root = *ref_node;
+  if (!vect_match_expression_p (root, MINUS_EXPR))
     return IFN_LAST;
 
-  /* First two nodes must be a multiply.  */
-  auto_vec<slp_tree> muls;
-  if (vect_match_call_complex_mla (child, 0) != MULT_MULT
-      || vect_match_call_complex_mla (child, 1, &muls) != MULT_MULT)
+  auto nodes = SLP_TREE_CHILDREN (root);
+  if (!vect_match_expression_p (nodes[1], MULT_EXPR)
+      || vect_detect_pair_op (nodes[0]) != PLUS_MINUS)
     return IFN_LAST;
 
+  auto childs = SLP_TREE_CHILDREN (nodes[0]);
+  auto l0node = SLP_TREE_CHILDREN (childs[0]);
+
   /* Now operand2+4 may lead to another expression.  */
   auto_vec<slp_tree> left_op, right_op;
-  left_op.safe_splice (SLP_TREE_CHILDREN (muls[0]));
-  right_op.safe_splice (SLP_TREE_CHILDREN (muls[1]));
+  left_op.safe_splice (SLP_TREE_CHILDREN (l0node[1]));
+  right_op.safe_splice (SLP_TREE_CHILDREN (nodes[1]));
+
+  /* If these nodes don't have any children then they're
+     not ones we're interested in.  */
+  if (left_op.length () != 2 || right_op.length () != 2)
+    return IFN_LAST;
 
   bool is_neg = vect_normalize_conj_loc (left_op);
 
-  child = SLP_TREE_CHILDREN ((*ops)[1])[0];
   bool conj_first_operand = false;
   if (!vect_validate_multiplication (perm_cache, right_op, left_op, false,
 				     &conj_first_operand, true))
@@ -1340,28 +1228,28 @@ complex_fms_pattern::matches (complex_operation_t op,
   complex_perm_kinds_t kind = linear_loads_p (perm_cache, right_op[0]);
   if (kind == PERM_EVENODD)
     {
-      ops->quick_push (child);
+      ops->quick_push (l0node[0]);
       ops->quick_push (right_op[0]);
       ops->quick_push (right_op[1]);
       ops->quick_push (left_op[1]);
     }
   else if (kind == PERM_TOP)
     {
-      ops->quick_push (child);
+      ops->quick_push (l0node[0]);
       ops->quick_push (right_op[1]);
       ops->quick_push (right_op[0]);
       ops->quick_push (left_op[0]);
     }
   else if (kind == PERM_EVENEVEN && !is_neg)
     {
-      ops->quick_push (child);
+      ops->quick_push (l0node[0]);
       ops->quick_push (right_op[1]);
       ops->quick_push (right_op[0]);
       ops->quick_push (left_op[0]);
     }
   else
     {
-      ops->quick_push (child);
+      ops->quick_push (l0node[0]);
       ops->quick_push (right_op[1]);
       ops->quick_push (right_op[0]);
       ops->quick_push (left_op[1]);
@@ -1473,10 +1361,6 @@ complex_operations_pattern::recognize (slp_tree_to_load_perm_map_t *perm_cache,
   if (ifn != IFN_LAST)
     return complex_mul_pattern::mkInstance (node, &ops, ifn);
 
-  ifn  = complex_fma_pattern::matches (op, perm_cache, node, &ops);
-  if (ifn != IFN_LAST)
-    return complex_fma_pattern::mkInstance (node, &ops, ifn);
-
   ifn  = complex_add_pattern::matches (op, perm_cache, node, &ops);
   if (ifn != IFN_LAST)
     return complex_add_pattern::mkInstance (node, &ops, ifn);
diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c
index eff4640..94c7549 100644
--- a/gcc/tree-vect-slp.c
+++ b/gcc/tree-vect-slp.c
@@ -4354,7 +4354,7 @@ vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
 /* Initialize a bb_vec_info struct for the statements in BBS basic blocks.  */
 
 _bb_vec_info::_bb_vec_info (vec<basic_block> _bbs, vec_info_shared *shared)
-  : vec_info (vec_info::bb, init_cost (NULL, false), shared),
+  : vec_info (vec_info::bb, shared),
     bbs (_bbs),
     roots (vNULL)
 {
@@ -4889,16 +4889,15 @@ vect_slp_analyze_operations (vec_info *vinfo)
       else
 	{
 	  i++;
-
-	  /* For BB vectorization remember the SLP graph entry
-	     cost for later.  */
-	  if (is_a <bb_vec_info> (vinfo))
-	    instance->cost_vec = cost_vec;
-	  else
+	  if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
 	    {
-	      add_stmt_costs (vinfo, vinfo->target_cost_data, &cost_vec);
+	      add_stmt_costs (loop_vinfo->vector_costs, &cost_vec);
 	      cost_vec.release ();
 	    }
+	  else
+	    /* For BB vectorization remember the SLP graph entry
+	       cost for later.  */
+	    instance->cost_vec = cost_vec;
 	}
     }
 
@@ -5336,32 +5335,31 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo,
 	  continue;
 	}
 
-      void *scalar_target_cost_data = init_cost (NULL, true);
+      class vector_costs *scalar_target_cost_data = init_cost (bb_vinfo, true);
       do
 	{
-	  add_stmt_cost (bb_vinfo, scalar_target_cost_data,
-			 li_scalar_costs[si].second);
+	  add_stmt_cost (scalar_target_cost_data, li_scalar_costs[si].second);
 	  si++;
 	}
       while (si < li_scalar_costs.length ()
 	     && li_scalar_costs[si].first == sl);
       unsigned dummy;
-      finish_cost (scalar_target_cost_data, &dummy, &scalar_cost, &dummy);
-      destroy_cost_data (scalar_target_cost_data);
+      finish_cost (scalar_target_cost_data, nullptr,
+		   &dummy, &scalar_cost, &dummy);
+      delete scalar_target_cost_data;
 
       /* Complete the target-specific vector cost calculation.  */
-      void *vect_target_cost_data = init_cost (NULL, false);
+      class vector_costs *vect_target_cost_data = init_cost (bb_vinfo, false);
       do
 	{
-	  add_stmt_cost (bb_vinfo, vect_target_cost_data,
-			 li_vector_costs[vi].second);
+	  add_stmt_cost (vect_target_cost_data, li_vector_costs[vi].second);
 	  vi++;
 	}
       while (vi < li_vector_costs.length ()
 	     && li_vector_costs[vi].first == vl);
-      finish_cost (vect_target_cost_data, &vec_prologue_cost,
-		   &vec_inside_cost, &vec_epilogue_cost);
-      destroy_cost_data (vect_target_cost_data);
+      finish_cost (vect_target_cost_data, scalar_target_cost_data,
+		   &vec_prologue_cost, &vec_inside_cost, &vec_epilogue_cost);
+      delete vect_target_cost_data;
 
       vec_outside_cost = vec_prologue_cost + vec_epilogue_cost;
 
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index 8f52745..2284ad0 100644
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -1195,6 +1195,9 @@ vect_model_load_cost (vec_info *vinfo,
     {
       /* Invariant loads will ideally be hoisted and splat to a vector.  */
       prologue_cost += record_stmt_cost (cost_vec, 1,
+					 scalar_load, stmt_info, 0,
+					 vect_prologue);
+      prologue_cost += record_stmt_cost (cost_vec, 1,
 					 scalar_to_vec, stmt_info, 0,
 					 vect_prologue);
     }
@@ -1993,7 +1996,12 @@ get_negative_load_store_type (vec_info *vinfo,
       return VMAT_ELEMENTWISE;
     }
 
-  int misalignment = dr_misalignment (dr_info, vectype);
+  /* For backward running DRs the first access in vectype actually is
+     N-1 elements before the address of the DR.  */
+  *poffset = ((-TYPE_VECTOR_SUBPARTS (vectype) + 1)
+	      * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
+
+  int misalignment = dr_misalignment (dr_info, vectype, *poffset);
   alignment_support_scheme
     = vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment);
   if (alignment_support_scheme != dr_aligned
@@ -2002,6 +2010,7 @@ get_negative_load_store_type (vec_info *vinfo,
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
 			 "negative step but alignment required.\n");
+      *poffset = 0;
       return VMAT_ELEMENTWISE;
     }
 
@@ -2011,7 +2020,6 @@ get_negative_load_store_type (vec_info *vinfo,
 	dump_printf_loc (MSG_NOTE, vect_location,
 			 "negative step with invariant source;"
 			 " no permute needed.\n");
-      *poffset = -TYPE_VECTOR_SUBPARTS (vectype) + 1;
       return VMAT_CONTIGUOUS_DOWN;
     }
 
@@ -2020,10 +2028,10 @@ get_negative_load_store_type (vec_info *vinfo,
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
 			 "negative step and reversing not supported.\n");
+      *poffset = 0;
       return VMAT_ELEMENTWISE;
     }
 
-  *poffset = -TYPE_VECTOR_SUBPARTS (vectype) + 1;
   return VMAT_CONTIGUOUS_REVERSE;
 }
 
@@ -2312,7 +2320,7 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
     }
   else
     {
-      *misalignment = dr_misalignment (first_dr_info, vectype);
+      *misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
       *alignment_support_scheme
 	= vect_supportable_dr_alignment (vinfo, first_dr_info, vectype,
 					 *misalignment);
@@ -2462,7 +2470,7 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
 	  else
 	    *memory_access_type = VMAT_CONTIGUOUS;
 	  *misalignment = dr_misalignment (STMT_VINFO_DR_INFO (stmt_info),
-					   vectype);
+					   vectype, *poffset);
 	  *alignment_support_scheme
 	    = vect_supportable_dr_alignment (vinfo,
 					     STMT_VINFO_DR_INFO (stmt_info),
@@ -6330,8 +6338,12 @@ vectorizable_operation (vec_info *vinfo,
 static void
 ensure_base_align (dr_vec_info *dr_info)
 {
-  if (dr_info->misalignment == DR_MISALIGNMENT_UNINITIALIZED)
-    return;
+  /* Alignment is only analyzed for the first element of a DR group,
+     use that to look at base alignment we need to enforce.  */
+  if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt))
+    dr_info = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt));
+
+  gcc_assert (dr_info->misalignment != DR_MISALIGNMENT_UNINITIALIZED);
 
   if (dr_info->base_misaligned)
     {
@@ -8102,7 +8114,7 @@ vectorizable_store (vec_info *vinfo,
 	      = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
 					  simd_lane_access_p ? loop : NULL,
 					  offset, &dummy, gsi, &ptr_incr,
-					  simd_lane_access_p, NULL_TREE, bump);
+					  simd_lane_access_p, bump);
 	}
       else
 	{
@@ -8244,10 +8256,7 @@ vectorizable_store (vec_info *vinfo,
 
 	      align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
 	      if (alignment_support_scheme == dr_aligned)
-		{
-		  gcc_assert (aligned_access_p (first_dr_info, vectype));
-		  misalign = 0;
-		}
+		misalign = 0;
 	      else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
 		{
 		  align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
@@ -8332,7 +8341,7 @@ vectorizable_store (vec_info *vinfo,
 					  ? dataref_offset
 					  : build_int_cst (ref_type, 0));
 		  if (alignment_support_scheme == dr_aligned)
-		    gcc_assert (aligned_access_p (first_dr_info, vectype));
+		    ;
 		  else
 		    TREE_TYPE (data_ref)
 		      = build_aligned_type (TREE_TYPE (data_ref),
@@ -8513,7 +8522,6 @@ vectorizable_load (vec_info *vinfo,
   unsigned int group_size;
   poly_uint64 group_gap_adj;
   tree msq = NULL_TREE, lsq;
-  tree byte_offset = NULL_TREE;
   tree realignment_token = NULL_TREE;
   gphi *phi = NULL;
   vec<tree> dr_chain = vNULL;
@@ -9285,6 +9293,7 @@ vectorizable_load (vec_info *vinfo,
   bool diff_first_stmt_info
     = first_stmt_info_for_drptr && first_stmt_info != first_stmt_info_for_drptr;
 
+  tree offset = NULL_TREE;
   if ((alignment_support_scheme == dr_explicit_realign_optimized
        || alignment_support_scheme == dr_explicit_realign)
       && !compute_in_loop)
@@ -9301,19 +9310,18 @@ vectorizable_load (vec_info *vinfo,
       if (alignment_support_scheme == dr_explicit_realign_optimized)
 	{
 	  phi = as_a <gphi *> (SSA_NAME_DEF_STMT (msq));
-	  byte_offset = size_binop (MINUS_EXPR, TYPE_SIZE_UNIT (vectype),
-				    size_one_node);
+	  offset = size_binop (MINUS_EXPR, TYPE_SIZE_UNIT (vectype),
+			       size_one_node);
 	  gcc_assert (!first_stmt_info_for_drptr);
 	}
     }
   else
     at_loop = loop;
 
-  tree offset = NULL_TREE;
   if (!known_eq (poffset, 0))
-    offset = size_int (poffset);
-  if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
-    offset = size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1);
+    offset = (offset
+	      ? size_binop (PLUS_EXPR, offset, size_int (poffset))
+	      : size_int (poffset));
 
   tree bump;
   tree vec_offset = NULL_TREE;
@@ -9371,7 +9379,7 @@ vectorizable_load (vec_info *vinfo,
 		= vect_create_data_ref_ptr (vinfo, first_stmt_info_for_drptr,
 					    aggr_type, at_loop, offset, &dummy,
 					    gsi, &ptr_incr, simd_lane_access_p,
-					    byte_offset, bump);
+					    bump);
 	      /* Adjust the pointer by the difference to first_stmt.  */
 	      data_reference_p ptrdr
 		= STMT_VINFO_DATA_REF (first_stmt_info_for_drptr);
@@ -9403,8 +9411,7 @@ vectorizable_load (vec_info *vinfo,
 	      = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
 					  at_loop,
 					  offset, &dummy, gsi, &ptr_incr,
-					  simd_lane_access_p,
-					  byte_offset, bump);
+					  simd_lane_access_p, bump);
 	  if (mask)
 	    vec_mask = vec_masks[0];
 	}
@@ -9583,10 +9590,7 @@ vectorizable_load (vec_info *vinfo,
 		    align =
 		      known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
 		    if (alignment_support_scheme == dr_aligned)
-		      {
-			gcc_assert (aligned_access_p (first_dr_info, vectype));
-			misalign = 0;
-		      }
+		      misalign = 0;
 		    else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
 		      {
 			align = dr_alignment
@@ -10340,6 +10344,7 @@ vectorizable_condition (vec_info *vinfo,
 	  else
 	    {
 	      bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
+	      tree_code orig_code = cond.code;
 	      cond.code = invert_tree_comparison (cond.code, honor_nans);
 	      if (loop_vinfo->scalar_cond_masked_set.contains (cond))
 		{
@@ -10347,6 +10352,22 @@ vectorizable_condition (vec_info *vinfo,
 		  cond_code = cond.code;
 		  swap_cond_operands = true;
 		}
+	      else
+		{
+		  /* Try the inverse of the current mask.  We check if the
+		     inverse mask is live and if so we generate a negate of
+		     the current mask such that we still honor NaNs.  */
+		  cond.inverted_p = true;
+		  cond.code = orig_code;
+		  if (loop_vinfo->scalar_cond_masked_set.contains (cond))
+		    {
+		      bitop1 = orig_code;
+		      bitop2 = BIT_NOT_EXPR;
+		      masks = &LOOP_VINFO_MASKS (loop_vinfo);
+		      cond_code = cond.code;
+		      swap_cond_operands = true;
+		    }
+		}
 	    }
 	}
     }
diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c
index 4712dc6..0e1cee9 100644
--- a/gcc/tree-vectorizer.c
+++ b/gcc/tree-vectorizer.c
@@ -81,7 +81,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-pretty-print.h"
 #include "opt-problem.h"
 #include "internal-fn.h"
-
+#include "tree-ssa-sccvn.h"
 
 /* Loop or bb location, with hotness information.  */
 dump_user_location_t vect_location;
@@ -98,11 +98,10 @@ auto_purge_vect_location::~auto_purge_vect_location ()
 /* Dump a cost entry according to args to F.  */
 
 void
-dump_stmt_cost (FILE *f, void *data, int count, enum vect_cost_for_stmt kind,
+dump_stmt_cost (FILE *f, int count, enum vect_cost_for_stmt kind,
 		stmt_vec_info stmt_info, tree, int misalign, unsigned cost,
 		enum vect_cost_model_location where)
 {
-  fprintf (f, "%p ", data);
   if (stmt_info)
     {
       print_gimple_expr (f, STMT_VINFO_STMT (stmt_info), 0, TDF_SLIM);
@@ -252,11 +251,11 @@ simd_array_to_simduid::equal (const simd_array_to_simduid *p1,
    IFN_GOMP_SIMD_ORDERED_{START,END}.  */
 
 static void
-adjust_simduid_builtins (hash_table<simduid_to_vf> *htab)
+adjust_simduid_builtins (hash_table<simduid_to_vf> *htab, function *fun)
 {
   basic_block bb;
 
-  FOR_EACH_BB_FN (bb, cfun)
+  FOR_EACH_BB_FN (bb, fun)
     {
       gimple_stmt_iterator i;
 
@@ -384,7 +383,7 @@ note_simd_array_uses_cb (tree *tp, int *walk_subtrees, void *data)
    simduid.  */
 
 static void
-note_simd_array_uses (hash_table<simd_array_to_simduid> **htab)
+note_simd_array_uses (hash_table<simd_array_to_simduid> **htab, function *fun)
 {
   basic_block bb;
   gimple_stmt_iterator gsi;
@@ -395,7 +394,7 @@ note_simd_array_uses (hash_table<simd_array_to_simduid> **htab)
   wi.info = &ns;
   ns.htab = htab;
 
-  FOR_EACH_BB_FN (bb, cfun)
+  FOR_EACH_BB_FN (bb, fun)
     for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
       {
 	gimple *stmt = gsi_stmt (gsi);
@@ -457,12 +456,10 @@ shrink_simd_arrays
 /* Initialize the vec_info with kind KIND_IN and target cost data
    TARGET_COST_DATA_IN.  */
 
-vec_info::vec_info (vec_info::vec_kind kind_in, void *target_cost_data_in,
-		    vec_info_shared *shared_)
+vec_info::vec_info (vec_info::vec_kind kind_in, vec_info_shared *shared_)
   : kind (kind_in),
     shared (shared_),
-    stmt_vec_info_ro (false),
-    target_cost_data (target_cost_data_in)
+    stmt_vec_info_ro (false)
 {
   stmt_vec_infos.create (50);
 }
@@ -472,12 +469,12 @@ vec_info::~vec_info ()
   for (slp_instance &instance : slp_instances)
     vect_free_slp_instance (instance);
 
-  destroy_cost_data (target_cost_data);
   free_stmt_vec_infos ();
 }
 
 vec_info_shared::vec_info_shared ()
-  : datarefs (vNULL),
+  : n_stmts (0),
+    datarefs (vNULL),
     datarefs_copy (vNULL),
     ddrs (vNULL)
 {
@@ -882,7 +879,7 @@ vect_loop_vectorized_call (class loop *loop, gcond **cond)
    internal call.  */
 
 static gimple *
-vect_loop_dist_alias_call (class loop *loop)
+vect_loop_dist_alias_call (class loop *loop, function *fun)
 {
   basic_block bb;
   basic_block entry;
@@ -893,7 +890,7 @@ vect_loop_dist_alias_call (class loop *loop)
   if (loop->orig_loop_num == 0)
     return NULL;
 
-  orig = get_loop (cfun, loop->orig_loop_num);
+  orig = get_loop (fun, loop->orig_loop_num);
   if (orig == NULL)
     {
       /* The original loop is somehow destroyed.  Clear the information.  */
@@ -907,7 +904,7 @@ vect_loop_dist_alias_call (class loop *loop)
     bb = loop_preheader_edge (loop)->src;
 
   outer = bb->loop_father;
-  entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
+  entry = ENTRY_BLOCK_PTR_FOR_FN (fun);
 
   /* Look upward in dominance tree.  */
   for (; bb != entry && flow_bb_inside_loop_p (outer, bb);
@@ -936,12 +933,13 @@ vect_loop_dist_alias_call (class loop *loop)
    represented by LOOP_VINFO. LOOP_VECTORIZED_CALL is the internal
    call guarding the loop which has been if converted.  */
 static void
-set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
+set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call,
+		  function *fun)
 {
   tree arg = gimple_call_arg (loop_vectorized_call, 1);
   basic_block *bbs;
   unsigned int i;
-  class loop *scalar_loop = get_loop (cfun, tree_to_shwi (arg));
+  class loop *scalar_loop = get_loop (fun, tree_to_shwi (arg));
 
   LOOP_VINFO_SCALAR_LOOP (loop_vinfo) = scalar_loop;
   gcc_checking_assert (vect_loop_vectorized_call (scalar_loop)
@@ -956,7 +954,7 @@ set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
       if (g)
 	{
 	  arg = gimple_call_arg (g, 0);
-	  get_loop (cfun, tree_to_shwi (arg))->dont_vectorize = true;
+	  get_loop (fun, tree_to_shwi (arg))->dont_vectorize = true;
 	  fold_loop_internal_call (g, boolean_false_node);
 	}
     }
@@ -979,13 +977,59 @@ set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
   free (bbs);
 }
 
+/* Generate vectorized code for LOOP and its epilogues.  */
+
+static void
+vect_transform_loops (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
+		      loop_p loop, gimple *loop_vectorized_call,
+		      function *fun)
+{
+  loop_vec_info loop_vinfo = loop_vec_info_for_loop (loop);
+
+  if (loop_vectorized_call)
+    set_uid_loop_bbs (loop_vinfo, loop_vectorized_call, fun);
+
+  unsigned HOST_WIDE_INT bytes;
+  if (dump_enabled_p ())
+    {
+      if (GET_MODE_SIZE (loop_vinfo->vector_mode).is_constant (&bytes))
+	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+			 "loop vectorized using %wu byte vectors\n", bytes);
+      else
+	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+			 "loop vectorized using variable length vectors\n");
+    }
+
+  loop_p new_loop = vect_transform_loop (loop_vinfo,
+					 loop_vectorized_call);
+  /* Now that the loop has been vectorized, allow it to be unrolled
+     etc.  */
+  loop->force_vectorize = false;
+
+  if (loop->simduid)
+    {
+      simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf);
+      if (!simduid_to_vf_htab)
+	simduid_to_vf_htab = new hash_table<simduid_to_vf> (15);
+      simduid_to_vf_data->simduid = DECL_UID (loop->simduid);
+      simduid_to_vf_data->vf = loop_vinfo->vectorization_factor;
+      *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT)
+	  = simduid_to_vf_data;
+    }
+
+  /* Epilogue of vectorized loop must be vectorized too.  */
+  if (new_loop)
+    vect_transform_loops (simduid_to_vf_htab, new_loop, NULL, fun);
+}
+
 /* Try to vectorize LOOP.  */
 
 static unsigned
 try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 		      unsigned *num_vectorized_loops, loop_p loop,
 		      gimple *loop_vectorized_call,
-		      gimple *loop_dist_alias_call)
+		      gimple *loop_dist_alias_call,
+		      function *fun)
 {
   unsigned ret = 0;
   vec_info_shared shared;
@@ -999,17 +1043,9 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 		 LOCATION_FILE (vect_location.get_location_t ()),
 		 LOCATION_LINE (vect_location.get_location_t ()));
 
-  opt_loop_vec_info loop_vinfo = opt_loop_vec_info::success (NULL);
-  /* In the case of epilogue vectorization the loop already has its
-     loop_vec_info set, we do not require to analyze the loop in this case.  */
-  if (loop_vec_info vinfo = loop_vec_info_for_loop (loop))
-    loop_vinfo = opt_loop_vec_info::success (vinfo);
-  else
-    {
-      /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p.  */
-      loop_vinfo = vect_analyze_loop (loop, &shared);
-      loop->aux = loop_vinfo;
-    }
+  /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p.  */
+  opt_loop_vec_info loop_vinfo = vect_analyze_loop (loop, &shared);
+  loop->aux = loop_vinfo;
 
   if (!loop_vinfo)
     if (dump_enabled_p ())
@@ -1064,7 +1100,7 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 	  if (!require_loop_vectorize)
 	    {
 	      tree arg = gimple_call_arg (loop_vectorized_call, 1);
-	      class loop *scalar_loop = get_loop (cfun, tree_to_shwi (arg));
+	      class loop *scalar_loop = get_loop (fun, tree_to_shwi (arg));
 	      if (vect_slp_if_converted_bb (bb, scalar_loop))
 		{
 		  fold_loop_internal_call (loop_vectorized_call,
@@ -1083,8 +1119,7 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
       return ret;
     }
 
-  /* Only count the original scalar loops.  */
-  if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo) && !dbg_cnt (vect_loop))
+  if (!dbg_cnt (vect_loop))
     {
       /* Free existing information if loop is analyzed with some
 	 assumptions.  */
@@ -1093,62 +1128,22 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
       return ret;
     }
 
-  if (loop_vectorized_call)
-    set_uid_loop_bbs (loop_vinfo, loop_vectorized_call);
-
-  unsigned HOST_WIDE_INT bytes;
-  if (dump_enabled_p ())
-    {
-      if (GET_MODE_SIZE (loop_vinfo->vector_mode).is_constant (&bytes))
-	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
-			 "loop vectorized using %wu byte vectors\n", bytes);
-      else
-	dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
-			 "loop vectorized using variable length vectors\n");
-    }
-
-  loop_p new_loop = vect_transform_loop (loop_vinfo,
-					 loop_vectorized_call);
   (*num_vectorized_loops)++;
-  /* Now that the loop has been vectorized, allow it to be unrolled
-     etc.  */
-  loop->force_vectorize = false;
-
-  if (loop->simduid)
-    {
-      simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf);
-      if (!simduid_to_vf_htab)
-	simduid_to_vf_htab = new hash_table<simduid_to_vf> (15);
-      simduid_to_vf_data->simduid = DECL_UID (loop->simduid);
-      simduid_to_vf_data->vf = loop_vinfo->vectorization_factor;
-      *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT)
-	  = simduid_to_vf_data;
-    }
+  /* Transform LOOP and its epilogues.  */
+  vect_transform_loops (simduid_to_vf_htab, loop, loop_vectorized_call, fun);
 
   if (loop_vectorized_call)
     {
       fold_loop_internal_call (loop_vectorized_call, boolean_true_node);
-      loop_vectorized_call = NULL;
       ret |= TODO_cleanup_cfg;
     }
   if (loop_dist_alias_call)
     {
       tree value = gimple_call_arg (loop_dist_alias_call, 1);
       fold_loop_internal_call (loop_dist_alias_call, value);
-      loop_dist_alias_call = NULL;
       ret |= TODO_cleanup_cfg;
     }
 
-  /* Epilogue of vectorized loop must be vectorized too.  */
-  if (new_loop)
-    {
-      /* Don't include vectorized epilogues in the "vectorized loops" count.
-       */
-      unsigned dont_count = *num_vectorized_loops;
-      ret |= try_vectorize_loop_1 (simduid_to_vf_htab, &dont_count,
-				   new_loop, NULL, NULL);
-    }
-
   return ret;
 }
 
@@ -1156,7 +1151,8 @@ try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 
 static unsigned
 try_vectorize_loop (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
-		    unsigned *num_vectorized_loops, loop_p loop)
+		    unsigned *num_vectorized_loops, loop_p loop,
+		    function *fun)
 {
   if (!((flag_tree_loop_vectorize
 	 && optimize_loop_nest_for_speed_p (loop))
@@ -1165,16 +1161,50 @@ try_vectorize_loop (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
 
   return try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops, loop,
 			       vect_loop_vectorized_call (loop),
-			       vect_loop_dist_alias_call (loop));
+			       vect_loop_dist_alias_call (loop, fun), fun);
 }
 
 
+/* Loop autovectorization.  */
+
+namespace {
+
+const pass_data pass_data_vectorize =
+{
+  GIMPLE_PASS, /* type */
+  "vect", /* name */
+  OPTGROUP_LOOP | OPTGROUP_VEC, /* optinfo_flags */
+  TV_TREE_VECTORIZATION, /* tv_id */
+  ( PROP_cfg | PROP_ssa ), /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  0, /* todo_flags_finish */
+};
+
+class pass_vectorize : public gimple_opt_pass
+{
+public:
+  pass_vectorize (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_vectorize, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  virtual bool gate (function *fun)
+    {
+      return flag_tree_loop_vectorize || fun->has_force_vectorize_loops;
+    }
+
+  virtual unsigned int execute (function *);
+
+}; // class pass_vectorize
+
 /* Function vectorize_loops.
 
    Entry point to loop vectorization phase.  */
 
 unsigned
-vectorize_loops (void)
+pass_vectorize::execute (function *fun)
 {
   unsigned int i;
   unsigned int num_vectorized_loops = 0;
@@ -1185,7 +1215,7 @@ vectorize_loops (void)
   bool any_ifcvt_loops = false;
   unsigned ret = 0;
 
-  vect_loops_num = number_of_loops (cfun);
+  vect_loops_num = number_of_loops (fun);
 
   /* Bail out if there are no loops.  */
   if (vect_loops_num <= 1)
@@ -1193,15 +1223,15 @@ vectorize_loops (void)
 
   vect_slp_init ();
 
-  if (cfun->has_simduid_loops)
-    note_simd_array_uses (&simd_array_to_simduid_htab);
+  if (fun->has_simduid_loops)
+    note_simd_array_uses (&simd_array_to_simduid_htab, fun);
 
   /*  ----------- Analyze loops. -----------  */
 
   /* If some loop was duplicated, it gets bigger number
      than all previously defined loops.  This fact allows us to run
      only over initial loops skipping newly generated ones.  */
-  for (auto loop : loops_list (cfun, 0))
+  for (auto loop : loops_list (fun, 0))
     if (loop->dont_vectorize)
       {
 	any_ifcvt_loops = true;
@@ -1233,25 +1263,25 @@ vectorize_loops (void)
 	      {
 		tree arg = gimple_call_arg (loop_vectorized_call, 0);
 		class loop *vector_loop
-		  = get_loop (cfun, tree_to_shwi (arg));
+		  = get_loop (fun, tree_to_shwi (arg));
 		if (vector_loop && vector_loop != loop)
 		  {
 		    /* Make sure we don't vectorize it twice.  */
 		    vector_loop->dont_vectorize = true;
 		    ret |= try_vectorize_loop (simduid_to_vf_htab,
 					       &num_vectorized_loops,
-					       vector_loop);
+					       vector_loop, fun);
 		  }
 	      }
 	  }
       }
     else
       ret |= try_vectorize_loop (simduid_to_vf_htab, &num_vectorized_loops,
-				 loop);
+				 loop, fun);
 
   vect_location = dump_user_location_t ();
 
-  statistics_counter_event (cfun, "Vectorized loops", num_vectorized_loops);
+  statistics_counter_event (fun, "Vectorized loops", num_vectorized_loops);
   if (dump_enabled_p ()
       || (num_vectorized_loops > 0 && dump_enabled_p ()))
     dump_printf_loc (MSG_NOTE, vect_location,
@@ -1261,9 +1291,9 @@ vectorize_loops (void)
   /*  ----------- Finalize. -----------  */
 
   if (any_ifcvt_loops)
-    for (i = 1; i < number_of_loops (cfun); i++)
+    for (i = 1; i < number_of_loops (fun); i++)
       {
-	loop = get_loop (cfun, i);
+	loop = get_loop (fun, i);
 	if (loop && loop->dont_vectorize)
 	  {
 	    gimple *g = vect_loop_vectorized_call (loop);
@@ -1274,7 +1304,7 @@ vectorize_loops (void)
 		g = NULL;
 	      }
 	    else
-	      g = vect_loop_dist_alias_call (loop);
+	      g = vect_loop_dist_alias_call (loop, fun);
 
 	    if (g)
 	      {
@@ -1284,38 +1314,19 @@ vectorize_loops (void)
 	  }
       }
 
-  for (i = 1; i < number_of_loops (cfun); i++)
-    {
-      loop_vec_info loop_vinfo;
-      bool has_mask_store;
-
-      loop = get_loop (cfun, i);
-      if (!loop || !loop->aux)
-	continue;
-      loop_vinfo = (loop_vec_info) loop->aux;
-      has_mask_store = LOOP_VINFO_HAS_MASK_STORE (loop_vinfo);
-      delete loop_vinfo;
-      if (has_mask_store
-	  && targetm.vectorize.empty_mask_is_expensive (IFN_MASK_STORE))
-	optimize_mask_stores (loop);
-      loop->aux = NULL;
-    }
-
   /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE,ORDERED_{START,END}} builtins.  */
-  if (cfun->has_simduid_loops)
+  if (fun->has_simduid_loops)
     {
-      adjust_simduid_builtins (simduid_to_vf_htab);
+      adjust_simduid_builtins (simduid_to_vf_htab, fun);
       /* Avoid stale SCEV cache entries for the SIMD_LANE defs.  */
       scev_reset ();
     }
-
   /* Shrink any "omp array simd" temporary arrays to the
      actual vectorization factors.  */
   if (simd_array_to_simduid_htab)
     shrink_simd_arrays (simd_array_to_simduid_htab, simduid_to_vf_htab);
   delete simduid_to_vf_htab;
-  cfun->has_simduid_loops = false;
-  vect_slp_fini ();
+  fun->has_simduid_loops = false;
 
   if (num_vectorized_loops > 0)
     {
@@ -1323,12 +1334,49 @@ vectorize_loops (void)
 	 ???  Also while we try hard to update loop-closed SSA form we fail
 	 to properly do this in some corner-cases (see PR56286).  */
       rewrite_into_loop_closed_ssa (NULL, TODO_update_ssa_only_virtuals);
-      return TODO_cleanup_cfg;
+      ret |= TODO_cleanup_cfg;
+    }
+
+  for (i = 1; i < number_of_loops (fun); i++)
+    {
+      loop_vec_info loop_vinfo;
+      bool has_mask_store;
+
+      loop = get_loop (fun, i);
+      if (!loop || !loop->aux)
+	continue;
+      loop_vinfo = (loop_vec_info) loop->aux;
+      has_mask_store = LOOP_VINFO_HAS_MASK_STORE (loop_vinfo);
+      delete loop_vinfo;
+      if (has_mask_store
+	  && targetm.vectorize.empty_mask_is_expensive (IFN_MASK_STORE))
+	optimize_mask_stores (loop);
+
+      auto_bitmap exit_bbs;
+      /* Perform local CSE, this esp. helps because we emit code for
+	 predicates that need to be shared for optimal predicate usage.
+	 However reassoc will re-order them and prevent CSE from working
+	 as it should.  CSE only the loop body, not the entry.  */
+      bitmap_set_bit (exit_bbs, single_exit (loop)->dest->index);
+
+      edge entry = EDGE_PRED (loop_preheader_edge (loop)->src, 0);
+      do_rpo_vn (fun, entry, exit_bbs);
+
+      loop->aux = NULL;
     }
 
+  vect_slp_fini ();
+
   return ret;
 }
 
+} // anon namespace
+
+gimple_opt_pass *
+make_pass_vectorize (gcc::context *ctxt)
+{
+  return new pass_vectorize (ctxt);
+}
 
 /* Entry point to the simduid cleanup pass.  */
 
@@ -1366,10 +1414,10 @@ pass_simduid_cleanup::execute (function *fun)
 {
   hash_table<simd_array_to_simduid> *simd_array_to_simduid_htab = NULL;
 
-  note_simd_array_uses (&simd_array_to_simduid_htab);
+  note_simd_array_uses (&simd_array_to_simduid_htab, fun);
 
   /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE,ORDERED_{START,END}} builtins.  */
-  adjust_simduid_builtins (NULL);
+  adjust_simduid_builtins (NULL, fun);
 
   /* Shrink any "omp array simd" temporary arrays to the
      actual vectorization factors.  */
@@ -1675,6 +1723,7 @@ scalar_cond_masked_key::get_cond_ops_from_tree (tree t)
       this->code = TREE_CODE (t);
       this->op0 = TREE_OPERAND (t, 0);
       this->op1 = TREE_OPERAND (t, 1);
+      this->inverted_p = false;
       return;
     }
 
@@ -1687,11 +1736,290 @@ scalar_cond_masked_key::get_cond_ops_from_tree (tree t)
 	    this->code = code;
 	    this->op0 = gimple_assign_rhs1 (stmt);
 	    this->op1 = gimple_assign_rhs2 (stmt);
+	    this->inverted_p = false;
 	    return;
 	  }
+	else if (code == BIT_NOT_EXPR)
+	  {
+	    tree n_op = gimple_assign_rhs1 (stmt);
+	    if ((stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (n_op))))
+	      {
+		code = gimple_assign_rhs_code (stmt);
+		if (TREE_CODE_CLASS (code) == tcc_comparison)
+		  {
+		    this->code = code;
+		    this->op0 = gimple_assign_rhs1 (stmt);
+		    this->op1 = gimple_assign_rhs2 (stmt);
+		    this->inverted_p = true;
+		    return;
+		  }
+	      }
+	  }
       }
 
   this->code = NE_EXPR;
   this->op0 = t;
   this->op1 = build_zero_cst (TREE_TYPE (t));
+  this->inverted_p = false;
+}
+
+/* See the comment above the declaration for details.  */
+
+unsigned int
+vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
+			     stmt_vec_info stmt_info, tree vectype,
+			     int misalign, vect_cost_model_location where)
+{
+  unsigned int cost
+    = builtin_vectorization_cost (kind, vectype, misalign) * count;
+  return record_stmt_cost (stmt_info, where, cost);
+}
+
+/* See the comment above the declaration for details.  */
+
+void
+vector_costs::finish_cost (const vector_costs *)
+{
+  gcc_assert (!m_finished);
+  m_finished = true;
+}
+
+/* Record a base cost of COST units against WHERE.  If STMT_INFO is
+   nonnull, use it to adjust the cost based on execution frequency
+   (where appropriate).  */
+
+unsigned int
+vector_costs::record_stmt_cost (stmt_vec_info stmt_info,
+				vect_cost_model_location where,
+				unsigned int cost)
+{
+  cost = adjust_cost_for_freq (stmt_info, where, cost);
+  m_costs[where] += cost;
+  return cost;
+}
+
+/* COST is the base cost we have calculated for an operation in location WHERE.
+   If STMT_INFO is nonnull, use it to adjust the cost based on execution
+   frequency (where appropriate).  Return the adjusted cost.  */
+
+unsigned int
+vector_costs::adjust_cost_for_freq (stmt_vec_info stmt_info,
+				    vect_cost_model_location where,
+				    unsigned int cost)
+{
+  /* Statements in an inner loop relative to the loop being
+     vectorized are weighted more heavily.  The value here is
+     arbitrary and could potentially be improved with analysis.  */
+  if (where == vect_body
+      && stmt_info
+      && stmt_in_inner_loop_p (m_vinfo, stmt_info))
+    {
+      loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
+      cost *= LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo);
+    }
+  return cost;
+}
+
+/* See the comment above the declaration for details.  */
+
+bool
+vector_costs::better_main_loop_than_p (const vector_costs *other) const
+{
+  int diff = compare_inside_loop_cost (other);
+  if (diff != 0)
+    return diff < 0;
+
+  /* If there's nothing to choose between the loop bodies, see whether
+     there's a difference in the prologue and epilogue costs.  */
+  diff = compare_outside_loop_cost (other);
+  if (diff != 0)
+    return diff < 0;
+
+  return false;
+}
+
+
+/* See the comment above the declaration for details.  */
+
+bool
+vector_costs::better_epilogue_loop_than_p (const vector_costs *other,
+					   loop_vec_info main_loop) const
+{
+  loop_vec_info this_loop_vinfo = as_a<loop_vec_info> (this->m_vinfo);
+  loop_vec_info other_loop_vinfo = as_a<loop_vec_info> (other->m_vinfo);
+
+  poly_int64 this_vf = LOOP_VINFO_VECT_FACTOR (this_loop_vinfo);
+  poly_int64 other_vf = LOOP_VINFO_VECT_FACTOR (other_loop_vinfo);
+
+  poly_uint64 main_poly_vf = LOOP_VINFO_VECT_FACTOR (main_loop);
+  unsigned HOST_WIDE_INT main_vf;
+  unsigned HOST_WIDE_INT other_factor, this_factor, other_cost, this_cost;
+  /* If we can determine how many iterations are left for the epilogue
+     loop, that is if both the main loop's vectorization factor and number
+     of iterations are constant, then we use them to calculate the cost of
+     the epilogue loop together with a 'likely value' for the epilogues
+     vectorization factor.  Otherwise we use the main loop's vectorization
+     factor and the maximum poly value for the epilogue's.  If the target
+     has not provided with a sensible upper bound poly vectorization
+     factors are likely to be favored over constant ones.  */
+  if (main_poly_vf.is_constant (&main_vf)
+      && LOOP_VINFO_NITERS_KNOWN_P (main_loop))
+    {
+      unsigned HOST_WIDE_INT niters
+	= LOOP_VINFO_INT_NITERS (main_loop) % main_vf;
+      HOST_WIDE_INT other_likely_vf
+	= estimated_poly_value (other_vf, POLY_VALUE_LIKELY);
+      HOST_WIDE_INT this_likely_vf
+	= estimated_poly_value (this_vf, POLY_VALUE_LIKELY);
+
+      /* If the epilogue is using partial vectors we account for the
+	 partial iteration here too.  */
+      other_factor = niters / other_likely_vf;
+      if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (other_loop_vinfo)
+	  && niters % other_likely_vf != 0)
+	other_factor++;
+
+      this_factor = niters / this_likely_vf;
+      if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (this_loop_vinfo)
+	  && niters % this_likely_vf != 0)
+	this_factor++;
+    }
+  else
+    {
+      unsigned HOST_WIDE_INT main_vf_max
+	= estimated_poly_value (main_poly_vf, POLY_VALUE_MAX);
+
+      other_factor = main_vf_max / estimated_poly_value (other_vf,
+						       POLY_VALUE_MAX);
+      this_factor = main_vf_max / estimated_poly_value (this_vf,
+						       POLY_VALUE_MAX);
+
+      /* If the loop is not using partial vectors then it will iterate one
+	 time less than one that does.  It is safe to subtract one here,
+	 because the main loop's vf is always at least 2x bigger than that
+	 of an epilogue.  */
+      if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (other_loop_vinfo))
+	other_factor -= 1;
+      if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (this_loop_vinfo))
+	this_factor -= 1;
+    }
+
+  /* Compute the costs by multiplying the inside costs with the factor and
+     add the outside costs for a more complete picture.  The factor is the
+     amount of times we are expecting to iterate this epilogue.  */
+  other_cost = other->body_cost () * other_factor;
+  this_cost = this->body_cost () * this_factor;
+  other_cost += other->outside_cost ();
+  this_cost += this->outside_cost ();
+  return this_cost < other_cost;
+}
+
+/* A <=>-style subroutine of better_main_loop_than_p.  Check whether we can
+   determine the return value of better_main_loop_than_p by comparing the
+   inside (loop body) costs of THIS and OTHER.  Return:
+
+   * -1 if better_main_loop_than_p should return true.
+   * 1 if better_main_loop_than_p should return false.
+   * 0 if we can't decide.  */
+
+int
+vector_costs::compare_inside_loop_cost (const vector_costs *other) const
+{
+  loop_vec_info this_loop_vinfo = as_a<loop_vec_info> (this->m_vinfo);
+  loop_vec_info other_loop_vinfo = as_a<loop_vec_info> (other->m_vinfo);
+
+  struct loop *loop = LOOP_VINFO_LOOP (this_loop_vinfo);
+  gcc_assert (LOOP_VINFO_LOOP (other_loop_vinfo) == loop);
+
+  poly_int64 this_vf = LOOP_VINFO_VECT_FACTOR (this_loop_vinfo);
+  poly_int64 other_vf = LOOP_VINFO_VECT_FACTOR (other_loop_vinfo);
+
+  /* Limit the VFs to what is likely to be the maximum number of iterations,
+     to handle cases in which at least one loop_vinfo is fully-masked.  */
+  HOST_WIDE_INT estimated_max_niter = likely_max_stmt_executions_int (loop);
+  if (estimated_max_niter != -1)
+    {
+      if (known_le (estimated_max_niter, this_vf))
+	this_vf = estimated_max_niter;
+      if (known_le (estimated_max_niter, other_vf))
+	other_vf = estimated_max_niter;
+    }
+
+  /* Check whether the (fractional) cost per scalar iteration is lower or
+     higher: this_inside_cost / this_vf vs. other_inside_cost / other_vf.  */
+  poly_int64 rel_this = this_loop_vinfo->vector_costs->body_cost () * other_vf;
+  poly_int64 rel_other
+    = other_loop_vinfo->vector_costs->body_cost () * this_vf;
+
+  HOST_WIDE_INT est_rel_this_min
+    = estimated_poly_value (rel_this, POLY_VALUE_MIN);
+  HOST_WIDE_INT est_rel_this_max
+    = estimated_poly_value (rel_this, POLY_VALUE_MAX);
+
+  HOST_WIDE_INT est_rel_other_min
+    = estimated_poly_value (rel_other, POLY_VALUE_MIN);
+  HOST_WIDE_INT est_rel_other_max
+    = estimated_poly_value (rel_other, POLY_VALUE_MAX);
+
+  /* Check first if we can make out an unambigous total order from the minimum
+     and maximum estimates.  */
+  if (est_rel_this_min < est_rel_other_min
+      && est_rel_this_max < est_rel_other_max)
+    return -1;
+
+  if (est_rel_other_min < est_rel_this_min
+      && est_rel_other_max < est_rel_this_max)
+    return 1;
+
+  /* When other_loop_vinfo uses a variable vectorization factor,
+     we know that it has a lower cost for at least one runtime VF.
+     However, we don't know how likely that VF is.
+
+     One option would be to compare the costs for the estimated VFs.
+     The problem is that that can put too much pressure on the cost
+     model.  E.g. if the estimated VF is also the lowest possible VF,
+     and if other_loop_vinfo is 1 unit worse than this_loop_vinfo
+     for the estimated VF, we'd then choose this_loop_vinfo even
+     though (a) this_loop_vinfo might not actually be better than
+     other_loop_vinfo for that VF and (b) it would be significantly
+     worse at larger VFs.
+
+     Here we go for a hacky compromise: pick this_loop_vinfo if it is
+     no more expensive than other_loop_vinfo even after doubling the
+     estimated other_loop_vinfo VF.  For all but trivial loops, this
+     ensures that we only pick this_loop_vinfo if it is significantly
+     better than other_loop_vinfo at the estimated VF.  */
+  if (est_rel_other_min != est_rel_this_min
+      || est_rel_other_max != est_rel_this_max)
+    {
+      HOST_WIDE_INT est_rel_this_likely
+	= estimated_poly_value (rel_this, POLY_VALUE_LIKELY);
+      HOST_WIDE_INT est_rel_other_likely
+	= estimated_poly_value (rel_other, POLY_VALUE_LIKELY);
+
+      return est_rel_this_likely * 2 <= est_rel_other_likely ? -1 : 1;
+    }
+
+  return 0;
+}
+
+/* A <=>-style subroutine of better_main_loop_than_p, used when there is
+   nothing to choose between the inside (loop body) costs of THIS and OTHER.
+   Check whether we can determine the return value of better_main_loop_than_p
+   by comparing the outside (prologue and epilogue) costs of THIS and OTHER.
+   Return:
+
+   * -1 if better_main_loop_than_p should return true.
+   * 1 if better_main_loop_than_p should return false.
+   * 0 if we can't decide.  */
+
+int
+vector_costs::compare_outside_loop_cost (const vector_costs *other) const
+{
+  auto this_outside_cost = this->outside_cost ();
+  auto other_outside_cost = other->outside_cost ();
+  if (this_outside_cost != other_outside_cost)
+    return this_outside_cost < other_outside_cost ? -1 : 1;
+
+  return 0;
 }
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 866d813..bd6f334 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -266,6 +266,7 @@ struct scalar_cond_masked_key
   void get_cond_ops_from_tree (tree);
 
   unsigned ncopies;
+  bool inverted_p;
   tree_code code;
   tree op0;
   tree op1;
@@ -285,6 +286,7 @@ struct default_hash_traits<scalar_cond_masked_key>
     inchash::add_expr (v.op0, h, 0);
     inchash::add_expr (v.op1, h, 0);
     h.add_int (v.ncopies);
+    h.add_flag (v.inverted_p);
     return h.end ();
   }
 
@@ -292,9 +294,10 @@ struct default_hash_traits<scalar_cond_masked_key>
   equal (value_type existing, value_type candidate)
   {
     return (existing.ncopies == candidate.ncopies
-           && existing.code == candidate.code
-           && operand_equal_p (existing.op0, candidate.op0, 0)
-           && operand_equal_p (existing.op1, candidate.op1, 0));
+	    && existing.code == candidate.code
+	    && existing.inverted_p == candidate.inverted_p
+	    && operand_equal_p (existing.op0, candidate.op0, 0)
+	    && operand_equal_p (existing.op1, candidate.op1, 0));
   }
 
   static const bool empty_zero_p = true;
@@ -303,6 +306,7 @@ struct default_hash_traits<scalar_cond_masked_key>
   mark_empty (value_type &v)
   {
     v.ncopies = 0;
+    v.inverted_p = false;
   }
 
   static inline bool
@@ -350,6 +354,9 @@ public:
   void save_datarefs();
   void check_datarefs();
 
+  /* The number of scalar stmts.  */
+  unsigned n_stmts;
+
   /* All data references.  Freed by free_data_refs, so not an auto_vec.  */
   vec<data_reference_p> datarefs;
   vec<data_reference> datarefs_copy;
@@ -368,7 +375,7 @@ public:
   typedef hash_set<int_hash<machine_mode, E_VOIDmode, E_BLKmode> > mode_set;
   enum vec_kind { bb, loop };
 
-  vec_info (vec_kind, void *, vec_info_shared *);
+  vec_info (vec_kind, vec_info_shared *);
   ~vec_info ();
 
   stmt_vec_info add_stmt (gimple *);
@@ -405,9 +412,6 @@ public:
      stmt in the chain.  */
   auto_vec<stmt_vec_info> grouped_stores;
 
-  /* Cost data used by the target cost model.  */
-  void *target_cost_data;
-
   /* The set of vector modes used in the vectorized region.  */
   mode_set used_vector_modes;
 
@@ -587,6 +591,12 @@ public:
   /* Condition under which this loop is analyzed and versioned.  */
   tree num_iters_assumptions;
 
+  /* The cost of the vector code.  */
+  class vector_costs *vector_costs;
+
+  /* The cost of the scalar code.  */
+  class vector_costs *scalar_costs;
+
   /* Threshold of number of iterations below which vectorization will not be
      performed. It is calculated from MIN_PROFITABLE_ITERS and
      param_min_vect_loop_bound.  */
@@ -718,16 +728,6 @@ public:
      applied to the loop, i.e., no unrolling is needed, this is 1.  */
   poly_uint64 slp_unrolling_factor;
 
-  /* Cost of a single scalar iteration.  */
-  int single_scalar_iteration_cost;
-
-  /* The cost of the vector prologue and epilogue, including peeled
-     iterations and set-up code.  */
-  int vec_outside_cost;
-
-  /* The cost of the vector loop body.  */
-  int vec_inside_cost;
-
   /* The factor used to over weight those statements in an inner loop
      relative to the loop being vectorized.  */
   unsigned int inner_loop_cost_factor;
@@ -822,6 +822,7 @@ public:
 #define LOOP_VINFO_RGROUP_COMPARE_TYPE(L)  (L)->rgroup_compare_type
 #define LOOP_VINFO_RGROUP_IV_TYPE(L)       (L)->rgroup_iv_type
 #define LOOP_VINFO_PTR_MASK(L)             (L)->ptr_mask
+#define LOOP_VINFO_N_STMTS(L)		   (L)->shared->n_stmts
 #define LOOP_VINFO_LOOP_NEST(L)            (L)->shared->loop_nest
 #define LOOP_VINFO_DATAREFS(L)             (L)->shared->datarefs
 #define LOOP_VINFO_DDRS(L)                 (L)->shared->ddrs
@@ -839,7 +840,6 @@ public:
 #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
 #define LOOP_VINFO_REDUCTIONS(L)           (L)->reductions
 #define LOOP_VINFO_REDUCTION_CHAINS(L)     (L)->reduction_chains
-#define LOOP_VINFO_TARGET_COST_DATA(L)     (L)->target_cost_data
 #define LOOP_VINFO_PEELING_FOR_GAPS(L)     (L)->peeling_for_gaps
 #define LOOP_VINFO_PEELING_FOR_NITER(L)    (L)->peeling_for_niter
 #define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies
@@ -847,7 +847,6 @@ public:
 #define LOOP_VINFO_SCALAR_LOOP_SCALING(L)  (L)->scalar_loop_scaling
 #define LOOP_VINFO_HAS_MASK_STORE(L)       (L)->has_mask_store
 #define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec
-#define LOOP_VINFO_SINGLE_SCALAR_ITERATION_COST(L) (L)->single_scalar_iteration_cost
 #define LOOP_VINFO_ORIG_LOOP_INFO(L)       (L)->orig_loop_info
 #define LOOP_VINFO_SIMD_IF_COND(L)         (L)->simd_if_cond
 #define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
@@ -928,12 +927,6 @@ public:
 #define BB_VINFO_DATAREFS(B)         (B)->shared->datarefs
 #define BB_VINFO_DDRS(B)             (B)->shared->ddrs
 
-static inline bb_vec_info
-vec_info_for_bb (basic_block bb)
-{
-  return (bb_vec_info) bb->aux;
-}
-
 /*-----------------------------------------------------------------*/
 /* Info on vectorized defs.                                        */
 /*-----------------------------------------------------------------*/
@@ -1395,6 +1388,142 @@ struct gather_scatter_info {
 #define PURE_SLP_STMT(S)                  ((S)->slp_type == pure_slp)
 #define STMT_SLP_TYPE(S)                   (S)->slp_type
 
+/* Contains the scalar or vector costs for a vec_info.  */
+class vector_costs
+{
+public:
+  vector_costs (vec_info *, bool);
+  virtual ~vector_costs () {}
+
+  /* Update the costs in response to adding COUNT copies of a statement.
+
+     - WHERE specifies whether the cost occurs in the loop prologue,
+       the loop body, or the loop epilogue.
+     - KIND is the kind of statement, which is always meaningful.
+     - STMT_INFO, if nonnull, describes the statement that will be
+       vectorized.
+     - VECTYPE, if nonnull, is the vector type that the vectorized
+       statement will operate on.  Note that this should be used in
+       preference to STMT_VINFO_VECTYPE (STMT_INFO) since the latter
+       is not correct for SLP.
+     - for unaligned_load and unaligned_store statements, MISALIGN is
+       the byte misalignment of the load or store relative to the target's
+       preferred alignment for VECTYPE, or DR_MISALIGNMENT_UNKNOWN
+       if the misalignment is not known.
+
+     Return the calculated cost as well as recording it.  The return
+     value is used for dumping purposes.  */
+  virtual unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
+				      stmt_vec_info stmt_info, tree vectype,
+				      int misalign,
+				      vect_cost_model_location where);
+
+  /* Finish calculating the cost of the code.  The results can be
+     read back using the functions below.
+
+     If the costs describe vector code, SCALAR_COSTS gives the costs
+     of the corresponding scalar code, otherwise it is null.  */
+  virtual void finish_cost (const vector_costs *scalar_costs);
+
+  /* The costs in THIS and OTHER both describe ways of vectorizing
+     a main loop.  Return true if the costs described by THIS are
+     cheaper than the costs described by OTHER.  Return false if any
+     of the following are true:
+
+     - THIS and OTHER are of equal cost
+     - OTHER is better than THIS
+     - we can't be sure about the relative costs of THIS and OTHER.  */
+  virtual bool better_main_loop_than_p (const vector_costs *other) const;
+
+  /* Likewise, but the costs in THIS and OTHER both describe ways of
+     vectorizing an epilogue loop of MAIN_LOOP.  */
+  virtual bool better_epilogue_loop_than_p (const vector_costs *other,
+					    loop_vec_info main_loop) const;
+
+  unsigned int prologue_cost () const;
+  unsigned int body_cost () const;
+  unsigned int epilogue_cost () const;
+  unsigned int outside_cost () const;
+  unsigned int total_cost () const;
+
+protected:
+  unsigned int record_stmt_cost (stmt_vec_info, vect_cost_model_location,
+				 unsigned int);
+  unsigned int adjust_cost_for_freq (stmt_vec_info, vect_cost_model_location,
+				     unsigned int);
+  int compare_inside_loop_cost (const vector_costs *) const;
+  int compare_outside_loop_cost (const vector_costs *) const;
+
+  /* The region of code that we're considering vectorizing.  */
+  vec_info *m_vinfo;
+
+  /* True if we're costing the scalar code, false if we're costing
+     the vector code.  */
+  bool m_costing_for_scalar;
+
+  /* The costs of the three regions, indexed by vect_cost_model_location.  */
+  unsigned int m_costs[3];
+
+  /* True if finish_cost has been called.  */
+  bool m_finished;
+};
+
+/* Create costs for VINFO.  COSTING_FOR_SCALAR is true if the costs
+   are for scalar code, false if they are for vector code.  */
+
+inline
+vector_costs::vector_costs (vec_info *vinfo, bool costing_for_scalar)
+  : m_vinfo (vinfo),
+    m_costing_for_scalar (costing_for_scalar),
+    m_costs (),
+    m_finished (false)
+{
+}
+
+/* Return the cost of the prologue code (in abstract units).  */
+
+inline unsigned int
+vector_costs::prologue_cost () const
+{
+  gcc_checking_assert (m_finished);
+  return m_costs[vect_prologue];
+}
+
+/* Return the cost of the body code (in abstract units).  */
+
+inline unsigned int
+vector_costs::body_cost () const
+{
+  gcc_checking_assert (m_finished);
+  return m_costs[vect_body];
+}
+
+/* Return the cost of the epilogue code (in abstract units).  */
+
+inline unsigned int
+vector_costs::epilogue_cost () const
+{
+  gcc_checking_assert (m_finished);
+  return m_costs[vect_epilogue];
+}
+
+/* Return the cost of the prologue and epilogue code (in abstract units).  */
+
+inline unsigned int
+vector_costs::outside_cost () const
+{
+  return prologue_cost () + epilogue_cost ();
+}
+
+/* Return the cost of the prologue, body and epilogue code
+   (in abstract units).  */
+
+inline unsigned int
+vector_costs::total_cost () const
+{
+  return body_cost () + outside_cost ();
+}
+
 #define VECT_MAX_COST 1000
 
 /* The maximum number of intermediate steps required in multi-step type
@@ -1531,29 +1660,28 @@ int vect_get_stmt_cost (enum vect_cost_for_stmt type_of_cost)
 
 /* Alias targetm.vectorize.init_cost.  */
 
-static inline void *
-init_cost (class loop *loop_info, bool costing_for_scalar)
+static inline vector_costs *
+init_cost (vec_info *vinfo, bool costing_for_scalar)
 {
-  return targetm.vectorize.init_cost (loop_info, costing_for_scalar);
+  return targetm.vectorize.create_costs (vinfo, costing_for_scalar);
 }
 
-extern void dump_stmt_cost (FILE *, void *, int, enum vect_cost_for_stmt,
+extern void dump_stmt_cost (FILE *, int, enum vect_cost_for_stmt,
 			    stmt_vec_info, tree, int, unsigned,
 			    enum vect_cost_model_location);
 
 /* Alias targetm.vectorize.add_stmt_cost.  */
 
 static inline unsigned
-add_stmt_cost (vec_info *vinfo, void *data, int count,
+add_stmt_cost (vector_costs *costs, int count,
 	       enum vect_cost_for_stmt kind,
 	       stmt_vec_info stmt_info, tree vectype, int misalign,
 	       enum vect_cost_model_location where)
 {
-  unsigned cost = targetm.vectorize.add_stmt_cost (vinfo, data, count, kind,
-						   stmt_info, vectype,
-						   misalign, where);
+  unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, vectype,
+					misalign, where);
   if (dump_file && (dump_flags & TDF_DETAILS))
-    dump_stmt_cost (dump_file, data, count, kind, stmt_info, vectype, misalign,
+    dump_stmt_cost (dump_file, count, kind, stmt_info, vectype, misalign,
 		    cost, where);
   return cost;
 }
@@ -1561,36 +1689,32 @@ add_stmt_cost (vec_info *vinfo, void *data, int count,
 /* Alias targetm.vectorize.add_stmt_cost.  */
 
 static inline unsigned
-add_stmt_cost (vec_info *vinfo, void *data, stmt_info_for_cost *i)
+add_stmt_cost (vector_costs *costs, stmt_info_for_cost *i)
 {
-  return add_stmt_cost (vinfo, data, i->count, i->kind, i->stmt_info,
+  return add_stmt_cost (costs, i->count, i->kind, i->stmt_info,
 			i->vectype, i->misalign, i->where);
 }
 
 /* Alias targetm.vectorize.finish_cost.  */
 
 static inline void
-finish_cost (void *data, unsigned *prologue_cost,
-	     unsigned *body_cost, unsigned *epilogue_cost)
-{
-  targetm.vectorize.finish_cost (data, prologue_cost, body_cost, epilogue_cost);
-}
-
-/* Alias targetm.vectorize.destroy_cost_data.  */
-
-static inline void
-destroy_cost_data (void *data)
+finish_cost (vector_costs *costs, const vector_costs *scalar_costs,
+	     unsigned *prologue_cost, unsigned *body_cost,
+	     unsigned *epilogue_cost)
 {
-  targetm.vectorize.destroy_cost_data (data);
+  costs->finish_cost (scalar_costs);
+  *prologue_cost = costs->prologue_cost ();
+  *body_cost = costs->body_cost ();
+  *epilogue_cost = costs->epilogue_cost ();
 }
 
 inline void
-add_stmt_costs (vec_info *vinfo, void *data, stmt_vector_for_cost *cost_vec)
+add_stmt_costs (vector_costs *costs, stmt_vector_for_cost *cost_vec)
 {
   stmt_info_for_cost *cost;
   unsigned i;
   FOR_EACH_VEC_ELT (*cost_vec, i, cost)
-    add_stmt_cost (vinfo, data, cost->count, cost->kind, cost->stmt_info,
+    add_stmt_cost (costs, cost->count, cost->kind, cost->stmt_info,
 		   cost->vectype, cost->misalign, cost->where);
 }
 
@@ -1606,7 +1730,8 @@ set_dr_misalignment (dr_vec_info *dr_info, int val)
   dr_info->misalignment = val;
 }
 
-extern int dr_misalignment (dr_vec_info *dr_info, tree vectype);
+extern int dr_misalignment (dr_vec_info *dr_info, tree vectype,
+			    poly_int64 offset = 0);
 
 #define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
 
@@ -2002,7 +2127,7 @@ extern tree vect_create_data_ref_ptr (vec_info *,
 				      stmt_vec_info, tree, class loop *, tree,
 				      tree *, gimple_stmt_iterator *,
 				      gimple **, bool,
-				      tree = NULL_TREE, tree = NULL_TREE);
+				      tree = NULL_TREE);
 extern tree bump_vector_ptr (vec_info *, tree, gimple *, gimple_stmt_iterator *,
 			     stmt_vec_info, tree);
 extern void vect_copy_ref_info (tree, tree);
@@ -2027,9 +2152,10 @@ extern tree vect_get_new_ssa_name (tree, enum vect_var_kind,
 				   const char * = NULL);
 extern tree vect_create_addr_base_for_vector_ref (vec_info *,
 						  stmt_vec_info, gimple_seq *,
-						  tree, tree = NULL_TREE);
+						  tree);
 
 /* In tree-vect-loop.c.  */
+extern tree neutral_op_for_reduction (tree, tree_code, tree);
 extern widest_int vect_iv_limit_for_partial_vectors (loop_vec_info loop_vinfo);
 bool vect_rgroup_iv_might_wrap_p (loop_vec_info, rgroup_controls *);
 /* Used in tree-vect-loop-manip.c */
@@ -2060,8 +2186,18 @@ extern bool reduction_fn_for_scalar_code (enum tree_code, internal_fn *);
 
 /* Drive for loop transformation stage.  */
 extern class loop *vect_transform_loop (loop_vec_info, gimple *);
-extern opt_loop_vec_info vect_analyze_loop_form (class loop *,
-						 vec_info_shared *);
+struct vect_loop_form_info
+{
+  tree number_of_iterations;
+  tree number_of_iterationsm1;
+  tree assumptions;
+  gcond *loop_cond;
+  gcond *inner_loop_cond;
+};
+extern opt_result vect_analyze_loop_form (class loop *, vect_loop_form_info *);
+extern loop_vec_info vect_create_loop_vinfo (class loop *, vec_info_shared *,
+					     const vect_loop_form_info *,
+					     loop_vec_info = nullptr);
 extern bool vectorizable_live_operation (vec_info *,
 					 stmt_vec_info, gimple_stmt_iterator *,
 					 slp_tree, slp_instance, int,
diff --git a/gcc/tree-vrp.c b/gcc/tree-vrp.c
index c24c67f..dd77236 100644
--- a/gcc/tree-vrp.c
+++ b/gcc/tree-vrp.c
@@ -43,12 +43,13 @@ along with GCC; see the file COPYING3.  If not see
 #include "cfgloop.h"
 #include "tree-scalar-evolution.h"
 #include "tree-ssa-propagate.h"
-#include "tree-ssa-threadedge.h"
 #include "domwalk.h"
 #include "vr-values.h"
 #include "gimple-array-bounds.h"
 #include "gimple-range.h"
 #include "gimple-range-path.h"
+#include "value-pointer-equiv.h"
+#include "gimple-fold.h"
 
 /* Set of SSA names found live during the RPO traversal of the function
    for still active basic-blocks.  */
@@ -3682,9 +3683,7 @@ vrp_asserts::all_imm_uses_in_stmt_or_feed_cond (tree var,
 
    However, by converting the assertion into the implied copy
    operation N_i = N_j, we will then copy-propagate N_j into the uses
-   of N_i and lose the range information.  We may want to hold on to
-   ASSERT_EXPRs a little while longer as the ranges could be used in
-   things like jump threading.
+   of N_i and lose the range information.
 
    The problem with keeping ASSERT_EXPRs around is that passes after
    VRP need to handle them appropriately.
@@ -4029,6 +4028,7 @@ class vrp_folder : public substitute_and_fold_engine
     : substitute_and_fold_engine (/* Fold all stmts.  */ true),
       m_vr_values (v), simplifier (v)
     {  }
+  void simplify_casted_conds (function *fun);
 
 private:
   tree value_of_expr (tree name, gimple *stmt) OVERRIDE
@@ -4115,71 +4115,26 @@ vrp_folder::fold_stmt (gimple_stmt_iterator *si)
   return simplifier.simplify (si);
 }
 
-/* STMT is a conditional at the end of a basic block.
-
-   If the conditional is of the form SSA_NAME op constant and the SSA_NAME
-   was set via a type conversion, try to replace the SSA_NAME with the RHS
-   of the type conversion.  Doing so makes the conversion dead which helps
-   subsequent passes.  */
+/* A comparison of an SSA_NAME against a constant where the SSA_NAME
+   was set by a type conversion can often be rewritten to use the RHS
+   of the type conversion.  Do this optimization for all conditionals
+   in FUN.  */
 
-static void
-vrp_simplify_cond_using_ranges (range_query *query, gcond *stmt)
+void
+vrp_folder::simplify_casted_conds (function *fun)
 {
-  tree op0 = gimple_cond_lhs (stmt);
-  tree op1 = gimple_cond_rhs (stmt);
-
-  /* If we have a comparison of an SSA_NAME (OP0) against a constant,
-     see if OP0 was set by a type conversion where the source of
-     the conversion is another SSA_NAME with a range that fits
-     into the range of OP0's type.
-
-     If so, the conversion is redundant as the earlier SSA_NAME can be
-     used for the comparison directly if we just massage the constant in the
-     comparison.  */
-  if (TREE_CODE (op0) == SSA_NAME
-      && TREE_CODE (op1) == INTEGER_CST)
+  basic_block bb;
+  FOR_EACH_BB_FN (bb, fun)
     {
-      gimple *def_stmt = SSA_NAME_DEF_STMT (op0);
-      tree innerop;
-
-      if (!is_gimple_assign (def_stmt))
-	return;
-
-      switch (gimple_assign_rhs_code (def_stmt))
-	{
-	CASE_CONVERT:
-	  innerop = gimple_assign_rhs1 (def_stmt);
-	  break;
-	case VIEW_CONVERT_EXPR:
-	  innerop = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
-	  if (!INTEGRAL_TYPE_P (TREE_TYPE (innerop)))
-	    return;
-	  break;
-	default:
-	  return;
-	}
-
-      if (TREE_CODE (innerop) == SSA_NAME
-	  && !POINTER_TYPE_P (TREE_TYPE (innerop))
-	  && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (innerop)
-	  && desired_pro_or_demotion_p (TREE_TYPE (innerop), TREE_TYPE (op0)))
+      gimple *last = last_stmt (bb);
+      if (last && gimple_code (last) == GIMPLE_COND)
 	{
-	  const value_range *vr = query->get_value_range (innerop);
-
-	  if (range_int_cst_p (vr)
-	      && range_fits_type_p (vr,
-				    TYPE_PRECISION (TREE_TYPE (op0)),
-				    TYPE_SIGN (TREE_TYPE (op0)))
-	      && int_fits_type_p (op1, TREE_TYPE (innerop)))
+	  if (simplifier.simplify_casted_cond (as_a <gcond *> (last)))
 	    {
-	      tree newconst = fold_convert (TREE_TYPE (innerop), op1);
-	      gimple_cond_set_lhs (stmt, innerop);
-	      gimple_cond_set_rhs (stmt, newconst);
-	      update_stmt (stmt);
 	      if (dump_file && (dump_flags & TDF_DETAILS))
 		{
 		  fprintf (dump_file, "Folded into: ");
-		  print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
+		  print_gimple_stmt (dump_file, last, 0, TDF_SLIM);
 		  fprintf (dump_file, "\n");
 		}
 	    }
@@ -4187,27 +4142,6 @@ vrp_simplify_cond_using_ranges (range_query *query, gcond *stmt)
     }
 }
 
-/* A comparison of an SSA_NAME against a constant where the SSA_NAME
-   was set by a type conversion can often be rewritten to use the RHS
-   of the type conversion.  Do this optimization for all conditionals
-   in FUN.
-
-   However, doing so inhibits jump threading through the comparison.
-   So that transformation is not performed until after jump threading
-   is complete.  */
-
-static void
-simplify_casted_conds (function *fun, range_query *query)
-{
-  basic_block bb;
-  FOR_EACH_BB_FN (bb, fun)
-    {
-      gimple *last = last_stmt (bb);
-      if (last && gimple_code (last) == GIMPLE_COND)
-	vrp_simplify_cond_using_ranges (query, as_a <gcond *> (last));
-    }
-}
-
 /* Main entry point to VRP (Value Range Propagation).  This pass is
    loosely based on J. R. C. Patterson, ``Accurate Static Branch
    Prediction by Value Range Propagation,'' in SIGPLAN Conference on
@@ -4299,13 +4233,10 @@ execute_vrp (struct function *fun, bool warn_array_bounds_p)
       array_checker.check ();
     }
 
-  simplify_casted_conds (fun, &vrp_vr_values);
+  folder.simplify_casted_conds (fun);
 
   free_numbers_of_iterations_estimates (fun);
 
-  /* ASSERT_EXPRs must be removed before finalizing jump threads
-     as finalizing jump threads calls the CFG cleanup code which
-     does not properly handle ASSERT_EXPRs.  */
   assert_engine.remove_range_assertions ();
 
   scev_finalize ();
@@ -4313,170 +4244,178 @@ execute_vrp (struct function *fun, bool warn_array_bounds_p)
   return 0;
 }
 
-namespace {
-
-const pass_data pass_data_vrp =
-{
-  GIMPLE_PASS, /* type */
-  "vrp", /* name */
-  OPTGROUP_NONE, /* optinfo_flags */
-  TV_TREE_VRP, /* tv_id */
-  PROP_ssa, /* properties_required */
-  0, /* properties_provided */
-  0, /* properties_destroyed */
-  0, /* todo_flags_start */
-  ( TODO_cleanup_cfg | TODO_update_ssa ), /* todo_flags_finish */
-};
+// This is a ranger based folder which continues to use the dominator
+// walk to access the substitute and fold machinery.  Ranges are calculated
+// on demand.
 
-class pass_vrp : public gimple_opt_pass
+class rvrp_folder : public substitute_and_fold_engine
 {
 public:
-  pass_vrp (gcc::context *ctxt)
-    : gimple_opt_pass (pass_data_vrp, ctxt), warn_array_bounds_p (false)
-  {}
 
-  /* opt_pass methods: */
-  opt_pass * clone () { return new pass_vrp (m_ctxt); }
-  void set_pass_param (unsigned int n, bool param)
-    {
-      gcc_assert (n == 0);
-      warn_array_bounds_p = param;
-    }
-  virtual bool gate (function *) { return flag_tree_vrp != 0; }
-  virtual unsigned int execute (function *fun)
-    { return execute_vrp (fun, warn_array_bounds_p); }
+  rvrp_folder (gimple_ranger *r) : substitute_and_fold_engine (),
+				   m_simplifier (r, r->non_executable_edge_flag)
+  {
+    m_ranger = r;
+    m_pta = new pointer_equiv_analyzer (m_ranger);
+  }
 
- private:
-  bool warn_array_bounds_p;
-}; // class pass_vrp
+  ~rvrp_folder ()
+  {
+    delete m_pta;
+  }
 
-} // anon namespace
+  tree value_of_expr (tree name, gimple *s = NULL) OVERRIDE
+  {
+    // Shortcircuit subst_and_fold callbacks for abnormal ssa_names.
+    if (TREE_CODE (name) == SSA_NAME && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name))
+      return NULL;
+    tree ret = m_ranger->value_of_expr (name, s);
+    if (!ret && supported_pointer_equiv_p (name))
+      ret = m_pta->get_equiv (name);
+    return ret;
+  }
 
-gimple_opt_pass *
-make_pass_vrp (gcc::context *ctxt)
-{
-  return new pass_vrp (ctxt);
-}
+  tree value_on_edge (edge e, tree name) OVERRIDE
+  {
+    // Shortcircuit subst_and_fold callbacks for abnormal ssa_names.
+    if (TREE_CODE (name) == SSA_NAME && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name))
+      return NULL;
+    tree ret = m_ranger->value_on_edge (e, name);
+    if (!ret && supported_pointer_equiv_p (name))
+      ret = m_pta->get_equiv (name);
+    return ret;
+  }
 
-// This is the dom walker for the hybrid threader.  The reason this is
-// here, as opposed to the generic threading files, is because the
-// other client would be DOM, and they have their own custom walker.
+  tree value_of_stmt (gimple *s, tree name = NULL) OVERRIDE
+  {
+    // Shortcircuit subst_and_fold callbacks for abnormal ssa_names.
+    if (TREE_CODE (name) == SSA_NAME && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name))
+      return NULL;
+    return m_ranger->value_of_stmt (s, name);
+  }
 
-class hybrid_threader : public dom_walker
-{
-public:
-  hybrid_threader ();
-  ~hybrid_threader ();
+  void pre_fold_bb (basic_block bb) OVERRIDE
+  {
+    m_pta->enter (bb);
+  }
 
-  void thread_jumps (function *fun)
+  void post_fold_bb (basic_block bb) OVERRIDE
   {
-    walk (fun->cfg->x_entry_block_ptr);
+    m_pta->leave (bb);
   }
-  bool thread_through_all_blocks ()
+
+  void pre_fold_stmt (gimple *stmt) OVERRIDE
   {
-    return m_threader->thread_through_all_blocks (false);
+    m_pta->visit_stmt (stmt);
   }
 
-private:
-  edge before_dom_children (basic_block) override;
-  void after_dom_children (basic_block bb) override;
+  bool fold_stmt (gimple_stmt_iterator *gsi) OVERRIDE
+  {
+    if (m_simplifier.simplify (gsi))
+      return true;
+    return m_ranger->fold_stmt (gsi, follow_single_use_edges);
+  }
 
-  hybrid_jt_simplifier *m_simplifier;
-  jump_threader *m_threader;
-  jt_state *m_state;
+private:
+  DISABLE_COPY_AND_ASSIGN (rvrp_folder);
   gimple_ranger *m_ranger;
-  path_range_query *m_query;
+  simplify_using_ranges m_simplifier;
+  pointer_equiv_analyzer *m_pta;
 };
 
-hybrid_threader::hybrid_threader () : dom_walker (CDI_DOMINATORS, REACHABLE_BLOCKS)
+/* Main entry point for a VRP pass using just ranger. This can be called
+  from anywhere to perform a VRP pass, including from EVRP.  */
+
+unsigned int
+execute_ranger_vrp (struct function *fun, bool warn_array_bounds_p)
 {
   loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
+  rewrite_into_loop_closed_ssa (NULL, TODO_update_ssa);
   scev_initialize ();
   calculate_dominance_info (CDI_DOMINATORS);
-  mark_dfs_back_edges ();
-
-  m_ranger = new gimple_ranger;
-  m_query = new path_range_query (*m_ranger, /*resolve=*/true);
-  m_simplifier = new hybrid_jt_simplifier (m_ranger, m_query);
-  m_state = new hybrid_jt_state;
-  m_threader = new jump_threader (m_simplifier, m_state);
-}
-
-hybrid_threader::~hybrid_threader ()
-{
-  delete m_simplifier;
-  delete m_threader;
-  delete m_state;
-  delete m_ranger;
-  delete m_query;
 
-  scev_finalize ();
-  loop_optimizer_finalize ();
-}
-
-edge
-hybrid_threader::before_dom_children (basic_block bb)
-{
-  gimple_stmt_iterator gsi;
-  int_range<2> r;
+  gimple_ranger *ranger = enable_ranger (fun);
+  rvrp_folder folder (ranger);
+  folder.substitute_and_fold ();
+  ranger->export_global_ranges ();
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    ranger->dump (dump_file);
 
-  for (gsi = gsi_start_nondebug_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+  if (warn_array_bounds && warn_array_bounds_p)
     {
-      gimple *stmt = gsi_stmt (gsi);
-      m_ranger->range_of_stmt (r, stmt);
+      // Set all edges as executable, except those ranger says aren't.
+      int non_exec_flag = ranger->non_executable_edge_flag;
+      basic_block bb;
+      FOR_ALL_BB_FN (bb, fun)
+	{
+	  edge_iterator ei;
+	  edge e;
+	  FOR_EACH_EDGE (e, ei, bb->succs)
+	    if (e->flags & non_exec_flag)
+	      e->flags &= ~EDGE_EXECUTABLE;
+	    else
+	      e->flags |= EDGE_EXECUTABLE;
+	}
+      scev_reset ();
+      array_bounds_checker array_checker (fun, ranger);
+      array_checker.check ();
     }
-  return NULL;
-}
-
-void
-hybrid_threader::after_dom_children (basic_block bb)
-{
-  m_threader->thread_outgoing_edges (bb);
-}
 
-static unsigned int
-execute_vrp_threader (function *fun)
-{
-  hybrid_threader threader;
-  threader.thread_jumps (fun);
-  if (threader.thread_through_all_blocks ())
-    return (TODO_cleanup_cfg | TODO_update_ssa);
+  disable_ranger (fun);
+  scev_finalize ();
+  loop_optimizer_finalize ();
   return 0;
 }
 
 namespace {
 
-const pass_data pass_data_vrp_threader =
+const pass_data pass_data_vrp =
 {
   GIMPLE_PASS, /* type */
-  "vrp-thread", /* name */
+  "vrp", /* name */
   OPTGROUP_NONE, /* optinfo_flags */
-  TV_TREE_VRP_THREADER, /* tv_id */
+  TV_TREE_VRP, /* tv_id */
   PROP_ssa, /* properties_required */
   0, /* properties_provided */
   0, /* properties_destroyed */
   0, /* todo_flags_start */
-  0 /* todo_flags_finish */
+  ( TODO_cleanup_cfg | TODO_update_ssa ), /* todo_flags_finish */
 };
 
-class pass_vrp_threader : public gimple_opt_pass
+static int vrp_pass_num = 0;
+class pass_vrp : public gimple_opt_pass
 {
 public:
-  pass_vrp_threader (gcc::context *ctxt)
-    : gimple_opt_pass (pass_data_vrp_threader, ctxt)
+  pass_vrp (gcc::context *ctxt)
+    : gimple_opt_pass (pass_data_vrp, ctxt), warn_array_bounds_p (false),
+      my_pass (++vrp_pass_num)
   {}
 
   /* opt_pass methods: */
-  opt_pass * clone () { return new pass_vrp_threader (m_ctxt); }
+  opt_pass * clone () { return new pass_vrp (m_ctxt); }
+  void set_pass_param (unsigned int n, bool param)
+    {
+      gcc_assert (n == 0);
+      warn_array_bounds_p = param;
+    }
   virtual bool gate (function *) { return flag_tree_vrp != 0; }
   virtual unsigned int execute (function *fun)
-    { return execute_vrp_threader (fun); }
-};
+    {
+      if ((my_pass == 1 && param_vrp1_mode == VRP_MODE_RANGER)
+	  || (my_pass == 2 && param_vrp2_mode == VRP_MODE_RANGER))
+	return execute_ranger_vrp (fun, warn_array_bounds_p);
+      return execute_vrp (fun, warn_array_bounds_p);
+    }
+
+ private:
+  bool warn_array_bounds_p;
+  int my_pass;
+}; // class pass_vrp
 
-} // namespace {
+} // anon namespace
 
 gimple_opt_pass *
-make_pass_vrp_threader (gcc::context *ctxt)
+make_pass_vrp (gcc::context *ctxt)
 {
-  return new pass_vrp_threader (ctxt);
+  return new pass_vrp (ctxt);
 }
diff --git a/gcc/tree-vrp.h b/gcc/tree-vrp.h
index 3392ecc..cf379de 100644
--- a/gcc/tree-vrp.h
+++ b/gcc/tree-vrp.h
@@ -65,4 +65,6 @@ extern void maybe_set_nonzero_bits (edge, tree);
 extern wide_int masked_increment (const wide_int &val_in, const wide_int &mask,
 				  const wide_int &sgnbit, unsigned int prec);
 
+extern unsigned int execute_ranger_vrp (struct function *fun,
+					bool warn_array_bounds_p = false);
 #endif /* GCC_TREE_VRP_H */
diff --git a/gcc/tree.c b/gcc/tree.c
index 7bfd641..f2c829f 100644
--- a/gcc/tree.c
+++ b/gcc/tree.c
@@ -330,7 +330,7 @@ unsigned const char omp_clause_num_ops[] =
   1, /* OMP_CLAUSE_DIST_SCHEDULE  */
   0, /* OMP_CLAUSE_INBRANCH  */
   0, /* OMP_CLAUSE_NOTINBRANCH  */
-  1, /* OMP_CLAUSE_NUM_TEAMS  */
+  2, /* OMP_CLAUSE_NUM_TEAMS  */
   1, /* OMP_CLAUSE_THREAD_LIMIT  */
   0, /* OMP_CLAUSE_PROC_BIND  */
   1, /* OMP_CLAUSE_SAFELEN  */
@@ -5280,6 +5280,18 @@ build_decl (location_t loc, enum tree_code code, tree name,
   return t;
 }
 
+/* Create and return a DEBUG_EXPR_DECL node of the given TYPE.  */
+
+tree
+build_debug_expr_decl (tree type)
+{
+  tree vexpr = make_node (DEBUG_EXPR_DECL);
+  DECL_ARTIFICIAL (vexpr) = 1;
+  TREE_TYPE (vexpr) = type;
+  SET_DECL_MODE (vexpr, TYPE_MODE (type));
+  return vexpr;
+}
+
 /* Builds and returns function declaration with NAME and TYPE.  */
 
 tree
diff --git a/gcc/tree.h b/gcc/tree.h
index 7542d97..03719b18 100644
--- a/gcc/tree.h
+++ b/gcc/tree.h
@@ -1689,6 +1689,11 @@ class auto_suppress_location_wrappers
    map clause.  */
 #define OMP_CLAUSE_MAP_IMPLICIT(NODE) \
   (OMP_CLAUSE_SUBCODE_CHECK (NODE, OMP_CLAUSE_MAP)->base.default_def_flag)
+/* Nonzero if this map clause is to be indicated to the runtime as 'implicit',
+   due to being created through implicit data-mapping rules in the middle-end.
+   NOTE: this is different than OMP_CLAUSE_MAP_IMPLICIT.  */
+#define OMP_CLAUSE_MAP_RUNTIME_IMPLICIT_P(NODE) \
+  (OMP_CLAUSE_SUBCODE_CHECK (NODE, OMP_CLAUSE_MAP)->base.deprecated_flag)
 
 /* True on an OMP_CLAUSE_USE_DEVICE_PTR with an OpenACC 'if_present'
    clause.  */
@@ -1803,9 +1808,12 @@ class auto_suppress_location_wrappers
 #define OMP_CLAUSE_ALLOCATE_COMBINED(NODE) \
   (OMP_CLAUSE_SUBCODE_CHECK (NODE, OMP_CLAUSE_ALLOCATE)->base.public_flag)
 
-#define OMP_CLAUSE_NUM_TEAMS_EXPR(NODE) \
+#define OMP_CLAUSE_NUM_TEAMS_UPPER_EXPR(NODE) \
   OMP_CLAUSE_OPERAND (OMP_CLAUSE_SUBCODE_CHECK (NODE, OMP_CLAUSE_NUM_TEAMS), 0)
 
+#define OMP_CLAUSE_NUM_TEAMS_LOWER_EXPR(NODE) \
+  OMP_CLAUSE_OPERAND (OMP_CLAUSE_SUBCODE_CHECK (NODE, OMP_CLAUSE_NUM_TEAMS), 1)
+
 #define OMP_CLAUSE_THREAD_LIMIT_EXPR(NODE) \
   OMP_CLAUSE_OPERAND (OMP_CLAUSE_SUBCODE_CHECK (NODE, \
 						OMP_CLAUSE_THREAD_LIMIT), 0)
@@ -4567,6 +4575,7 @@ extern tree build_tree_list (tree, tree CXX_MEM_STAT_INFO);
 extern tree build_tree_list_vec (const vec<tree, va_gc> * CXX_MEM_STAT_INFO);
 extern tree build_decl (location_t, enum tree_code,
 			tree, tree CXX_MEM_STAT_INFO);
+extern tree build_debug_expr_decl (tree type);
 extern tree build_fn_decl (const char *, tree);
 extern tree build_translation_unit_decl (tree);
 extern tree build_block (tree, tree, tree, tree);
diff --git a/gcc/value-pointer-equiv.cc b/gcc/value-pointer-equiv.cc
index 7d2658b..5eb9558 100644
--- a/gcc/value-pointer-equiv.cc
+++ b/gcc/value-pointer-equiv.cc
@@ -41,7 +41,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "domwalk.h"
 #include "tree-cfgcleanup.h"
 #include "vr-values.h"
-#include "gimple-ssa-evrp-analyze.h"
 #include "gimple-range.h"
 #include "fold-const.h"
 #include "value-pointer-equiv.h"
@@ -58,7 +57,7 @@ public:
   void enter (basic_block);
   void leave (basic_block);
   void push_replacement (tree name, tree replacement);
-  tree get_replacement (tree name) const;
+  tree get_replacement (tree name);
 
 private:
   auto_vec<std::pair <tree, tree>> m_stack;
@@ -68,7 +67,7 @@ private:
 
 ssa_equiv_stack::ssa_equiv_stack ()
 {
-  m_replacements.safe_grow_cleared (num_ssa_names);
+  m_replacements.safe_grow_cleared (num_ssa_names + 1);
 }
 
 // Pushes a marker at the given point.
@@ -99,29 +98,38 @@ ssa_equiv_stack::leave (basic_block)
 void
 ssa_equiv_stack::push_replacement (tree name, tree replacement)
 {
-  tree old = m_replacements[SSA_NAME_VERSION (name)];
-  m_replacements[SSA_NAME_VERSION (name)] = replacement;
+  unsigned v = SSA_NAME_VERSION (name);
+
+  if (v >= m_replacements.length ())
+    m_replacements.safe_grow_cleared (num_ssa_names + 1);
+
+  tree old = m_replacements[v];
+  m_replacements[v] = replacement;
   m_stack.safe_push (std::make_pair (name, old));
 }
 
 // Return the equivalence of NAME.
 
 tree
-ssa_equiv_stack::get_replacement (tree name) const
+ssa_equiv_stack::get_replacement (tree name)
 {
-  return m_replacements[SSA_NAME_VERSION (name)];
+  unsigned v = SSA_NAME_VERSION (name);
+
+  if (v >= m_replacements.length ())
+    m_replacements.safe_grow_cleared (num_ssa_names + 1);
+
+  return m_replacements[v];
 }
 
 pointer_equiv_analyzer::pointer_equiv_analyzer (gimple_ranger *r)
 {
   m_ranger = r;
-  m_global_points = new tree[num_ssa_names] ();
+  m_global_points.safe_grow_cleared (num_ssa_names + 1);
   m_cond_points = new ssa_equiv_stack;
 }
 
 pointer_equiv_analyzer::~pointer_equiv_analyzer ()
 {
-  delete[] m_global_points;
   delete m_cond_points;
 }
 
@@ -130,7 +138,12 @@ pointer_equiv_analyzer::~pointer_equiv_analyzer ()
 void
 pointer_equiv_analyzer::set_global_equiv (tree ssa, tree pointee)
 {
-  m_global_points[SSA_NAME_VERSION (ssa)] = pointee;
+  unsigned v = SSA_NAME_VERSION (ssa);
+
+  if (v >= m_global_points.length ())
+    m_global_points.safe_grow_cleared (num_ssa_names + 1);
+
+  m_global_points[v] = pointee;
 }
 
 // Set the conditional pointer equivalency for SSA to POINTEE.
@@ -146,9 +159,14 @@ pointer_equiv_analyzer::set_cond_equiv (tree ssa, tree pointee)
 // conditional info.
 
 tree
-pointer_equiv_analyzer::get_equiv (tree ssa) const
+pointer_equiv_analyzer::get_equiv (tree ssa)
 {
-  tree ret = m_global_points[SSA_NAME_VERSION (ssa)];
+  unsigned v = SSA_NAME_VERSION (ssa);
+
+  if (v >= m_global_points.length ())
+    m_global_points.safe_grow_cleared (num_ssa_names + 1);
+
+  tree ret = m_global_points[v];
   if (ret)
     return ret;
   return m_cond_points->get_replacement (ssa);
@@ -211,7 +229,7 @@ pointer_equiv_analyzer::leave (basic_block bb)
 // nor an invariant.
 
 tree
-pointer_equiv_analyzer::get_equiv_expr (tree_code code, tree expr) const
+pointer_equiv_analyzer::get_equiv_expr (tree_code code, tree expr)
 {
   if (code == SSA_NAME)
     return get_equiv (expr);
diff --git a/gcc/value-pointer-equiv.h b/gcc/value-pointer-equiv.h
index 0921579..dc747d0 100644
--- a/gcc/value-pointer-equiv.h
+++ b/gcc/value-pointer-equiv.h
@@ -38,17 +38,17 @@ public:
   void enter (basic_block);
   void leave (basic_block);
   void visit_stmt (gimple *stmt);
-  tree get_equiv (tree ssa) const;
+  tree get_equiv (tree ssa);
 
 private:
   void visit_edge (edge e);
-  tree get_equiv_expr (tree_code code, tree expr) const;
+  tree get_equiv_expr (tree_code code, tree expr);
   void set_global_equiv (tree ssa, tree pointee);
   void set_cond_equiv (tree ssa, tree pointee);
 
   gimple_ranger *m_ranger;
   // Global pointer equivalency indexed by SSA_NAME_VERSION.
-  tree *m_global_points;
+  auto_vec<tree> m_global_points;
   // Conditional pointer equivalency.
   class ssa_equiv_stack *m_cond_points;
 };
diff --git a/gcc/value-query.cc b/gcc/value-query.cc
index 17ebd86..b7d9e66 100644
--- a/gcc/value-query.cc
+++ b/gcc/value-query.cc
@@ -435,14 +435,6 @@ gimple_range_global (tree name)
 
 global_range_query global_ranges;
 
-// Like get_range_query, but for accessing global ranges.
-
-range_query *
-get_global_range_query ()
-{
-  return &global_ranges;
-}
-
 bool
 global_range_query::range_of_expr (irange &r, tree expr, gimple *stmt)
 {
diff --git a/gcc/value-query.h b/gcc/value-query.h
index 5161d23..38e23bb 100644
--- a/gcc/value-query.h
+++ b/gcc/value-query.h
@@ -127,6 +127,22 @@ public:
 };
 
 extern global_range_query global_ranges;
+
+inline range_query *
+get_global_range_query ()
+{
+  return &global_ranges;
+}
+
+/* Returns the currently active range access class.  When there is no active
+   range class, global ranges are used.  Never returns null.  */
+
+ATTRIBUTE_RETURNS_NONNULL inline range_query *
+get_range_query (const struct function *fun)
+{
+  return fun->x_range_query ? fun->x_range_query : &global_ranges;
+}
+
 extern value_range gimple_range_global (tree name);
 extern bool update_global_range (irange &r, tree name);
 
diff --git a/gcc/value-range.cc b/gcc/value-range.cc
index caef249..82509fa 100644
--- a/gcc/value-range.cc
+++ b/gcc/value-range.cc
@@ -270,6 +270,14 @@ irange::irange_set_anti_range (tree min, tree max)
 void
 irange::set (tree min, tree max, value_range_kind kind)
 {
+  if (kind != VR_UNDEFINED)
+    {
+      if (TREE_OVERFLOW_P (min))
+	min = drop_tree_overflow (min);
+      if (TREE_OVERFLOW_P (max))
+	max = drop_tree_overflow (max);
+    }
+
   if (!legacy_mode_p ())
     {
       if (kind == VR_RANGE)
diff --git a/gcc/value-relation.cc b/gcc/value-relation.cc
index ac5f3f9..9c3ecd9 100644
--- a/gcc/value-relation.cc
+++ b/gcc/value-relation.cc
@@ -877,7 +877,13 @@ relation_oracle::register_edge (edge e, relation_kind k, tree op1, tree op2)
 void
 dom_oracle::register_relation (basic_block bb, relation_kind k, tree op1,
 			       tree op2)
-{  // Equivalencies are handled by the equivalence oracle.
+{
+  // If the 2 ssa_names are the same, do nothing.  An equivalence is implied,
+  // and no other relation makes sense.
+  if (op1 == op2)
+    return;
+
+  // Equivalencies are handled by the equivalence oracle.
   if (k == EQ_EXPR)
     equiv_oracle::register_relation (bb, k, op1, op2);
   else
@@ -1229,6 +1235,7 @@ path_oracle::path_oracle (relation_oracle *oracle)
   m_equiv.m_next = NULL;
   m_relations.m_names = BITMAP_ALLOC (&m_bitmaps);
   m_relations.m_head = NULL;
+  m_killed_defs = BITMAP_ALLOC (&m_bitmaps);
 }
 
 path_oracle::~path_oracle ()
@@ -1285,6 +1292,61 @@ path_oracle::register_equiv (basic_block bb, tree ssa1, tree ssa2)
   bitmap_ior_into (m_equiv.m_names, b);
 }
 
+// Register killing definition of an SSA_NAME.
+
+void
+path_oracle::killing_def (tree ssa)
+{
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, " Registering killing_def (path_oracle) ");
+      print_generic_expr (dump_file, ssa, TDF_SLIM);
+      fprintf (dump_file, "\n");
+    }
+
+  unsigned v = SSA_NAME_VERSION (ssa);
+
+  bitmap_set_bit (m_killed_defs, v);
+
+  // Walk the equivalency list and remove SSA from any equivalencies.
+  if (bitmap_bit_p (m_equiv.m_names, v))
+    {
+      for (equiv_chain *ptr = m_equiv.m_next; ptr; ptr = ptr->m_next)
+	if (bitmap_bit_p (ptr->m_names, v))
+	  bitmap_clear_bit (ptr->m_names, v);
+    }
+  else
+    bitmap_set_bit (m_equiv.m_names, v);
+
+  // Now add an equivalency with itself so we don't look to the root oracle.
+  bitmap b = BITMAP_ALLOC (&m_bitmaps);
+  bitmap_set_bit (b, v);
+  equiv_chain *ptr = (equiv_chain *) obstack_alloc (&m_chain_obstack,
+						    sizeof (equiv_chain));
+  ptr->m_names = b;
+  ptr->m_bb = NULL;
+  ptr->m_next = m_equiv.m_next;
+  m_equiv.m_next = ptr;
+
+  // Walk the relation list and remove SSA from any relations.
+  if (!bitmap_bit_p (m_relations.m_names, v))
+    return;
+
+  bitmap_clear_bit (m_relations.m_names, v);
+  relation_chain **prev = &(m_relations.m_head);
+  relation_chain *next = NULL;
+  for (relation_chain *ptr = m_relations.m_head; ptr; ptr = next)
+    {
+      gcc_checking_assert (*prev == ptr);
+      next = ptr->m_next;
+      if (SSA_NAME_VERSION (ptr->op1 ()) == v
+	  || SSA_NAME_VERSION (ptr->op2 ()) == v)
+	*prev = ptr->m_next;
+      else
+	prev = &(ptr->m_next);
+    }
+}
+
 // Register relation K between SSA1 and SSA2, resolving unknowns by
 // querying from BB.
 
@@ -1330,6 +1392,12 @@ path_oracle::query_relation (basic_block bb, const_bitmap b1, const_bitmap b2)
 
   relation_kind k = m_relations.find_relation (b1, b2);
 
+  // Do not look at the root oracle for names that have been killed
+  // along the path.
+  if (bitmap_intersect_p (m_killed_defs, b1)
+      || bitmap_intersect_p (m_killed_defs, b2))
+    return k;
+
   if (k == VREL_NONE && m_root)
     k = m_root->query_relation (bb, b1, b2);
 
@@ -1380,10 +1448,14 @@ void
 path_oracle::dump (FILE *f) const
 {
   equiv_chain *ptr = m_equiv.m_next;
+  relation_chain *ptr2 = m_relations.m_head;
+
+  if (ptr || ptr2)
+    fprintf (f, "\npath_oracle:\n");
+
   for (; ptr; ptr = ptr->m_next)
     ptr->dump (f);
 
-  relation_chain *ptr2 = m_relations.m_head;
   for (; ptr2; ptr2 = ptr2->m_next)
     {
       fprintf (f, "Relational : ");
diff --git a/gcc/value-relation.h b/gcc/value-relation.h
index 53cefbf..8086f55 100644
--- a/gcc/value-relation.h
+++ b/gcc/value-relation.h
@@ -222,6 +222,7 @@ public:
   ~path_oracle ();
   const_bitmap equiv_set (tree, basic_block);
   void register_relation (basic_block, relation_kind, tree, tree);
+  void killing_def (tree);
   relation_kind query_relation (basic_block, tree, tree);
   relation_kind query_relation (basic_block, const_bitmap, const_bitmap);
   void reset_path ();
@@ -232,6 +233,7 @@ private:
   equiv_chain m_equiv;
   relation_chain_head m_relations;
   relation_oracle *m_root;
+  bitmap m_killed_defs;
 
   bitmap_obstack m_bitmaps;
   struct obstack m_chain_obstack;
diff --git a/gcc/vr-values.c b/gcc/vr-values.c
index d0f7cb4..ea925f7 100644
--- a/gcc/vr-values.c
+++ b/gcc/vr-values.c
@@ -3495,12 +3495,18 @@ simplify_using_ranges::fold_cond (gcond *cond)
       if (TREE_CODE (gimple_cond_lhs (cond)) != SSA_NAME
 	  && TREE_CODE (gimple_cond_rhs (cond)) != SSA_NAME)
 	return false;
+      if (dump_file)
+	{
+	  fprintf (dump_file, "Folding predicate ");
+	  print_gimple_expr (dump_file, cond, 0);
+	  fprintf (dump_file, " to ");
+	}
       edge e0 = EDGE_SUCC (gimple_bb (cond), 0);
       edge e1 = EDGE_SUCC (gimple_bb (cond), 1);
       if (r.zero_p ())
 	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    fprintf (dump_file, "\nPredicate evaluates to: 0\n");
+	  if (dump_file)
+	    fprintf (dump_file, "0\n");
 	  gimple_cond_make_false (cond);
 	  if (e0->flags & EDGE_TRUE_VALUE)
 	    set_and_propagate_unexecutable (e0);
@@ -3509,8 +3515,8 @@ simplify_using_ranges::fold_cond (gcond *cond)
 	}
       else
 	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    fprintf (dump_file, "\nPredicate evaluates to: 1\n");
+	  if (dump_file)
+	    fprintf (dump_file, "1\n");
 	  gimple_cond_make_true (cond);
 	  if (e0->flags & EDGE_FALSE_VALUE)
 	    set_and_propagate_unexecutable (e0);
@@ -3630,6 +3636,75 @@ simplify_using_ranges::simplify_cond_using_ranges_1 (gcond *stmt)
 	    }
 	}
     }
+  // Try to simplify casted conditions.
+  return simplify_casted_cond (stmt);
+}
+
+/* STMT is a conditional at the end of a basic block.
+
+   If the conditional is of the form SSA_NAME op constant and the SSA_NAME
+   was set via a type conversion, try to replace the SSA_NAME with the RHS
+   of the type conversion.  Doing so makes the conversion dead which helps
+   subsequent passes.  */
+
+bool
+simplify_using_ranges::simplify_casted_cond (gcond *stmt)
+{
+  tree op0 = gimple_cond_lhs (stmt);
+  tree op1 = gimple_cond_rhs (stmt);
+
+  /* If we have a comparison of an SSA_NAME (OP0) against a constant,
+     see if OP0 was set by a type conversion where the source of
+     the conversion is another SSA_NAME with a range that fits
+     into the range of OP0's type.
+
+     If so, the conversion is redundant as the earlier SSA_NAME can be
+     used for the comparison directly if we just massage the constant in the
+     comparison.  */
+  if (TREE_CODE (op0) == SSA_NAME
+      && TREE_CODE (op1) == INTEGER_CST)
+    {
+      gimple *def_stmt = SSA_NAME_DEF_STMT (op0);
+      tree innerop;
+
+      if (!is_gimple_assign (def_stmt))
+	return false;
+
+      switch (gimple_assign_rhs_code (def_stmt))
+	{
+	CASE_CONVERT:
+	  innerop = gimple_assign_rhs1 (def_stmt);
+	  break;
+	case VIEW_CONVERT_EXPR:
+	  innerop = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
+	  if (!INTEGRAL_TYPE_P (TREE_TYPE (innerop)))
+	    return false;
+	  break;
+	default:
+	  return false;
+	}
+
+      if (TREE_CODE (innerop) == SSA_NAME
+	  && !POINTER_TYPE_P (TREE_TYPE (innerop))
+	  && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (innerop)
+	  && desired_pro_or_demotion_p (TREE_TYPE (innerop), TREE_TYPE (op0)))
+	{
+	  const value_range *vr = query->get_value_range (innerop);
+
+	  if (range_int_cst_p (vr)
+	      && range_fits_type_p (vr,
+				    TYPE_PRECISION (TREE_TYPE (op0)),
+				    TYPE_SIGN (TREE_TYPE (op0)))
+	      && int_fits_type_p (op1, TREE_TYPE (innerop)))
+	    {
+	      tree newconst = fold_convert (TREE_TYPE (innerop), op1);
+	      gimple_cond_set_lhs (stmt, innerop);
+	      gimple_cond_set_rhs (stmt, newconst);
+	      update_stmt (stmt);
+	      return true;
+	    }
+	}
+    }
   return false;
 }
 
diff --git a/gcc/vr-values.h b/gcc/vr-values.h
index 821bcb9..21dc4ea 100644
--- a/gcc/vr-values.h
+++ b/gcc/vr-values.h
@@ -44,6 +44,7 @@ public:
   tree vrp_evaluate_conditional_warnv_with_ops (gimple *stmt, enum tree_code,
 						tree, tree, bool,
 						bool *, bool *);
+  bool simplify_casted_cond (gcond *);
 
 private:
   bool simplify_truth_ops_using_ranges (gimple_stmt_iterator *, gimple *);
diff --git a/gnattools/ChangeLog b/gnattools/ChangeLog
index d86a991..fb2f11c 100644
--- a/gnattools/ChangeLog
+++ b/gnattools/ChangeLog
@@ -1,3 +1,8 @@
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Makefile.in: Add dummy install-dvi target.
+
 2021-05-07  Bob Duff  <duff@adacore.com>
 
 	* Makefile.in (GENERATED_FILES_FOR_TOOLS): New variable.
diff --git a/gnattools/Makefile.in b/gnattools/Makefile.in
index 055a269..e8fc4af 100644
--- a/gnattools/Makefile.in
+++ b/gnattools/Makefile.in
@@ -259,9 +259,11 @@ install-info:
 
 install-pdf:
 
+install-dvi:
+
 install-html:
 
-.PHONY: install install-strip install-info install-pdf install-html
+.PHONY: install install-strip install-info install-pdf install-dvi install-html
 
 # Cleaning rules.
 mostlyclean:
diff --git a/include/ChangeLog b/include/ChangeLog
index 9a639c0..67c0672 100644
--- a/include/ChangeLog
+++ b/include/ChangeLog
@@ -1,3 +1,16 @@
+2021-11-12  Chung-Lin Tang  <cltang@codesourcery.com>
+
+	* gomp-constants.h (GOMP_MAP_FLAG_SPECIAL_3): Define special bit macro.
+	(GOMP_MAP_IMPLICIT): New special map kind bits value.
+	(GOMP_MAP_FLAG_SPECIAL_BITS): Define helper mask for whole set of
+	special map kind bits.
+	(GOMP_MAP_IMPLICIT_P): New predicate macro for implicit map kinds.
+
+2021-11-05  Gerald Pfeifer  <gerald@pfeifer.com>
+	    Jakub Jelinek  <jakub@redhat.com>
+
+	* md5.h (USE_SYSTEM_MD5): Introduce.
+
 2021-09-27  Vincent Lefevre  <vincent@vinc17.net>
 
 	* floatformat.h: Update pathname for IBM long double description.
diff --git a/include/gomp-constants.h b/include/gomp-constants.h
index ebd0801..3e42d71 100644
--- a/include/gomp-constants.h
+++ b/include/gomp-constants.h
@@ -40,11 +40,22 @@
 #define GOMP_MAP_FLAG_SPECIAL_0		(1 << 2)
 #define GOMP_MAP_FLAG_SPECIAL_1		(1 << 3)
 #define GOMP_MAP_FLAG_SPECIAL_2		(1 << 4)
+#define GOMP_MAP_FLAG_SPECIAL_3		(1 << 5)
 #define GOMP_MAP_FLAG_SPECIAL_4		(1 << 6)
 #define GOMP_MAP_FLAG_SPECIAL		(GOMP_MAP_FLAG_SPECIAL_1 \
 					 | GOMP_MAP_FLAG_SPECIAL_0)
 #define GOMP_MAP_DEEP_COPY		(GOMP_MAP_FLAG_SPECIAL_4 \
 					 | GOMP_MAP_FLAG_SPECIAL_2)
+/* This value indicates the map was created implicitly according to
+   OpenMP rules.  */
+#define GOMP_MAP_IMPLICIT		(GOMP_MAP_FLAG_SPECIAL_3 \
+					 | GOMP_MAP_FLAG_SPECIAL_4)
+/* Mask for entire set of special map kind bits.  */
+#define GOMP_MAP_FLAG_SPECIAL_BITS	(GOMP_MAP_FLAG_SPECIAL_0 \
+					 | GOMP_MAP_FLAG_SPECIAL_1 \
+					 | GOMP_MAP_FLAG_SPECIAL_2 \
+					 | GOMP_MAP_FLAG_SPECIAL_3 \
+					 | GOMP_MAP_FLAG_SPECIAL_4)
 /* Flag to force a specific behavior (or else, trigger a run-time error).  */
 #define GOMP_MAP_FLAG_FORCE		(1 << 7)
 
@@ -186,6 +197,9 @@ enum gomp_map_kind
 #define GOMP_MAP_ALWAYS_P(X) \
   (GOMP_MAP_ALWAYS_TO_P (X) || ((X) == GOMP_MAP_ALWAYS_FROM))
 
+#define GOMP_MAP_IMPLICIT_P(X) \
+  (((X) & GOMP_MAP_FLAG_SPECIAL_BITS) == GOMP_MAP_IMPLICIT)
+
 
 /* Asynchronous behavior.  Keep in sync with
    libgomp/{openacc.h,openacc.f90,openacc_lib.h}:acc_async_t.  */
diff --git a/include/md5.h b/include/md5.h
index 03f7d29..c5bb607 100644
--- a/include/md5.h
+++ b/include/md5.h
@@ -21,6 +21,10 @@
 #ifndef _MD5_H
 #define _MD5_H 1
 
+#ifdef USE_SYSTEM_MD5
+#include_next <md5.h>
+#else
+
 #include <stdio.h>
 
 #if defined HAVE_LIMITS_H || _LIBC
@@ -151,4 +155,6 @@ extern void *md5_buffer (const char *buffer, size_t len, void *resblock);
 }
 #endif
 
+#endif // USE_SYSTEM_MD5
+
 #endif
diff --git a/libada/ChangeLog b/libada/ChangeLog
index 441d15a..1b75702 100644
--- a/libada/ChangeLog
+++ b/libada/ChangeLog
@@ -1,3 +1,8 @@
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Makefile.in: Add dummy install-dvi target.
+
 2020-01-24  Maciej W. Rozycki  <macro@wdc.com>
 
 	* Makefile.in (configure_deps): Add `toolexeclibdir.m4'.
diff --git a/libada/Makefile.in b/libada/Makefile.in
index cfa9048..85038e0 100644
--- a/libada/Makefile.in
+++ b/libada/Makefile.in
@@ -151,9 +151,11 @@ install-info:
 
 install-pdf:
 
+install-dvi:
+
 install-html:
 
-.PHONY: install install-strip install-info install-pdf install-html
+.PHONY: install install-strip install-info install-pdf install-dvi install-html
 
 # Cleaning rules.
 mostlyclean:
diff --git a/libbacktrace/ChangeLog b/libbacktrace/ChangeLog
index dc986b9..983f816 100644
--- a/libbacktrace/ChangeLog
+++ b/libbacktrace/ChangeLog
@@ -1,3 +1,20 @@
+2021-11-12  Martin Liska  <mliska@suse.cz>
+
+	PR libbacktrace/103167
+	* elf.c (elf_uncompress_lzma_block): Cast to unsigned int.
+	(elf_uncompress_lzma): Likewise.
+	* xztest.c (test_samples): memcpy only if v > 0.
+
+2021-10-22  Martin Liska  <mliska@suse.cz>
+
+	PR testsuite/102742
+	* btest.c (MIN_DESCRIPTOR): New.
+	(MAX_DESCRIPTOR): Likewise.
+	(check_available_files): Likewise.
+	(check_open_files): Check only file descriptors that
+	were not available at the entry.
+	(main): Call check_available_files.
+
 2021-08-13  Sergei Trofimovich  <siarheit@google.com>
 
 	* install-debuginfo-for-buildid.sh.in: Force non-localized readelf
diff --git a/libbacktrace/btest.c b/libbacktrace/btest.c
index 9f9c03b..7ef6d32 100644
--- a/libbacktrace/btest.c
+++ b/libbacktrace/btest.c
@@ -38,6 +38,7 @@ POSSIBILITY OF SUCH DAMAGE.  */
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <sys/stat.h>
 
 #include "filenames.h"
 
@@ -458,16 +459,29 @@ test5 (void)
   return failures;
 }
 
+#define MIN_DESCRIPTOR 3
+#define MAX_DESCRIPTOR 10
+
+static int fstat_status[MAX_DESCRIPTOR];
+
+/* Check files that are available.  */
+
+static void
+check_available_files (void)
+{
+  struct stat s;
+  for (unsigned i = MIN_DESCRIPTOR; i < MAX_DESCRIPTOR; i++)
+    fstat_status[i] = fstat (i, &s);
+}
+
 /* Check that are no files left open.  */
 
 static void
 check_open_files (void)
 {
-  int i;
-
-  for (i = 3; i < 10; i++)
+  for (unsigned i = MIN_DESCRIPTOR; i < MAX_DESCRIPTOR; i++)
     {
-      if (close (i) == 0)
+      if (fstat_status[i] != 0 && close (i) == 0)
 	{
 	  fprintf (stderr,
 		   "ERROR: descriptor %d still open after tests complete\n",
@@ -482,6 +496,8 @@ check_open_files (void)
 int
 main (int argc ATTRIBUTE_UNUSED, char **argv)
 {
+  check_available_files ();
+
   state = backtrace_create_state (argv[0], BACKTRACE_SUPPORTS_THREADS,
 				  error_callback_create, NULL);
 
diff --git a/libbacktrace/elf.c b/libbacktrace/elf.c
index 79d5614..8b87b2d 100644
--- a/libbacktrace/elf.c
+++ b/libbacktrace/elf.c
@@ -3172,10 +3172,10 @@ elf_uncompress_lzma_block (const unsigned char *compressed,
   /* Block header CRC.  */
   computed_crc = elf_crc32 (0, compressed + block_header_offset,
 			    block_header_size - 4);
-  stream_crc = (compressed[off]
-		| (compressed[off + 1] << 8)
-		| (compressed[off + 2] << 16)
-		| (compressed[off + 3] << 24));
+  stream_crc = ((uint32_t)compressed[off]
+		| ((uint32_t)compressed[off + 1] << 8)
+		| ((uint32_t)compressed[off + 2] << 16)
+		| ((uint32_t)compressed[off + 3] << 24));
   if (unlikely (computed_crc != stream_crc))
     {
       elf_uncompress_failed ();
@@ -3785,10 +3785,10 @@ elf_uncompress_lzma (struct backtrace_state *state,
 
   /* Next comes a CRC of the stream flags.  */
   computed_crc = elf_crc32 (0, compressed + 6, 2);
-  stream_crc = (compressed[8]
-		| (compressed[9] << 8)
-		| (compressed[10] << 16)
-		| (compressed[11] << 24));
+  stream_crc = ((uint32_t)compressed[8]
+		| ((uint32_t)compressed[9] << 8)
+		| ((uint32_t)compressed[10] << 16)
+		| ((uint32_t)compressed[11] << 24));
   if (unlikely (computed_crc != stream_crc))
     {
       elf_uncompress_failed ();
@@ -3829,10 +3829,10 @@ elf_uncompress_lzma (struct backtrace_state *state,
 
   /* Before that is a footer CRC.  */
   computed_crc = elf_crc32 (0, compressed + offset, 6);
-  stream_crc = (compressed[offset - 4]
-		| (compressed[offset - 3] << 8)
-		| (compressed[offset - 2] << 16)
-		| (compressed[offset - 1] << 24));
+  stream_crc = ((uint32_t)compressed[offset - 4]
+		| ((uint32_t)compressed[offset - 3] << 8)
+		| ((uint32_t)compressed[offset - 2] << 16)
+		| ((uint32_t)compressed[offset - 1] << 24));
   if (unlikely (computed_crc != stream_crc))
     {
       elf_uncompress_failed ();
@@ -3888,10 +3888,10 @@ elf_uncompress_lzma (struct backtrace_state *state,
   /* Next is a CRC of the index.  */
   computed_crc = elf_crc32 (0, compressed + index_offset,
 			    offset - index_offset);
-  stream_crc = (compressed[offset]
-		| (compressed[offset + 1] << 8)
-		| (compressed[offset + 2] << 16)
-		| (compressed[offset + 3] << 24));
+  stream_crc = ((uint32_t)compressed[offset]
+		| ((uint32_t)compressed[offset + 1] << 8)
+		| ((uint32_t)compressed[offset + 2] << 16)
+		| ((uint32_t)compressed[offset + 3] << 24));
   if (unlikely (computed_crc != stream_crc))
     {
       elf_uncompress_failed ();
diff --git a/libbacktrace/xztest.c b/libbacktrace/xztest.c
index b2533cb..6c60ff5 100644
--- a/libbacktrace/xztest.c
+++ b/libbacktrace/xztest.c
@@ -172,7 +172,7 @@ test_samples (struct backtrace_state *state)
 		       tests[i].name, uncompressed_len, v);
 	      ++failures;
 	    }
-	  else if (memcmp (tests[i].uncompressed, uncompressed, v) != 0)
+	  else if (v > 0 && memcmp (tests[i].uncompressed, uncompressed, v) != 0)
 	    {
 	      size_t j;
 
diff --git a/libcody/ChangeLog b/libcody/ChangeLog
index 0582696..3fec73f 100644
--- a/libcody/ChangeLog
+++ b/libcody/ChangeLog
@@ -1,3 +1,13 @@
+2021-11-01  Martin Liska  <mliska@suse.cz>
+
+	PR other/102657
+	* Makefile.in: Add mostlyclean Makefile target.
+
+2021-10-21  Jonathan Wakely  <jwakely@redhat.com>
+
+	* netserver.cc (ListenInet6): Set pointer to null after
+	deallocation.
+
 2021-04-06  Nathan Sidwell  <nathan@acm.org>
 
 	* LICENSE.gcc: Delete.
diff --git a/libcody/Makefile.in b/libcody/Makefile.in
index b8b45a2..7eaf8ac 100644
--- a/libcody/Makefile.in
+++ b/libcody/Makefile.in
@@ -60,6 +60,8 @@ vpath %.cc $(srcdir)
 
 all:: Makefile
 
+mostlyclean::
+
 clean:: Makefile
 
 # FIXME: Delete
@@ -77,7 +79,9 @@ revision.stamp: $(srcdir)/.
 revision: revision.stamp
 	@cmp -s $< $@ || cp -f $< $@
 
-clean::
+mostlyclean::
+
+clean:: mostlyclean
 	rm -f revision.stamp revision
 
 distclean:: clean
@@ -102,16 +106,18 @@ endif
 config.status: $(srcdir)/configure $(srcdir)/config.h.in
 	if test -x $@; then ./$@ -recheck; else $< @configure_args@; fi
 
+mostlyclean::
+
+clean:: mostlyclean
+	rm -f $(shell find $(srcdir) -name '*~')
+
 distclean:: clean
 	rm -f config.h
 
 maintainer-clean:: distclean
 	rm -f $(srcdir)/config.h.in
 
-clean::
-	rm -f $(shell find $(srcdir) -name '*~')
-
-.PHONY: all check clean distclean maintainer-clean
+.PHONY: all check clean distclean maintainer-clean mostlyclean
 
 CXXFLAGS/ := -I$(srcdir)
 LIBCODY.O := buffer.o client.o fatal.o netclient.o netserver.o \
@@ -123,7 +129,9 @@ libcody.a: $(LIBCODY.O)
 	$(AR) -cr $@ $^
 	$(RANLIB) $@
 
-clean::
+mostlyclean::
+
+clean:: mostlyclean
 	rm -f $(LIBCODY.O) $(LIBCODY.O:.o=.d)
 	rm -f libcody.a
 
diff --git a/libcody/netserver.cc b/libcody/netserver.cc
index 30202c5..0499b57 100644
--- a/libcody/netserver.cc
+++ b/libcody/netserver.cc
@@ -140,6 +140,7 @@ int ListenInet6 (char const **e, char const *name, int port, unsigned backlog)
 
  listen:;
   freeaddrinfo (addrs);
+  addrs = nullptr;
 
   if (listen (fd, backlog ? backlog : 17) < 0)
     {
diff --git a/libcpp/ChangeLog b/libcpp/ChangeLog
index 67d866d..e0e71a4 100644
--- a/libcpp/ChangeLog
+++ b/libcpp/ChangeLog
@@ -1,3 +1,64 @@
+2021-11-01  David Malcolm  <dmalcolm@redhat.com>
+
+	* charset.c (convert_escape): Use encoding_rich_location when
+	complaining about nonprintable unknown escape sequences.
+	(cpp_display_width_computation::::cpp_display_width_computation):
+	Pass in policy rather than tabstop.
+	(cpp_display_width_computation::process_next_codepoint): Add "out"
+	param and populate *out if non-NULL.
+	(cpp_display_width_computation::advance_display_cols): Pass NULL
+	to process_next_codepoint.
+	(cpp_byte_column_to_display_column): Pass in policy rather than
+	tabstop.  Pass NULL to process_next_codepoint.
+	(cpp_display_column_to_byte_column): Pass in policy rather than
+	tabstop.
+	* errors.c (cpp_diagnostic_get_current_location): New function,
+	splitting out the logic from...
+	(cpp_diagnostic): ...here.
+	(cpp_warning_at): New function.
+	(cpp_pedwarning_at): New function.
+	* include/cpplib.h (cpp_warning_at): New decl for rich_location.
+	(cpp_pedwarning_at): Likewise.
+	(struct cpp_decoded_char): New.
+	(struct cpp_char_column_policy): New.
+	(cpp_display_width_computation::cpp_display_width_computation):
+	Replace "tabstop" param with "policy".
+	(cpp_display_width_computation::process_next_codepoint): Add "out"
+	param.
+	(cpp_display_width_computation::m_tabstop): Replace with...
+	(cpp_display_width_computation::m_policy): ...this.
+	(cpp_byte_column_to_display_column): Replace "tabstop" param with
+	"policy".
+	(cpp_display_width): Likewise.
+	(cpp_display_column_to_byte_column): Likewise.
+	* include/line-map.h (rich_location::escape_on_output_p): New.
+	(rich_location::set_escape_on_output): New.
+	(rich_location::m_escape_on_output): New.
+	* internal.h (cpp_diagnostic_get_current_location): New decl.
+	(class encoding_rich_location): New.
+	* lex.c (skip_whitespace): Use encoding_rich_location when
+	complaining about null characters.
+	(warn_about_normalization): Generate a source range when
+	complaining about improperly normalized tokens, rather than just a
+	point, and use encoding_rich_location so that the source code
+	is escaped on printing.
+	* line-map.c (rich_location::rich_location): Initialize
+	m_escape_on_output.
+
+2021-10-29  Tobias Burnus  <tobias@codesourcery.com>
+
+	PR c++/102409
+	* directives.c (destringize_and_run): Add PRAGMA_OP to the
+	CPP_PRAGMA token's flags to mark is as coming from _Pragma.
+	* include/cpplib.h (PRAGMA_OP): #define, to be used with token flags.
+	* macro.c (collect_args): Only handle CPP_PRAGMA special if PRAGMA_OP
+	is set.
+
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Makefile.in: Add dummy install-dvi target.
+
 2021-10-18  Martin Liska  <mliska@suse.cz>
 
 	* charset.c (convert_oct): Remove unused but set variable.
diff --git a/libcpp/Makefile.in b/libcpp/Makefile.in
index 6c403a8..34e4206 100644
--- a/libcpp/Makefile.in
+++ b/libcpp/Makefile.in
@@ -195,6 +195,7 @@ html:
 info:
 install-info:
 install-pdf:
+install-dvi:
 install-man:
 install-html:
 
@@ -202,7 +203,7 @@ update-po: $(CATALOGS:.gmo=.pox)
 
 .PHONY: installdirs install install-strip mostlyclean clean distclean \
   maintainer-clean check installcheck dvi pdf html info install-info \
-  install-man update-po install-html
+  install-man update-po install-html install-pdf install-dvi
 
 # Dependency rule.
 COMPILE.base = $(COMPILER) $(DEFS) $(INCLUDES) $(CPPFLAGS) $(COMPILER_FLAGS) -c
diff --git a/libcpp/charset.c b/libcpp/charset.c
index e4e45f6..0b0ccc6 100644
--- a/libcpp/charset.c
+++ b/libcpp/charset.c
@@ -1582,12 +1582,14 @@ convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
 		   "unknown escape sequence: '\\%c'", (int) c);
       else
 	{
+	  encoding_rich_location rich_loc (pfile);
+
 	  /* diagnostic.c does not support "%03o".  When it does, this
 	     code can use %03o directly in the diagnostic again.  */
 	  char buf[32];
 	  sprintf(buf, "%03o", (int) c);
-	  cpp_error (pfile, CPP_DL_PEDWARN,
-		     "unknown escape sequence: '\\%s'", buf);
+	  cpp_error_at (pfile, CPP_DL_PEDWARN, &rich_loc,
+			"unknown escape sequence: '\\%s'", buf);
 	}
     }
 
@@ -2345,14 +2347,16 @@ cpp_string_location_reader::get_next ()
 }
 
 cpp_display_width_computation::
-cpp_display_width_computation (const char *data, int data_length, int tabstop) :
+cpp_display_width_computation (const char *data, int data_length,
+			       const cpp_char_column_policy &policy) :
   m_begin (data),
   m_next (m_begin),
   m_bytes_left (data_length),
-  m_tabstop (tabstop),
+  m_policy (policy),
   m_display_cols (0)
 {
-  gcc_assert (m_tabstop > 0);
+  gcc_assert (policy.m_tabstop > 0);
+  gcc_assert (policy.m_width_cb);
 }
 
 
@@ -2364,19 +2368,28 @@ cpp_display_width_computation (const char *data, int data_length, int tabstop) :
    point to a valid UTF-8-encoded sequence, then it will be treated as a single
    byte with display width 1.  m_cur_display_col is the current display column,
    relative to which tab stops should be expanded.  Returns the display width of
-   the codepoint just processed.  */
+   the codepoint just processed.
+   If OUT is non-NULL, it is populated.  */
 
 int
-cpp_display_width_computation::process_next_codepoint ()
+cpp_display_width_computation::process_next_codepoint (cpp_decoded_char *out)
 {
   cppchar_t c;
   int next_width;
 
+  if (out)
+    out->m_start_byte = m_next;
+
   if (*m_next == '\t')
     {
       ++m_next;
       --m_bytes_left;
-      next_width = m_tabstop - (m_display_cols % m_tabstop);
+      next_width = m_policy.m_tabstop - (m_display_cols % m_policy.m_tabstop);
+      if (out)
+	{
+	  out->m_ch = '\t';
+	  out->m_valid_ch = true;
+	}
     }
   else if (one_utf8_to_cppchar ((const uchar **) &m_next, &m_bytes_left, &c)
 	   != 0)
@@ -2386,14 +2399,24 @@ cpp_display_width_computation::process_next_codepoint ()
 	 of one.  */
       ++m_next;
       --m_bytes_left;
-      next_width = 1;
+      next_width = m_policy.m_undecoded_byte_width;
+      if (out)
+	out->m_valid_ch = false;
     }
   else
     {
       /*  one_utf8_to_cppchar() has updated m_next and m_bytes_left for us.  */
-      next_width = cpp_wcwidth (c);
+      next_width = m_policy.m_width_cb (c);
+      if (out)
+	{
+	  out->m_ch = c;
+	  out->m_valid_ch = true;
+	}
     }
 
+  if (out)
+    out->m_next_byte = m_next;
+
   m_display_cols += next_width;
   return next_width;
 }
@@ -2409,7 +2432,7 @@ cpp_display_width_computation::advance_display_cols (int n)
   const int start = m_display_cols;
   const int target = start + n;
   while (m_display_cols < target && !done ())
-    process_next_codepoint ();
+    process_next_codepoint (NULL);
   return m_display_cols - start;
 }
 
@@ -2417,29 +2440,33 @@ cpp_display_width_computation::advance_display_cols (int n)
     how many display columns are occupied by the first COLUMN bytes.  COLUMN
     may exceed DATA_LENGTH, in which case the phantom bytes at the end are
     treated as if they have display width 1.  Tabs are expanded to the next tab
-    stop, relative to the start of DATA.  */
+    stop, relative to the start of DATA, and non-printable-ASCII characters
+    will be escaped as per POLICY.  */
 
 int
 cpp_byte_column_to_display_column (const char *data, int data_length,
-				   int column, int tabstop)
+				   int column,
+				   const cpp_char_column_policy &policy)
 {
   const int offset = MAX (0, column - data_length);
-  cpp_display_width_computation dw (data, column - offset, tabstop);
+  cpp_display_width_computation dw (data, column - offset, policy);
   while (!dw.done ())
-    dw.process_next_codepoint ();
+    dw.process_next_codepoint (NULL);
   return dw.display_cols_processed () + offset;
 }
 
 /*  For the string of length DATA_LENGTH bytes that begins at DATA, compute
     the least number of bytes that will result in at least DISPLAY_COL display
     columns.  The return value may exceed DATA_LENGTH if the entire string does
-    not occupy enough display columns.  */
+    not occupy enough display columns.  Non-printable-ASCII characters
+    will be escaped as per POLICY.  */
 
 int
 cpp_display_column_to_byte_column (const char *data, int data_length,
-				   int display_col, int tabstop)
+				   int display_col,
+				   const cpp_char_column_policy &policy)
 {
-  cpp_display_width_computation dw (data, data_length, tabstop);
+  cpp_display_width_computation dw (data, data_length, policy);
   const int avail_display = dw.advance_display_cols (display_col);
   return dw.bytes_processed () + MAX (0, display_col - avail_display);
 }
diff --git a/libcpp/directives.c b/libcpp/directives.c
index b4bc8b4..34f7677 100644
--- a/libcpp/directives.c
+++ b/libcpp/directives.c
@@ -1907,6 +1907,8 @@ destringize_and_run (cpp_reader *pfile, const cpp_string *in,
   save_directive = pfile->directive;
   pfile->directive = &dtable[T_PRAGMA];
   do_pragma (pfile);
+  if (pfile->directive_result.type == CPP_PRAGMA)
+    pfile->directive_result.flags |= PRAGMA_OP;
   end_directive (pfile, 1);
   pfile->directive = save_directive;
 
diff --git a/libcpp/errors.c b/libcpp/errors.c
index 5e1bf33..f34334a 100644
--- a/libcpp/errors.c
+++ b/libcpp/errors.c
@@ -27,6 +27,31 @@ along with this program; see the file COPYING3.  If not see
 #include "cpplib.h"
 #include "internal.h"
 
+/* Get a location_t for the current location in PFILE,
+   generally that of the previously lexed token.  */
+
+location_t
+cpp_diagnostic_get_current_location (cpp_reader *pfile)
+{
+  if (CPP_OPTION (pfile, traditional))
+    {
+      if (pfile->state.in_directive)
+	return pfile->directive_line;
+      else
+	return pfile->line_table->highest_line;
+    }
+  /* We don't want to refer to a token before the beginning of the
+     current run -- that is invalid.  */
+  else if (pfile->cur_token == pfile->cur_run->base)
+    {
+      return 0;
+    }
+  else
+    {
+      return pfile->cur_token[-1].src_loc;
+    }
+}
+
 /* Print a diagnostic at the given location.  */
 
 ATTRIBUTE_FPTR_PRINTF(5,0)
@@ -52,25 +77,7 @@ cpp_diagnostic (cpp_reader * pfile, enum cpp_diagnostic_level level,
 		enum cpp_warning_reason reason,
 		const char *msgid, va_list *ap)
 {
-  location_t src_loc;
-
-  if (CPP_OPTION (pfile, traditional))
-    {
-      if (pfile->state.in_directive)
-	src_loc = pfile->directive_line;
-      else
-	src_loc = pfile->line_table->highest_line;
-    }
-  /* We don't want to refer to a token before the beginning of the
-     current run -- that is invalid.  */
-  else if (pfile->cur_token == pfile->cur_run->base)
-    {
-      src_loc = 0;
-    }
-  else
-    {
-      src_loc = pfile->cur_token[-1].src_loc;
-    }
+  location_t src_loc = cpp_diagnostic_get_current_location (pfile);
   rich_location richloc (pfile->line_table, src_loc);
   return cpp_diagnostic_at (pfile, level, reason, &richloc, msgid, ap);
 }
@@ -144,6 +151,43 @@ cpp_warning_syshdr (cpp_reader * pfile, enum cpp_warning_reason reason,
   return ret;
 }
 
+/* As cpp_warning above, but use RICHLOC as the location of the diagnostic.  */
+
+bool cpp_warning_at (cpp_reader *pfile, enum cpp_warning_reason reason,
+		     rich_location *richloc, const char *msgid, ...)
+{
+  va_list ap;
+  bool ret;
+
+  va_start (ap, msgid);
+
+  ret = cpp_diagnostic_at (pfile, CPP_DL_WARNING, reason, richloc,
+			   msgid, &ap);
+
+  va_end (ap);
+  return ret;
+
+}
+
+/* As cpp_pedwarning above, but use RICHLOC as the location of the
+   diagnostic.  */
+
+bool
+cpp_pedwarning_at (cpp_reader * pfile, enum cpp_warning_reason reason,
+		   rich_location *richloc, const char *msgid, ...)
+{
+  va_list ap;
+  bool ret;
+
+  va_start (ap, msgid);
+
+  ret = cpp_diagnostic_at (pfile, CPP_DL_PEDWARN, reason, richloc,
+			   msgid, &ap);
+
+  va_end (ap);
+  return ret;
+}
+
 /* Print a diagnostic at a specific location.  */
 
 ATTRIBUTE_FPTR_PRINTF(6,0)
diff --git a/libcpp/include/cpplib.h b/libcpp/include/cpplib.h
index 6e2fcb6..176f8c5 100644
--- a/libcpp/include/cpplib.h
+++ b/libcpp/include/cpplib.h
@@ -198,6 +198,7 @@ struct GTY(()) cpp_string {
 				    operator, or before this token
 				    after a # operator.  */
 #define NO_EXPAND	(1 << 10) /* Do not macro-expand this token.  */
+#define PRAGMA_OP	(1 << 11) /* _Pragma token.  */
 
 /* Specify which field, if any, of the cpp_token union is used.  */
 
@@ -1267,6 +1268,14 @@ extern bool cpp_warning_syshdr (cpp_reader *, enum cpp_warning_reason reason,
 				const char *msgid, ...)
   ATTRIBUTE_PRINTF_3;
 
+/* As their counterparts above, but use RICHLOC.  */
+extern bool cpp_warning_at (cpp_reader *, enum cpp_warning_reason,
+			    rich_location *richloc, const char *msgid, ...)
+  ATTRIBUTE_PRINTF_4;
+extern bool cpp_pedwarning_at (cpp_reader *, enum cpp_warning_reason,
+			       rich_location *richloc, const char *msgid, ...)
+  ATTRIBUTE_PRINTF_4;
+
 /* Output a diagnostic with "MSGID: " preceding the
    error string of errno.  No location is printed.  */
 extern bool cpp_errno (cpp_reader *, enum cpp_diagnostic_level,
@@ -1441,43 +1450,95 @@ extern const char * cpp_get_userdef_suffix
 
 /* In charset.c */
 
+/* The result of attempting to decode a run of UTF-8 bytes.  */
+
+struct cpp_decoded_char
+{
+  const char *m_start_byte;
+  const char *m_next_byte;
+
+  bool m_valid_ch;
+  cppchar_t m_ch;
+};
+
+/* Information for mapping between code points and display columns.
+
+   This is a tabstop value, along with a callback for getting the
+   widths of characters.  Normally this callback is cpp_wcwidth, but we
+   support other schemes for escaping non-ASCII unicode as a series of
+   ASCII chars when printing the user's source code in diagnostic-show-locus.c
+
+   For example, consider:
+   - the Unicode character U+03C0 "GREEK SMALL LETTER PI" (UTF-8: 0xCF 0x80)
+   - the Unicode character U+1F642 "SLIGHTLY SMILING FACE"
+     (UTF-8: 0xF0 0x9F 0x99 0x82)
+   - the byte 0xBF (a stray trailing byte of a UTF-8 character)
+   Normally U+03C0 would occupy one display column, U+1F642
+   would occupy two display columns, and the stray byte would be
+   printed verbatim as one display column.
+
+   However when escaping them as unicode code points as "<U+03C0>"
+   and "<U+1F642>" they occupy 8 and 9 display columns respectively,
+   and when escaping them as bytes as "<CF><80>" and "<F0><9F><99><82>"
+   they occupy 8 and 16 display columns respectively.  In both cases
+   the stray byte is escaped to <BF> as 4 display columns.  */
+
+struct cpp_char_column_policy
+{
+  cpp_char_column_policy (int tabstop,
+			  int (*width_cb) (cppchar_t c))
+  : m_tabstop (tabstop),
+    m_undecoded_byte_width (1),
+    m_width_cb (width_cb)
+  {}
+
+  int m_tabstop;
+  /* Width in display columns of a stray byte that isn't decodable
+     as UTF-8.  */
+  int m_undecoded_byte_width;
+  int (*m_width_cb) (cppchar_t c);
+};
+
 /* A class to manage the state while converting a UTF-8 sequence to cppchar_t
    and computing the display width one character at a time.  */
 class cpp_display_width_computation {
  public:
   cpp_display_width_computation (const char *data, int data_length,
-				 int tabstop);
+				 const cpp_char_column_policy &policy);
   const char *next_byte () const { return m_next; }
   int bytes_processed () const { return m_next - m_begin; }
   int bytes_left () const { return m_bytes_left; }
   bool done () const { return !bytes_left (); }
   int display_cols_processed () const { return m_display_cols; }
 
-  int process_next_codepoint ();
+  int process_next_codepoint (cpp_decoded_char *out);
   int advance_display_cols (int n);
 
  private:
   const char *const m_begin;
   const char *m_next;
   size_t m_bytes_left;
-  const int m_tabstop;
+  const cpp_char_column_policy &m_policy;
   int m_display_cols;
 };
 
 /* Convenience functions that are simple use cases for class
    cpp_display_width_computation.  Tab characters will be expanded to spaces
-   as determined by TABSTOP.  */
+   as determined by POLICY.m_tabstop, and non-printable-ASCII characters
+   will be escaped as per POLICY.  */
 
 int cpp_byte_column_to_display_column (const char *data, int data_length,
-				       int column, int tabstop);
+				       int column,
+				       const cpp_char_column_policy &policy);
 inline int cpp_display_width (const char *data, int data_length,
-			      int tabstop)
+			      const cpp_char_column_policy &policy)
 {
   return cpp_byte_column_to_display_column (data, data_length, data_length,
-					    tabstop);
+					    policy);
 }
 int cpp_display_column_to_byte_column (const char *data, int data_length,
-				       int display_col, int tabstop);
+				       int display_col,
+				       const cpp_char_column_policy &policy);
 int cpp_wcwidth (cppchar_t c);
 
 bool cpp_input_conversion_is_trivial (const char *input_charset);
diff --git a/libcpp/include/line-map.h b/libcpp/include/line-map.h
index 464494b..8b5e2f8 100644
--- a/libcpp/include/line-map.h
+++ b/libcpp/include/line-map.h
@@ -1787,6 +1787,18 @@ class rich_location
   const diagnostic_path *get_path () const { return m_path; }
   void set_path (const diagnostic_path *path) { m_path = path; }
 
+  /* A flag for hinting that the diagnostic involves character encoding
+     issues, and thus that it will be helpful to the user if we show some
+     representation of how the characters in the pertinent source lines
+     are encoded.
+     The default is false (i.e. do not escape).
+     When set to true, non-ASCII bytes in the pertinent source lines will
+     be escaped in a manner controlled by the user-supplied option
+     -fdiagnostics-escape-format=, so that the user can better understand
+     what's going on with the encoding in their source file.  */
+  bool escape_on_output_p () const { return m_escape_on_output; }
+  void set_escape_on_output (bool flag) { m_escape_on_output = flag; }
+
 private:
   bool reject_impossible_fixit (location_t where);
   void stop_supporting_fixits ();
@@ -1813,6 +1825,7 @@ protected:
   bool m_fixits_cannot_be_auto_applied;
 
   const diagnostic_path *m_path;
+  bool m_escape_on_output;
 };
 
 /* A struct for the result of range_label::get_text: a NUL-terminated buffer
diff --git a/libcpp/internal.h b/libcpp/internal.h
index fd44de6..8577cab 100644
--- a/libcpp/internal.h
+++ b/libcpp/internal.h
@@ -769,6 +769,9 @@ extern void _cpp_do_file_change (cpp_reader *, enum lc_reason, const char *,
 extern void _cpp_pop_buffer (cpp_reader *);
 extern char *_cpp_bracket_include (cpp_reader *);
 
+/* In errors.c  */
+extern location_t cpp_diagnostic_get_current_location (cpp_reader *);
+
 /* In traditional.c.  */
 extern bool _cpp_scan_out_logical_line (cpp_reader *, cpp_macro *, bool);
 extern bool _cpp_read_logical_line_trad (cpp_reader *);
@@ -935,6 +938,26 @@ int linemap_get_expansion_line (class line_maps *,
 const char* linemap_get_expansion_filename (class line_maps *,
 					    location_t);
 
+/* A subclass of rich_location for emitting a diagnostic
+   at the current location of the reader, but flagging
+   it with set_escape_on_output (true).  */
+class encoding_rich_location : public rich_location
+{
+ public:
+  encoding_rich_location (cpp_reader *pfile)
+  : rich_location (pfile->line_table,
+		   cpp_diagnostic_get_current_location (pfile))
+  {
+    set_escape_on_output (true);
+  }
+
+  encoding_rich_location (cpp_reader *pfile, location_t loc)
+  : rich_location (pfile->line_table, loc)
+  {
+    set_escape_on_output (true);
+  }
+};
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/libcpp/lex.c b/libcpp/lex.c
index 8e3ef09..fa2253d 100644
--- a/libcpp/lex.c
+++ b/libcpp/lex.c
@@ -1268,7 +1268,11 @@ skip_whitespace (cpp_reader *pfile, cppchar_t c)
   while (is_nvspace (c));
 
   if (saw_NUL)
-    cpp_error (pfile, CPP_DL_WARNING, "null character(s) ignored");
+    {
+      encoding_rich_location rich_loc (pfile);
+      cpp_error_at (pfile, CPP_DL_WARNING, &rich_loc,
+		    "null character(s) ignored");
+    }
 
   buffer->cur--;
 }
@@ -1297,6 +1301,28 @@ warn_about_normalization (cpp_reader *pfile,
   if (CPP_OPTION (pfile, warn_normalize) < NORMALIZE_STATE_RESULT (s)
       && !pfile->state.skipping)
     {
+      location_t loc = token->src_loc;
+
+      /* If possible, create a location range for the token.  */
+      if (loc >= RESERVED_LOCATION_COUNT
+	  && token->type != CPP_EOF
+	  /* There must be no line notes to process.  */
+	  && (!(pfile->buffer->cur
+		>= pfile->buffer->notes[pfile->buffer->cur_note].pos
+		&& !pfile->overlaid_buffer)))
+	{
+	  source_range tok_range;
+	  tok_range.m_start = loc;
+	  tok_range.m_finish
+	    = linemap_position_for_column (pfile->line_table,
+					   CPP_BUF_COLUMN (pfile->buffer,
+							   pfile->buffer->cur));
+	  loc = COMBINE_LOCATION_DATA (pfile->line_table,
+				       loc, tok_range, NULL);
+	}
+
+      encoding_rich_location rich_loc (pfile, loc);
+
       /* Make sure that the token is printed using UCNs, even
 	 if we'd otherwise happily print UTF-8.  */
       unsigned char *buf = XNEWVEC (unsigned char, cpp_token_len (token));
@@ -1304,14 +1330,14 @@ warn_about_normalization (cpp_reader *pfile,
 
       sz = cpp_spell_token (pfile, token, buf, false) - buf;
       if (NORMALIZE_STATE_RESULT (s) == normalized_C)
-	cpp_warning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0,
-			       "`%.*s' is not in NFKC", (int) sz, buf);
+	cpp_warning_at (pfile, CPP_W_NORMALIZE, &rich_loc,
+			"`%.*s' is not in NFKC", (int) sz, buf);
       else if (CPP_OPTION (pfile, cxx23_identifiers))
-	cpp_pedwarning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0,
+	cpp_pedwarning_at (pfile, CPP_W_NORMALIZE, &rich_loc,
 				  "`%.*s' is not in NFC", (int) sz, buf);
       else
-	cpp_warning_with_line (pfile, CPP_W_NORMALIZE, token->src_loc, 0,
-			       "`%.*s' is not in NFC", (int) sz, buf);
+	cpp_warning_at (pfile, CPP_W_NORMALIZE, &rich_loc,
+			"`%.*s' is not in NFC", (int) sz, buf);
       free (buf);
     }
 }
diff --git a/libcpp/line-map.c b/libcpp/line-map.c
index 1a6902a..1957dd7 100644
--- a/libcpp/line-map.c
+++ b/libcpp/line-map.c
@@ -2086,7 +2086,8 @@ rich_location::rich_location (line_maps *set, location_t loc,
   m_fixit_hints (),
   m_seen_impossible_fixit (false),
   m_fixits_cannot_be_auto_applied (false),
-  m_path (NULL)
+  m_path (NULL),
+  m_escape_on_output (false)
 {
   add_range (loc, SHOW_RANGE_WITH_CARET, label);
 }
diff --git a/libcpp/macro.c b/libcpp/macro.c
index f214548..b2f797c 100644
--- a/libcpp/macro.c
+++ b/libcpp/macro.c
@@ -1259,7 +1259,7 @@ collect_args (cpp_reader *pfile, const cpp_hashnode *node,
 	  else if (token->type == CPP_EOF
 		   || (token->type == CPP_HASH && token->flags & BOL))
 	    break;
-	  else if (token->type == CPP_PRAGMA)
+	  else if (token->type == CPP_PRAGMA && !(token->flags & PRAGMA_OP))
 	    {
 	      cpp_token *newtok = _cpp_temp_token (pfile);
 
diff --git a/libdecnumber/ChangeLog b/libdecnumber/ChangeLog
index 4ffa860..da2efd5 100644
--- a/libdecnumber/ChangeLog
+++ b/libdecnumber/ChangeLog
@@ -1,3 +1,8 @@
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Makefile.in: Add dummy install-dvi target.
+
 2021-05-03  H.J. Lu  <hjl.tools@gmail.com>
 
 	PR bootstrap/99703
diff --git a/libdecnumber/Makefile.in b/libdecnumber/Makefile.in
index ec907e1..7e086fc 100644
--- a/libdecnumber/Makefile.in
+++ b/libdecnumber/Makefile.in
@@ -178,13 +178,14 @@ html:
 info:
 install-info:
 install-pdf:
+install-dvi:
 install-man:
 install-html:
 install:
 
 .PHONY: installdirs install install-strip mostlyclean clean distclean \
   maintainer-clean check installcheck dvi pdf html info install-info \
-  install-pdf install-man update-po install-html
+  install-pdf install-dvi install-man update-po install-html
 
 COMPILE = source='$<' object='$@' libtool=no $(CC) $(DEFS) $(INCLUDES) $(CPPFLAGS) $(ALL_CFLAGS) -c
 
diff --git a/libffi/ChangeLog b/libffi/ChangeLog
index e8fe0e0..83792b8 100644
--- a/libffi/ChangeLog
+++ b/libffi/ChangeLog
@@ -1,3 +1,33 @@
+2021-11-15  H.J. Lu  <hjl.tools@gmail.com>
+
+	* LOCAL_PATCHES: Add commit a91f844ef44.
+
+2021-11-15  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	PR libffi/102874
+	* src/x86/win64.S (epilogue): Use #define instead of .macro.
+
+2021-11-04  H.J. Lu  <hjl.tools@gmail.com>
+
+	* Makefile.am (AM_CFLAGS): Add $(CET_FLAGS).
+	(AM_CCASFLAGS): Likewise.
+	* configure.ac (CET_FLAGS): Add GCC_CET_FLAGS and AC_SUBST.
+	* Makefile.in: Regenerate.
+	* aclocal.m4: Likewise.
+	* configure: Likewise.
+	* include/Makefile.in: Likewise.
+	* man/Makefile.in: Likewise.
+	* testsuite/Makefile.in: Likewise.
+
+2021-10-27  H.J. Lu  <hjl.tools@gmail.com>
+
+	* LOCAL_PATCHES: Add commit 90454a90082.
+
+2021-10-25  Segher Boessenkool  <segher@kernel.crashing.org>
+
+	* src/powerpc/linux64.S: Enable AltiVec insns.
+	* src/powerpc/linux64_closure.S: Ditto.
+
 2021-10-20  H.J. Lu  <hjl.tools@gmail.com>
 
 	* LOCAL_PATCHES: New file.
diff --git a/libffi/LOCAL_PATCHES b/libffi/LOCAL_PATCHES
index a377c28..6315190 100644
--- a/libffi/LOCAL_PATCHES
+++ b/libffi/LOCAL_PATCHES
@@ -1,2 +1,4 @@
 5be7b66998127286fada45e4f23bd8a2056d553e
 4824ed41ba7cd63e60fd9f8769a58b79935a90d1
+90205f67e465ae7dfcf733c2b2b177ca7ff68da0
+a91f844ef449d0dd1cf2e0e47b0ade0d8a6304e1
diff --git a/libffi/Makefile.am b/libffi/Makefile.am
index 02e3617..c6d6f84 100644
--- a/libffi/Makefile.am
+++ b/libffi/Makefile.am
@@ -182,7 +182,7 @@ nodist_libffi_convenience_la_SOURCES = $(nodist_libffi_la_SOURCES)
 
 LTLDFLAGS = $(shell $(SHELL) $(top_srcdir)/../libtool-ldflags $(LDFLAGS))
 
-AM_CFLAGS = -Wall -g -fexceptions
+AM_CFLAGS = -Wall -g -fexceptions $(CET_FLAGS)
 if FFI_DEBUG
 # Build debug. Define FFI_DEBUG on the commandline so that, when building with
 # MSVC, it can link against the debug CRT.
@@ -218,7 +218,7 @@ libffi_la_LDFLAGS = -no-undefined $(libffi_version_info) $(libffi_version_script
 libffi_la_DEPENDENCIES = $(libffi_la_LIBADD) $(libffi_version_dep)
 
 AM_CPPFLAGS = -I. -I$(top_srcdir)/include -Iinclude -I$(top_srcdir)/src
-AM_CCASFLAGS = $(AM_CPPFLAGS)
+AM_CCASFLAGS = $(AM_CPPFLAGS) $(CET_FLAGS)
 
 # Multilib support.  Automake should provide these on its own.
 all-recursive: all-multi
diff --git a/libffi/Makefile.in b/libffi/Makefile.in
index 6ff0c67..5524a6a 100644
--- a/libffi/Makefile.in
+++ b/libffi/Makefile.in
@@ -99,7 +99,9 @@ subdir = .
 ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
 am__aclocal_m4_deps = $(top_srcdir)/../config/acx.m4 \
 	$(top_srcdir)/../config/asmcfi.m4 \
+	$(top_srcdir)/../config/cet.m4 \
 	$(top_srcdir)/../config/depstand.m4 \
+	$(top_srcdir)/../config/enable.m4 \
 	$(top_srcdir)/../config/lead-dot.m4 \
 	$(top_srcdir)/../config/multi.m4 \
 	$(top_srcdir)/../config/override.m4 \
@@ -320,6 +322,7 @@ CCAS = @CCAS@
 CCASDEPMODE = @CCASDEPMODE@
 CCASFLAGS = @CCASFLAGS@
 CCDEPMODE = @CCDEPMODE@
+CET_FLAGS = @CET_FLAGS@
 CFLAGS = @CFLAGS@
 CPP = @CPP@
 CPPFLAGS = @CPPFLAGS@
@@ -586,7 +589,7 @@ libffi_convenience_la_LIBADD = $(libffi_la_LIBADD)
 libffi_convenience_la_DEPENDENCIES = $(libffi_la_DEPENDENCIES)
 nodist_libffi_convenience_la_SOURCES = $(nodist_libffi_la_SOURCES)
 LTLDFLAGS = $(shell $(SHELL) $(top_srcdir)/../libtool-ldflags $(LDFLAGS))
-AM_CFLAGS = -Wall -g -fexceptions $(am__append_2)
+AM_CFLAGS = -Wall -g -fexceptions $(CET_FLAGS) $(am__append_2)
 @LIBFFI_BUILD_VERSIONED_SHLIB_FALSE@libffi_version_script = 
 @LIBFFI_BUILD_VERSIONED_SHLIB_GNU_TRUE@@LIBFFI_BUILD_VERSIONED_SHLIB_TRUE@libffi_version_script = -Wl,--version-script,libffi.map
 @LIBFFI_BUILD_VERSIONED_SHLIB_SUN_TRUE@@LIBFFI_BUILD_VERSIONED_SHLIB_TRUE@libffi_version_script = -Wl,-M,libffi.map-sun
@@ -597,7 +600,7 @@ libffi_version_info = -version-info `grep -v '^\#' $(srcdir)/libtool-version`
 libffi_la_LDFLAGS = -no-undefined $(libffi_version_info) $(libffi_version_script) $(LTLDFLAGS) $(AM_LTLDFLAGS)
 libffi_la_DEPENDENCIES = $(libffi_la_LIBADD) $(libffi_version_dep)
 AM_CPPFLAGS = -I. -I$(top_srcdir)/include -Iinclude -I$(top_srcdir)/src
-AM_CCASFLAGS = $(AM_CPPFLAGS)
+AM_CCASFLAGS = $(AM_CPPFLAGS) $(CET_FLAGS)
 MULTISRCTOP = 
 MULTIBUILDTOP = 
 MULTIDIRS = 
diff --git a/libffi/aclocal.m4 b/libffi/aclocal.m4
index 9c8c88f..736ec30 100644
--- a/libffi/aclocal.m4
+++ b/libffi/aclocal.m4
@@ -1189,7 +1189,9 @@ AC_SUBST([am__untar])
 
 m4_include([../config/acx.m4])
 m4_include([../config/asmcfi.m4])
+m4_include([../config/cet.m4])
 m4_include([../config/depstand.m4])
+m4_include([../config/enable.m4])
 m4_include([../config/lead-dot.m4])
 m4_include([../config/multi.m4])
 m4_include([../config/override.m4])
diff --git a/libffi/configure b/libffi/configure
index 4bababb..575641c 100755
--- a/libffi/configure
+++ b/libffi/configure
@@ -692,6 +692,7 @@ am__fastdepCCAS_TRUE
 CCASDEPMODE
 CCASFLAGS
 CCAS
+CET_FLAGS
 am__fastdepCXX_FALSE
 am__fastdepCXX_TRUE
 CXXDEPMODE
@@ -802,6 +803,7 @@ enable_multilib
 enable_silent_rules
 enable_generated_files_in_srcdir
 enable_dependency_tracking
+enable_cet
 enable_shared
 enable_static
 with_pic
@@ -1457,6 +1459,7 @@ Optional Features:
                           do not reject slow dependency extractors
   --disable-dependency-tracking
                           speeds up one-time build
+  --enable-cet            enable Intel CET in target libraries [default=auto]
   --enable-shared[=PKGS]  build shared libraries [default=yes]
   --enable-static[=PKGS]  build static libraries [default=yes]
   --enable-fast-install[=PKGS]
@@ -4877,6 +4880,95 @@ CFLAGS=$save_CFLAGS
 
 
 
+# Add CET specific flags if CET is enabled
+
+ # Check whether --enable-cet was given.
+if test "${enable_cet+set}" = set; then :
+  enableval=$enable_cet;
+      case "$enableval" in
+       yes|no|auto) ;;
+       *) as_fn_error $? "Unknown argument to enable/disable cet" "$LINENO" 5 ;;
+                          esac
+
+else
+  enable_cet=auto
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for CET support" >&5
+$as_echo_n "checking for CET support... " >&6; }
+
+# NB: Avoid nested save_CFLAGS and save_LDFLAGS.
+case "$host" in
+  i[34567]86-*-linux* | x86_64-*-linux*)
+    case "$enable_cet" in
+      auto)
+	# Check if target supports multi-byte NOPs
+	# and if compiler and assembler support CET insn.
+	cet_save_CFLAGS="$CFLAGS"
+	CFLAGS="$CFLAGS -fcf-protection"
+	cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+#if !defined(__SSE2__)
+#error target does not support multi-byte NOPs
+#else
+asm ("setssbsy");
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  enable_cet=yes
+else
+  enable_cet=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+	CFLAGS="$cet_save_CFLAGS"
+	;;
+      yes)
+	# Check if assembler supports CET.
+	cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main ()
+{
+asm ("setssbsy");
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+
+else
+  as_fn_error $? "assembler with CET support is required for --enable-cet" "$LINENO" 5
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+	;;
+    esac
+    ;;
+  *)
+    enable_cet=no
+    ;;
+esac
+if test x$enable_cet = xyes; then
+  CET_FLAGS="-fcf-protection -mshstk"
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+
 # By default we simply use the C compiler to build assembly code.
 
 test "${CCAS+set}" = set || CCAS=$CC
@@ -6903,7 +6995,6 @@ fi
 
 
 
-
 # Check whether --enable-libtool-lock was given.
 if test "${enable_libtool_lock+set}" = set; then :
   enableval=$enable_libtool_lock;
@@ -11481,7 +11572,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 11484 "configure"
+#line 11575 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -11587,7 +11678,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 11590 "configure"
+#line 11681 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
diff --git a/libffi/configure.ac b/libffi/configure.ac
index bff5a1e..014d89d 100644
--- a/libffi/configure.ac
+++ b/libffi/configure.ac
@@ -48,6 +48,10 @@ m4_rename_force([real_PRECIOUS],[_AC_ARG_VAR_PRECIOUS])
 
 AC_SUBST(CFLAGS)
 
+# Add CET specific flags if CET is enabled
+GCC_CET_FLAGS(CET_FLAGS)
+AC_SUBST(CET_FLAGS)
+
 AM_PROG_AS
 AM_PROG_CC_C_O
 AC_PROG_LIBTOOL
diff --git a/libffi/include/Makefile.in b/libffi/include/Makefile.in
index c76e2a8..879cb72 100644
--- a/libffi/include/Makefile.in
+++ b/libffi/include/Makefile.in
@@ -93,7 +93,9 @@ subdir = include
 ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
 am__aclocal_m4_deps = $(top_srcdir)/../config/acx.m4 \
 	$(top_srcdir)/../config/asmcfi.m4 \
+	$(top_srcdir)/../config/cet.m4 \
 	$(top_srcdir)/../config/depstand.m4 \
+	$(top_srcdir)/../config/enable.m4 \
 	$(top_srcdir)/../config/lead-dot.m4 \
 	$(top_srcdir)/../config/multi.m4 \
 	$(top_srcdir)/../config/override.m4 \
@@ -192,6 +194,7 @@ CCAS = @CCAS@
 CCASDEPMODE = @CCASDEPMODE@
 CCASFLAGS = @CCASFLAGS@
 CCDEPMODE = @CCDEPMODE@
+CET_FLAGS = @CET_FLAGS@
 CFLAGS = @CFLAGS@
 CPP = @CPP@
 CPPFLAGS = @CPPFLAGS@
diff --git a/libffi/man/Makefile.in b/libffi/man/Makefile.in
index 8dbe52f..9d4f966 100644
--- a/libffi/man/Makefile.in
+++ b/libffi/man/Makefile.in
@@ -92,7 +92,9 @@ subdir = man
 ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
 am__aclocal_m4_deps = $(top_srcdir)/../config/acx.m4 \
 	$(top_srcdir)/../config/asmcfi.m4 \
+	$(top_srcdir)/../config/cet.m4 \
 	$(top_srcdir)/../config/depstand.m4 \
+	$(top_srcdir)/../config/enable.m4 \
 	$(top_srcdir)/../config/lead-dot.m4 \
 	$(top_srcdir)/../config/multi.m4 \
 	$(top_srcdir)/../config/override.m4 \
@@ -174,6 +176,7 @@ CCAS = @CCAS@
 CCASDEPMODE = @CCASDEPMODE@
 CCASFLAGS = @CCASFLAGS@
 CCDEPMODE = @CCDEPMODE@
+CET_FLAGS = @CET_FLAGS@
 CFLAGS = @CFLAGS@
 CPP = @CPP@
 CPPFLAGS = @CPPFLAGS@
diff --git a/libffi/src/powerpc/linux64.S b/libffi/src/powerpc/linux64.S
index e92d64a..1f876ea 100644
--- a/libffi/src/powerpc/linux64.S
+++ b/libffi/src/powerpc/linux64.S
@@ -29,6 +29,8 @@
 #include <fficonfig.h>
 #include <ffi.h>
 
+	.machine altivec
+
 #ifdef POWERPC64
 	.hidden	ffi_call_LINUX64
 	.globl	ffi_call_LINUX64
diff --git a/libffi/src/powerpc/linux64_closure.S b/libffi/src/powerpc/linux64_closure.S
index 3469a2c..199981d 100644
--- a/libffi/src/powerpc/linux64_closure.S
+++ b/libffi/src/powerpc/linux64_closure.S
@@ -30,6 +30,8 @@
 
 	.file	"linux64_closure.S"
 
+	.machine altivec
+
 #ifdef POWERPC64
 	FFI_HIDDEN (ffi_closure_LINUX64)
 	.globl  ffi_closure_LINUX64
diff --git a/libffi/src/x86/win64.S b/libffi/src/x86/win64.S
index f3ace8d..58ec6a1 100644
--- a/libffi/src/x86/win64.S
+++ b/libffi/src/x86/win64.S
@@ -85,14 +85,13 @@ C(ffi_call_win64):
 
 /* Below, we're space constrained most of the time.  Thus we eschew the
    modern "mov, pop, ret" sequence (5 bytes) for "leave, ret" (2 bytes).  */
-.macro epilogue
-	leaveq
-	cfi_remember_state
-	cfi_def_cfa(%rsp, 8)
-	cfi_restore(%rbp)
-	ret
+#define epilogue		\
+	leaveq;			\
+	cfi_remember_state;	\
+	cfi_def_cfa(%rsp, 8);	\
+	cfi_restore(%rbp);	\
+	ret;			\
 	cfi_restore_state
-.endm
 
 	.align	8
 0:
diff --git a/libffi/testsuite/Makefile.in b/libffi/testsuite/Makefile.in
index 3e9e937..a19b523 100644
--- a/libffi/testsuite/Makefile.in
+++ b/libffi/testsuite/Makefile.in
@@ -92,7 +92,9 @@ subdir = testsuite
 ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
 am__aclocal_m4_deps = $(top_srcdir)/../config/acx.m4 \
 	$(top_srcdir)/../config/asmcfi.m4 \
+	$(top_srcdir)/../config/cet.m4 \
 	$(top_srcdir)/../config/depstand.m4 \
+	$(top_srcdir)/../config/enable.m4 \
 	$(top_srcdir)/../config/lead-dot.m4 \
 	$(top_srcdir)/../config/multi.m4 \
 	$(top_srcdir)/../config/override.m4 \
@@ -147,6 +149,7 @@ CCAS = @CCAS@
 CCASDEPMODE = @CCASDEPMODE@
 CCASFLAGS = @CCASFLAGS@
 CCDEPMODE = @CCDEPMODE@
+CET_FLAGS = @CET_FLAGS@
 CFLAGS = @CFLAGS@
 CPP = @CPP@
 CPPFLAGS = @CPPFLAGS@
diff --git a/libgcc/ChangeLog b/libgcc/ChangeLog
index 8dfb0d6..5d85569 100644
--- a/libgcc/ChangeLog
+++ b/libgcc/ChangeLog
@@ -1,3 +1,46 @@
+2021-11-11  Raphael Moreira Zinsly  <rzinsly@linux.ibm.com>
+
+	PR libgcc/103044
+	* config/rs6000/linux-unwind.h (ppc_backchain_fallback): Check if it's
+	called with a null argument or at the end of the backtrace and return.
+	* unwind.inc (_Unwind_ForcedUnwind_Phase2): Treat _URC_NORMAL_STOP.
+
+2021-11-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* config/rs6000/linux-unwind.h (ppc_back_fallback): Make it static,
+	formatting fix.
+
+2021-11-05  Iain Sandoe  <iain@sandoe.co.uk>
+
+	* config/darwin10-unwind-find-enc-func.c: Include libgcc_tm.h.
+	* config/i386/darwin-lib.h: Declare Darwin10 crt function.
+
+2021-11-05  John David Anglin  <danglin@gcc.gnu.org>
+
+	PR libgomp/96661
+	* config.host (hppa*64*-*-linux*): Revise tmake_file.
+	(hppa*64*-*-hpux11*): Likewise.
+	* config/pa/sfp-exceptions.c: New.
+	* config/pa/sfp-machine.h: New.
+	* config/pa/t-dimode: New.
+	* config/pa/t-softfp-sfdftf: New.
+
+2021-10-27  John David Anglin  <danglin@gcc.gnu.org>
+
+	* config.host (hppa*64*-*-linux*): Don't add pa/t-linux to
+	tmake_file.
+	* config/pa/linux-atomic.c: Define u8, u16 and u64 types.
+	Use them in FETCH_AND_OP_2, OP_AND_FETCH_2, COMPARE_AND_SWAP_2,
+	SYNC_LOCK_TEST_AND_SET_2 and SYNC_LOCK_RELEASE_1 macros.
+	* config/pa/t-linux64 (LIB1ASMSRC): New define.
+	(LIB1ASMFUNCS): Revise.
+	(HOST_LIBGCC2_CFLAGS): Add "-DLINUX=1".
+
+2021-10-22  Stafford Horne  <shorne@gmail.com>
+
+	* config/or1k/sfp-machine.h (_FP_TININESS_AFTER_ROUNDING):
+	Change to 0.
+
 2021-10-14  Raphael Moreira Zinsly  <rzinsly@linux.ibm.com>
 
 	* config/rs6000/linux-unwind.h (struct rt_sigframe): Move it to
diff --git a/libgcc/config.host b/libgcc/config.host
index 6c34b13..168535b 100644
--- a/libgcc/config.host
+++ b/libgcc/config.host
@@ -619,7 +619,8 @@ h8300-*-linux*)
 	tm_file="$tm_file h8300/h8300-lib.h"
 	;;
 hppa*64*-*-linux*)
-	tmake_file="$tmake_file pa/t-linux pa/t-linux64"
+	tmake_file="$tmake_file pa/t-linux64 pa/t-dimode"
+	tmake_file="$tmake_file pa/t-softfp-sfdftf t-softfp"
 	extra_parts="crtbegin.o crtbeginS.o crtbeginT.o crtend.o crtendS.o"
 	;;
 hppa*-*-linux*)
@@ -645,7 +646,8 @@ hppa[12]*-*-hpux10*)
 	md_unwind_header=pa/hpux-unwind.h
 	;;
 hppa*64*-*-hpux11*)
-	tmake_file="$tmake_file pa/t-hpux pa/t-pa64 pa/t-stublib t-libgcc-pic t-slibgcc"
+	tmake_file="$tmake_file pa/t-hpux pa/t-pa64 pa/t-dimode"
+	tmake_file="$tmake_file pa/t-stublib t-libgcc-pic t-slibgcc"
 	# Set the libgcc version number
 	if test x$ac_cv_sjlj_exceptions = xyes; then
 	    tmake_file="$tmake_file pa/t-slibgcc-sjlj-ver"
@@ -653,6 +655,7 @@ hppa*64*-*-hpux11*)
 	    tmake_file="$tmake_file pa/t-slibgcc-dwarf-ver"
 	fi
 	tmake_file="$tmake_file pa/t-slibgcc-hpux t-slibgcc-hpux"
+	tmake_file="$tmake_file pa/t-softfp-sfdftf t-softfp"
 	tm_file="$tm_file pa/pa64-hpux-lib.h"
 	extra_parts="crtbegin.o crtend.o crtbeginS.o crtendS.o crtbeginT.o \
 		     libgcc_stub.a"
diff --git a/libgcc/config/darwin10-unwind-find-enc-func.c b/libgcc/config/darwin10-unwind-find-enc-func.c
index 67c4375..882ec3a 100644
--- a/libgcc/config/darwin10-unwind-find-enc-func.c
+++ b/libgcc/config/darwin10-unwind-find-enc-func.c
@@ -1,6 +1,7 @@
 #include "tconfig.h"
 #include "tsystem.h"
 #include "unwind-dw2-fde.h"
+#include "libgcc_tm.h"
 
 void *
 _darwin10_Unwind_FindEnclosingFunction (void *pc)
diff --git a/libgcc/config/i386/darwin-lib.h b/libgcc/config/i386/darwin-lib.h
index e424b88..5aeee33 100644
--- a/libgcc/config/i386/darwin-lib.h
+++ b/libgcc/config/i386/darwin-lib.h
@@ -30,3 +30,5 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #define DECLARE_LIBRARY_RENAMES \
   asm(".text; ___divdc3: jmp ___ieee_divdc3 ; .globl ___divdc3");
 #endif
+
+extern void * _darwin10_Unwind_FindEnclosingFunction (void *);
diff --git a/libgcc/config/or1k/sfp-machine.h b/libgcc/config/or1k/sfp-machine.h
index eebe5b0..162c6bc 100644
--- a/libgcc/config/or1k/sfp-machine.h
+++ b/libgcc/config/or1k/sfp-machine.h
@@ -85,7 +85,7 @@ do {								\
 
 #define __BYTE_ORDER __BIG_ENDIAN
 
-#define _FP_TININESS_AFTER_ROUNDING 1
+#define _FP_TININESS_AFTER_ROUNDING 0
 
 /* Define ALIASNAME as a strong alias for NAME.  */
 # define strong_alias(name, aliasname) _strong_alias(name, aliasname)
diff --git a/libgcc/config/pa/linux-atomic.c b/libgcc/config/pa/linux-atomic.c
index c882b55..500a365 100644
--- a/libgcc/config/pa/linux-atomic.c
+++ b/libgcc/config/pa/linux-atomic.c
@@ -28,6 +28,14 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #define EBUSY   16
 #define ENOSYS 251 
 
+typedef unsigned char u8;
+typedef short unsigned int u16;
+#ifdef __LP64__
+typedef long unsigned int u64;
+#else
+typedef long long unsigned int u64;
+#endif
+
 /* PA-RISC 2.0 supports out-of-order execution for loads and stores.
    Thus, we need to synchonize memory accesses.  For more info, see:
    "Advanced Performance Features of the 64-bit PA-8000" by Doug Hunt.
@@ -117,26 +125,26 @@ __kernel_cmpxchg2 (volatile void *mem, const void *oldval, const void *newval,
     return tmp;								\
   }
 
-FETCH_AND_OP_2 (add,   , +, long long unsigned int, 8, 3)
-FETCH_AND_OP_2 (sub,   , -, long long unsigned int, 8, 3)
-FETCH_AND_OP_2 (or,    , |, long long unsigned int, 8, 3)
-FETCH_AND_OP_2 (and,   , &, long long unsigned int, 8, 3)
-FETCH_AND_OP_2 (xor,   , ^, long long unsigned int, 8, 3)
-FETCH_AND_OP_2 (nand, ~, &, long long unsigned int, 8, 3)
-
-FETCH_AND_OP_2 (add,   , +, short unsigned int, 2, 1)
-FETCH_AND_OP_2 (sub,   , -, short unsigned int, 2, 1)
-FETCH_AND_OP_2 (or,    , |, short unsigned int, 2, 1)
-FETCH_AND_OP_2 (and,   , &, short unsigned int, 2, 1)
-FETCH_AND_OP_2 (xor,   , ^, short unsigned int, 2, 1)
-FETCH_AND_OP_2 (nand, ~, &, short unsigned int, 2, 1)
-
-FETCH_AND_OP_2 (add,   , +, unsigned char, 1, 0)
-FETCH_AND_OP_2 (sub,   , -, unsigned char, 1, 0)
-FETCH_AND_OP_2 (or,    , |, unsigned char, 1, 0)
-FETCH_AND_OP_2 (and,   , &, unsigned char, 1, 0)
-FETCH_AND_OP_2 (xor,   , ^, unsigned char, 1, 0)
-FETCH_AND_OP_2 (nand, ~, &, unsigned char, 1, 0)
+FETCH_AND_OP_2 (add,   , +, u64, 8, 3)
+FETCH_AND_OP_2 (sub,   , -, u64, 8, 3)
+FETCH_AND_OP_2 (or,    , |, u64, 8, 3)
+FETCH_AND_OP_2 (and,   , &, u64, 8, 3)
+FETCH_AND_OP_2 (xor,   , ^, u64, 8, 3)
+FETCH_AND_OP_2 (nand, ~, &, u64, 8, 3)
+
+FETCH_AND_OP_2 (add,   , +, u16, 2, 1)
+FETCH_AND_OP_2 (sub,   , -, u16, 2, 1)
+FETCH_AND_OP_2 (or,    , |, u16, 2, 1)
+FETCH_AND_OP_2 (and,   , &, u16, 2, 1)
+FETCH_AND_OP_2 (xor,   , ^, u16, 2, 1)
+FETCH_AND_OP_2 (nand, ~, &, u16, 2, 1)
+
+FETCH_AND_OP_2 (add,   , +, u8, 1, 0)
+FETCH_AND_OP_2 (sub,   , -, u8, 1, 0)
+FETCH_AND_OP_2 (or,    , |, u8, 1, 0)
+FETCH_AND_OP_2 (and,   , &, u8, 1, 0)
+FETCH_AND_OP_2 (xor,   , ^, u8, 1, 0)
+FETCH_AND_OP_2 (nand, ~, &, u8, 1, 0)
 
 #define OP_AND_FETCH_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX)		\
   TYPE HIDDEN								\
@@ -154,26 +162,26 @@ FETCH_AND_OP_2 (nand, ~, &, unsigned char, 1, 0)
     return PFX_OP (tmp INF_OP val);					\
   }
 
-OP_AND_FETCH_2 (add,   , +, long long unsigned int, 8, 3)
-OP_AND_FETCH_2 (sub,   , -, long long unsigned int, 8, 3)
-OP_AND_FETCH_2 (or,    , |, long long unsigned int, 8, 3)
-OP_AND_FETCH_2 (and,   , &, long long unsigned int, 8, 3)
-OP_AND_FETCH_2 (xor,   , ^, long long unsigned int, 8, 3)
-OP_AND_FETCH_2 (nand, ~, &, long long unsigned int, 8, 3)
-
-OP_AND_FETCH_2 (add,   , +, short unsigned int, 2, 1)
-OP_AND_FETCH_2 (sub,   , -, short unsigned int, 2, 1)
-OP_AND_FETCH_2 (or,    , |, short unsigned int, 2, 1)
-OP_AND_FETCH_2 (and,   , &, short unsigned int, 2, 1)
-OP_AND_FETCH_2 (xor,   , ^, short unsigned int, 2, 1)
-OP_AND_FETCH_2 (nand, ~, &, short unsigned int, 2, 1)
-
-OP_AND_FETCH_2 (add,   , +, unsigned char, 1, 0)
-OP_AND_FETCH_2 (sub,   , -, unsigned char, 1, 0)
-OP_AND_FETCH_2 (or,    , |, unsigned char, 1, 0)
-OP_AND_FETCH_2 (and,   , &, unsigned char, 1, 0)
-OP_AND_FETCH_2 (xor,   , ^, unsigned char, 1, 0)
-OP_AND_FETCH_2 (nand, ~, &, unsigned char, 1, 0)
+OP_AND_FETCH_2 (add,   , +, u64, 8, 3)
+OP_AND_FETCH_2 (sub,   , -, u64, 8, 3)
+OP_AND_FETCH_2 (or,    , |, u64, 8, 3)
+OP_AND_FETCH_2 (and,   , &, u64, 8, 3)
+OP_AND_FETCH_2 (xor,   , ^, u64, 8, 3)
+OP_AND_FETCH_2 (nand, ~, &, u64, 8, 3)
+
+OP_AND_FETCH_2 (add,   , +, u16, 2, 1)
+OP_AND_FETCH_2 (sub,   , -, u16, 2, 1)
+OP_AND_FETCH_2 (or,    , |, u16, 2, 1)
+OP_AND_FETCH_2 (and,   , &, u16, 2, 1)
+OP_AND_FETCH_2 (xor,   , ^, u16, 2, 1)
+OP_AND_FETCH_2 (nand, ~, &, u16, 2, 1)
+
+OP_AND_FETCH_2 (add,   , +, u8, 1, 0)
+OP_AND_FETCH_2 (sub,   , -, u8, 1, 0)
+OP_AND_FETCH_2 (or,    , |, u8, 1, 0)
+OP_AND_FETCH_2 (and,   , &, u8, 1, 0)
+OP_AND_FETCH_2 (xor,   , ^, u8, 1, 0)
+OP_AND_FETCH_2 (nand, ~, &, u8, 1, 0)
 
 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP)				\
   unsigned int HIDDEN							\
@@ -254,9 +262,9 @@ typedef unsigned char bool;
     return (failure == 0);						\
   }
 
-COMPARE_AND_SWAP_2 (long long unsigned int, 8, 3)
-COMPARE_AND_SWAP_2 (short unsigned int, 2, 1)
-COMPARE_AND_SWAP_2 (unsigned char, 1, 0)
+COMPARE_AND_SWAP_2 (u64, 8, 3)
+COMPARE_AND_SWAP_2 (u16, 2, 1)
+COMPARE_AND_SWAP_2 (u8, 1, 0)
 
 unsigned int HIDDEN
 __sync_val_compare_and_swap_4 (volatile void *ptr, unsigned int oldval,
@@ -304,9 +312,9 @@ TYPE HIDDEN								\
     return oldval;							\
   }
 
-SYNC_LOCK_TEST_AND_SET_2 (long long unsigned int, 8, 3)
-SYNC_LOCK_TEST_AND_SET_2 (short unsigned int, 2, 1)
-SYNC_LOCK_TEST_AND_SET_2 (unsigned char, 1, 0)
+SYNC_LOCK_TEST_AND_SET_2 (u64, 8, 3)
+SYNC_LOCK_TEST_AND_SET_2 (u16, 2, 1)
+SYNC_LOCK_TEST_AND_SET_2 (u8, 1, 0)
 
 unsigned int HIDDEN
 __sync_lock_test_and_set_4 (volatile void *ptr, unsigned int val)
@@ -336,9 +344,9 @@ __sync_lock_test_and_set_4 (volatile void *ptr, unsigned int val)
     } while (failure != 0);					\
   }
 
-SYNC_LOCK_RELEASE_1 (long long unsigned int, 8, 3)
-SYNC_LOCK_RELEASE_1 (short unsigned int, 2, 1)
-SYNC_LOCK_RELEASE_1 (unsigned char, 1, 0)
+SYNC_LOCK_RELEASE_1 (u64, 8, 3)
+SYNC_LOCK_RELEASE_1 (u16, 2, 1)
+SYNC_LOCK_RELEASE_1 (u8, 1, 0)
 
 void HIDDEN
 __sync_lock_release_4 (volatile void *ptr)
diff --git a/libgcc/config/pa/sfp-exceptions.c b/libgcc/config/pa/sfp-exceptions.c
new file mode 100644
index 0000000..4dadaf4
--- /dev/null
+++ b/libgcc/config/pa/sfp-exceptions.c
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 1997-2021 Free Software Foundation, Inc.
+ *
+ * This file is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 3, or (at your option) any
+ * later version.
+ *
+ * This file is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * Under Section 7 of GPL version 3, you are granted additional
+ * permissions described in the GCC Runtime Library Exception, version
+ * 3.1, as published by the Free Software Foundation.
+ *
+ * You should have received a copy of the GNU General Public License and
+ * a copy of the GCC Runtime Library Exception along with this program;
+ * see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "sfp-machine.h"
+
+#define HUGE_VAL (__builtin_huge_val ())
+
+/* Please see section 10,
+   page 10-5 "Delayed Trapping" in the PA-RISC 2.0 Architecture manual */
+
+void
+__sfp_handle_exceptions (int _fex)
+{
+  /* Raise exceptions represented by _FEX.  But we must raise only one
+     signal at a time.  It is important that if the overflow/underflow
+     exception and the divide by zero exception are given at the same
+     time, the overflow/underflow exception follows the divide by zero
+     exception.  */
+
+  /* We do these bits in assembly to be certain GCC doesn't optimize
+     away something important, and so we can force delayed traps to
+     occur. */
+
+  /* We use "fldd 0(%%sr0,%%sp),%0" to flush the delayed exception */
+
+  /* First: Invalid exception.  */
+  if (_fex & FP_EX_INVALID)
+    {
+      /* One example of an invalid operation is 0 * Infinity.  */
+      double d = HUGE_VAL;
+      __asm__ __volatile__ (
+		"	fcpy,dbl %%fr0,%%fr22\n"
+		"	fmpy,dbl %0,%%fr22,%0\n"
+		"	fldd 0(%%sr0,%%sp),%0"
+		: "+f" (d) : : "%fr22" );
+    }
+
+  /* Second: Division by zero.  */
+  if (_fex & FP_EX_DIVZERO)
+    {
+      double d = 1.0;
+      __asm__ __volatile__ (
+		"	fcpy,dbl %%fr0,%%fr22\n"
+		"	fdiv,dbl %0,%%fr22,%0\n"
+		"	fldd 0(%%sr0,%%sp),%0"
+		: "+f" (d) : : "%fr22" );
+    }
+
+  /* Third: Overflow.  */
+  if (_fex & FP_EX_OVERFLOW)
+    {
+      double d = __DBL_MAX__;
+      __asm__ __volatile__ (
+		"	fadd,dbl %0,%0,%0\n"
+		"	fldd 0(%%sr0,%%sp),%0"
+		: "+f" (d) );
+    }
+
+  /* Fourth: Underflow.  */
+  if (_fex & FP_EX_UNDERFLOW)
+    {
+      double d = __DBL_MIN__;
+      double e = 3.0;
+      __asm__ __volatile__ (
+		"	fdiv,dbl %0,%1,%0\n"
+		"	fldd 0(%%sr0,%%sp),%0"
+		: "+f" (d) : "f" (e) );
+    }
+
+  /* Fifth: Inexact */
+  if (_fex & FP_EX_INEXACT)
+    {
+      double d = 3.14159265358979323846;
+      double e = 69.69;
+      __asm__ __volatile__ (
+		"	fdiv,dbl %0,%1,%%fr22\n"
+		"	fcnvfxt,dbl,sgl %%fr22,%%fr22L\n"
+		"	fldd 0(%%sr0,%%sp),%%fr22"
+		: : "f" (d), "f" (e) : "%fr22" );
+    }
+}
diff --git a/libgcc/config/pa/sfp-machine.h b/libgcc/config/pa/sfp-machine.h
new file mode 100644
index 0000000..661b1cf
--- /dev/null
+++ b/libgcc/config/pa/sfp-machine.h
@@ -0,0 +1,155 @@
+/* softfp machine description for PA-RISC.
+   Copyright (C) 2009-2021 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+Under Section 7 of GPL version 3, you are granted additional
+permissions described in the GCC Runtime Library Exception, version
+3.1, as published by the Free Software Foundation.
+
+You should have received a copy of the GNU General Public License and
+a copy of the GCC Runtime Library Exception along with this program;
+see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+<http://www.gnu.org/licenses/>.  */
+
+#ifdef __LP64__
+#define _FP_W_TYPE_SIZE		64
+#define _FP_W_TYPE		unsigned long
+#define _FP_WS_TYPE		signed long
+#define _FP_I_TYPE		long
+
+typedef int TItype __attribute__ ((mode (TI)));
+typedef unsigned int UTItype __attribute__ ((mode (TI)));
+#define TI_BITS (__CHAR_BIT__ * (int) sizeof (TItype))
+
+#define _FP_MUL_MEAT_S(R,X,Y)				\
+  _FP_MUL_MEAT_1_wide(_FP_WFRACBITS_S,R,X,Y,umul_ppmm)
+#define _FP_MUL_MEAT_D(R,X,Y)				\
+  _FP_MUL_MEAT_1_wide(_FP_WFRACBITS_D,R,X,Y,umul_ppmm)
+#define _FP_MUL_MEAT_Q(R,X,Y)				\
+  _FP_MUL_MEAT_2_wide(_FP_WFRACBITS_Q,R,X,Y,umul_ppmm)
+
+#define _FP_DIV_MEAT_S(R,X,Y)	_FP_DIV_MEAT_1_udiv_norm(S,R,X,Y)
+#define _FP_DIV_MEAT_D(R,X,Y)	_FP_DIV_MEAT_1_udiv_norm(D,R,X,Y)
+#define _FP_DIV_MEAT_Q(R,X,Y)	_FP_DIV_MEAT_2_udiv(Q,R,X,Y)
+
+#define _FP_NANFRAC_S		(_FP_QNANBIT_S - 1)
+#define _FP_NANFRAC_D		(_FP_QNANBIT_D - 1)
+#define _FP_NANFRAC_Q		(_FP_QNANBIT_Q - 1), -1
+#else
+#define _FP_W_TYPE_SIZE		32
+#define _FP_W_TYPE		unsigned int
+#define _FP_WS_TYPE		signed int
+#define _FP_I_TYPE		int
+
+#define _FP_MUL_MEAT_S(R,X,Y)				\
+  _FP_MUL_MEAT_1_wide(_FP_WFRACBITS_S,R,X,Y,umul_ppmm)
+#define _FP_MUL_MEAT_D(R,X,Y)				\
+  _FP_MUL_MEAT_2_wide(_FP_WFRACBITS_D,R,X,Y,umul_ppmm)
+#define _FP_MUL_MEAT_Q(R,X,Y)				\
+  _FP_MUL_MEAT_4_wide(_FP_WFRACBITS_Q,R,X,Y,umul_ppmm)
+
+#define _FP_DIV_MEAT_S(R,X,Y)	_FP_DIV_MEAT_1_udiv_norm(S,R,X,Y)
+#define _FP_DIV_MEAT_D(R,X,Y)	_FP_DIV_MEAT_2_udiv(D,R,X,Y)
+#define _FP_DIV_MEAT_Q(R,X,Y)	_FP_DIV_MEAT_4_udiv(Q,R,X,Y)
+
+#define _FP_NANFRAC_S		(_FP_QNANBIT_S - 1)
+#define _FP_NANFRAC_D		(_FP_QNANBIT_D - 1), -1
+#define _FP_NANFRAC_Q		(_FP_QNANBIT_Q - 1), -1, -1, -1
+#endif
+
+/* The type of the result of a floating point comparison.  This must
+   match __libgcc_cmp_return__ in GCC for the target.  */
+typedef int __gcc_CMPtype __attribute__ ((mode (__libgcc_cmp_return__)));
+#define CMPtype __gcc_CMPtype
+
+#define _FP_NANSIGN_S		0
+#define _FP_NANSIGN_D		0
+#define _FP_NANSIGN_Q		0
+
+#define _FP_KEEPNANFRACP	1
+#define _FP_QNANNEGATEDP	1
+
+/* Comment from glibc: */
+/* From my experiments it seems X is chosen unless one of the
+   NaNs is sNaN,  in which case the result is NANSIGN/NANFRAC.  */
+# define _FP_CHOOSENAN(fs, wc, R, X, Y, OP)			\
+  do {								\
+    if ((_FP_FRAC_HIGH_RAW_##fs(X) |				\
+	 _FP_FRAC_HIGH_RAW_##fs(Y)) & _FP_QNANBIT_##fs)		\
+      {								\
+	R##_s = _FP_NANSIGN_##fs;				\
+	_FP_FRAC_SET_##wc(R,_FP_NANFRAC_##fs);			\
+      }								\
+    else							\
+      {								\
+	R##_s = X##_s;						\
+	_FP_FRAC_COPY_##wc(R,X);				\
+      }								\
+    R##_c = FP_CLS_NAN;						\
+  } while (0)
+
+#define FP_RND_NEAREST		(0 << 9)
+#define FP_RND_ZERO		(1 << 9)
+#define FP_RND_PINF		(2 << 9)
+#define FP_RND_MINF		(3 << 9)
+#define FP_RND_MASK		(3 << 9)
+
+#define FP_EX_INEXACT		0x01
+#define FP_EX_UNDERFLOW		0x02
+#define FP_EX_OVERFLOW		0x04
+#define FP_EX_DIVZERO		0x08
+#define FP_EX_INVALID		0x10
+#define FP_EX_ALL		0x1F
+#define FP_EX_SHIFT		27
+
+#define _FP_TININESS_AFTER_ROUNDING 1
+
+#define _FP_DECL_EX \
+  unsigned int _fcsr __attribute__ ((unused)) = FP_RND_NEAREST
+
+/* Get floating-point status.  */
+#define _FPU_GETCW(cw)						\
+({								\
+  union { unsigned long long __fpreg;				\
+	  unsigned int __halfreg[2]; } __fullfp;		\
+  /* Get the current status word. */				\
+  __asm__ ("fstd %%fr0,0(%1)\n\t"				\
+	   "fldd 0(%1),%%fr0\n\t"				\
+	    : "=m" (__fullfp.__fpreg)				\
+	    : "r" (&__fullfp.__fpreg)				\
+	    : "%r0");						\
+  cw = __fullfp.__halfreg[0];					\
+})
+
+#define FP_INIT_ROUNDMODE _FPU_GETCW(_fcsr)
+#define FP_ROUNDMODE (_fcsr & FP_RND_MASK)
+#define FP_TRAPPING_EXCEPTIONS (_fcsr & FP_EX_ALL)
+
+void __sfp_handle_exceptions (int);
+
+#define FP_HANDLE_EXCEPTIONS				\
+  do {							\
+    if (__builtin_expect (_fex, 0))			\
+      __sfp_handle_exceptions (_fex);			\
+  } while (0)
+
+#define	__LITTLE_ENDIAN	1234
+#define	__BIG_ENDIAN	4321
+
+#define __BYTE_ORDER __BIG_ENDIAN
+
+/* Define ALIASNAME as a strong alias for NAME.  */
+# define strong_alias(name, aliasname) _strong_alias(name, aliasname)
+# define _strong_alias(name, aliasname) \
+  extern __typeof (name) aliasname __attribute__ ((alias (#name)));
diff --git a/libgcc/config/pa/t-dimode b/libgcc/config/pa/t-dimode
new file mode 100644
index 0000000..1344e93
--- /dev/null
+++ b/libgcc/config/pa/t-dimode
@@ -0,0 +1,33 @@
+# Extra DImode libgcc2 functions to build.
+lib2difuncs = _muldi3 _negdi2 _lshrdi _ashldi3 _ashrdi3 _cmpdi2 _ucmpdi2
+
+# Build lib2difuncs.
+lib2difuncs-o = $(patsubst %,%_di$(objext),$(lib2difuncs))
+$(lib2difuncs-o): %_di$(objext): $(srcdir)/libgcc2.c
+	$(gcc_compile) -DLIBGCC2_UNITS_PER_WORD=4 -DL$* -c $< $(vis_hide)
+libgcc-objects += $(lib2difuncs-o)
+
+ifeq ($(enable_shared),yes)
+lib2difuncs-s-o = $(patsubst %,%_di_s$(objext),$(lib2difuncs))
+$(lib2difuncs-s-o): %_di_s$(objext): $(srcdir)/libgcc2.c
+	$(gcc_s_compile) -DLIBGCC2_UNITS_PER_WORD=4 -DL$* -c $<
+libgcc-s-objects += $(lib2difuncs-s-o)
+endif
+
+# Extra DImode divmod functions to build.
+lib2didivmodfuncs = _divdi3 _moddi3 _udivdi3 _umoddi3
+
+# Build lib2didivmodfuncs.
+lib2didivmod-o = $(patsubst %,%_di$(objext),$(lib2didivmodfuncs))
+$(lib2didivmod-o): %_di$(objext): $(srcdir)/libgcc2.c
+	$(gcc_compile) -DLIBGCC2_UNITS_PER_WORD=4 -DL$* -c $< \
+	  $(LIB2_DIVMOD_EXCEPTION_FLAGS) $(vis_hide)
+libgcc-objects += $(lib2didivmod-o)
+
+ifeq ($(enable_shared),yes)
+lib2didivmod-s-o = $(patsubst %,%_di_s$(objext),$(lib2didivmodfuncs))
+$(lib2didivmod-s-o): %_di_s$(objext): $(srcdir)/libgcc2.c
+	$(gcc_s_compile) -DLIBGCC2_UNITS_PER_WORD=4 -DL$* -c $< \
+	  $(LIB2_DIVMOD_EXCEPTION_FLAGS)
+libgcc-s-objects += $(lib2didivmod-s-o)
+endif
diff --git a/libgcc/config/pa/t-linux64 b/libgcc/config/pa/t-linux64
index 1d0a6ad..026b48b 100644
--- a/libgcc/config/pa/t-linux64
+++ b/libgcc/config/pa/t-linux64
@@ -1,8 +1,8 @@
 # Plug millicode routines into libgcc.a  We want these on both native and
 # cross compiles.
-# FIXME: Explain.
-LIB1ASMFUNCS := $(filter-out _dyncall, $(LIB1ASMFUNCS))
+LIB1ASMSRC = pa/milli64.S
+LIB1ASMFUNCS = _divI _divU _remI _remU _div_const _mulI
 
-LIB2ADD_ST = $(srcdir)/config/pa/linux-atomic.c
+HOST_LIBGCC2_CFLAGS += -Dpa64=1 -DELF=1 -DLINUX=1
 
-HOST_LIBGCC2_CFLAGS += -Dpa64=1 -DELF=1
+LIB2ADD_ST = $(srcdir)/config/pa/linux-atomic.c
diff --git a/libgcc/config/pa/t-softfp-sfdftf b/libgcc/config/pa/t-softfp-sfdftf
new file mode 100644
index 0000000..51ff051
--- /dev/null
+++ b/libgcc/config/pa/t-softfp-sfdftf
@@ -0,0 +1,8 @@
+# Soft float configuration for PA64
+LIB2ADD += $(srcdir)/config/pa/sfp-exceptions.c
+
+softfp_float_modes := sf df tf
+softfp_int_modes := si di ti
+softfp_extensions := sfdf sftf dftf
+softfp_truncations := dfsf tfsf tfdf
+softfp_exclude_libgcc2 := n
diff --git a/libgcc/config/rs6000/linux-unwind.h b/libgcc/config/rs6000/linux-unwind.h
index 8deccc1d..5ef9c1a 100644
--- a/libgcc/config/rs6000/linux-unwind.h
+++ b/libgcc/config/rs6000/linux-unwind.h
@@ -395,14 +395,21 @@ struct frame_layout
 };
 
 
-void ppc_backchain_fallback (struct _Unwind_Context *context, void *a)
+static void
+ppc_backchain_fallback (struct _Unwind_Context *context, void *a)
 {
   struct frame_layout *current;
   struct trace_arg *arg = a;
   int count;
 
-  /* Get the last address computed and start with the next.  */
+  /* Get the last address computed.  */
   current = context->cfa;
+
+  /* If the trace CFA is not the context CFA the backtrace is done.  */
+  if (arg == NULL || arg->cfa != current)
+	return;
+
+  /* Start with next address.  */
   current = current->backchain;
 
   for (count = arg->count; current != NULL; current = current->backchain)
diff --git a/libgcc/unwind.inc b/libgcc/unwind.inc
index 456a5ee..dc2f9c1 100644
--- a/libgcc/unwind.inc
+++ b/libgcc/unwind.inc
@@ -160,12 +160,13 @@ _Unwind_ForcedUnwind_Phase2 (struct _Unwind_Exception *exc,
 
       /* Set up fs to describe the FDE for the caller of cur_context.  */
       code = uw_frame_state_for (context, &fs);
-      if (code != _URC_NO_REASON && code != _URC_END_OF_STACK)
+      if (code != _URC_NO_REASON && code != _URC_END_OF_STACK
+	  && code != _URC_NORMAL_STOP)
 	return _URC_FATAL_PHASE2_ERROR;
 
       /* Unwind successful.  */
       action = _UA_FORCE_UNWIND | _UA_CLEANUP_PHASE;
-      if (code == _URC_END_OF_STACK)
+      if (code == _URC_END_OF_STACK || code == _URC_NORMAL_STOP)
 	action |= _UA_END_OF_STACK;
       stop_code = (*stop) (1, action, exc->exception_class, exc,
 			   context, stop_argument);
diff --git a/libgomp/ChangeLog b/libgomp/ChangeLog
index 50d14b3..21c9271 100644
--- a/libgomp/ChangeLog
+++ b/libgomp/ChangeLog
@@ -1,3 +1,142 @@
+2021-11-15  Tobias Burnus  <tobias@codesourcery.com>
+
+	* testsuite/libgomp.fortran/thread-limit-1.f90: New test.
+
+2021-11-15  Jakub Jelinek  <jakub@redhat.com>
+
+	* task.c (gomp_create_target_task): Copy args array as well.
+	* target.c (gomp_target_fallback): Add args argument.
+	Set gomp_icv (true)->thread_limit_var if thread_limit is present.
+	(GOMP_target): Adjust gomp_target_fallback caller.
+	(GOMP_target_ext): Likewise.
+	(gomp_target_task_fn): Likewise.
+	* config/nvptx/team.c (gomp_nvptx_main): Set
+	gomp_global_icv.thread_limit_var.
+	* testsuite/libgomp.c-c++-common/thread-limit-1.c: New test.
+
+2021-11-15  Jakub Jelinek  <jakub@redhat.com>
+
+	* config/nvptx/team.c (__gomp_team_num): Define as
+	__attribute__((shared)) var.
+	(gomp_nvptx_main): Initialize __gomp_team_num to 0.
+	* config/nvptx/target.c (__gomp_team_num): Declare as
+	extern __attribute__((shared)) var.
+	(GOMP_teams4): Use __gomp_team_num as the team number instead of
+	%ctaid.x.  If first, initialize it to %ctaid.x.  If num_teams_lower
+	is bigger than num_blocks, use num_teams_lower teams and arrange for
+	bumping of __gomp_team_num if !first and returning false once we run
+	out of teams.
+	* config/nvptx/teams.c (__gomp_team_num): Declare as
+	extern __attribute__((shared)) var.
+	(omp_get_team_num): Return __gomp_team_num value instead of %ctaid.x.
+
+2021-11-15  Jakub Jelinek  <jakub@redhat.com>
+
+	* testsuite/libgomp.c/teams-5.c: New test.
+
+2021-11-12  Jakub Jelinek  <jakub@redhat.com>
+
+	PR target/103201
+	* config/gcn/icv-device.c (omp_get_num_teams, omp_get_team_num): Move
+	to ...
+	* config/gcn/teams.c: ... here.  New file.
+
+2021-11-12  Chung-Lin Tang  <cltang@codesourcery.com>
+
+	* target.c (gomp_map_vars_existing): Add 'bool implicit' parameter, add
+	implicit map handling to allow a "superset" existing map as valid case.
+	(get_kind): Adjust to filter out GOMP_MAP_IMPLICIT bits in return value.
+	(get_implicit): New function to extract implicit status.
+	(gomp_map_fields_existing): Adjust arguments in calls to
+	gomp_map_vars_existing, and add uses of get_implicit.
+	(gomp_map_vars_internal): Likewise.
+	* testsuite/libgomp.c-c++-common/target-implicit-map-1.c: New test.
+
+2021-11-12  Jakub Jelinek  <jakub@redhat.com>
+
+	* libgomp_g.h (GOMP_teams4): Declare.
+	* libgomp.map (GOMP_5.1): Export GOMP_teams4.
+	* target.c (GOMP_teams4): New function.
+	* config/nvptx/target.c (GOMP_teams): Remove.
+	(GOMP_teams4): New function.
+	* config/gcn/target.c (GOMP_teams): Remove.
+	(GOMP_teams4): New function.
+	* testsuite/libgomp.c/teams-4.c (main): Expect exactly 2
+	teams instead of <= 2.
+	* testsuite/libgomp.c-c++-common/teams-2.c: New test.
+
+2021-11-11  Tobias Burnus  <tobias@codesourcery.com>
+
+	* testsuite/libgomp.fortran/teams-1.f90: New test.
+
+2021-11-11  Jakub Jelinek  <jakub@redhat.com>
+
+	* libgomp.h (struct gomp_thread): Add num_teams and team_num members.
+	* team.c (struct gomp_thread_start_data): Likewise.
+	(gomp_thread_start): Initialize thr->num_teams and thr->team_num.
+	(gomp_team_start): Initialize start_data->num_teams and
+	start_data->team_num.  Update nthr->num_teams and nthr->team_num.
+	* teams.c (gomp_num_teams, gomp_team_num): Remove.
+	(GOMP_teams_reg): Set and restore thr->num_teams and thr->team_num
+	instead of gomp_num_teams and gomp_team_num.
+	(omp_get_num_teams): Use thr->num_teams + 1 instead of gomp_num_teams.
+	(omp_get_team_num): Use thr->team_num instead of gomp_team_num.
+	* testsuite/libgomp.c/teams-4.c: New test.
+
+2021-11-11  Jakub Jelinek  <jakub@redhat.com>
+
+	* testsuite/libgomp.c-c++-common/teams-1.c: New test.
+
+2021-11-09  Thomas Schwinge  <thomas@codesourcery.com>
+
+	* env.c (parse_gomp_openacc_dim): Restore parsing.
+
+2021-10-30  Tobias Burnus  <tobias@codesourcery.com>
+
+	PR middle-end/102972
+	* testsuite/libgomp.c-c++-common/icv-3.c: Nest API calls inside
+	parallel construct.
+	* testsuite/libgomp.c-c++-common/icv-4.c: Likewise.
+	* testsuite/libgomp.c/target-3.c: Likewise.
+	* testsuite/libgomp.c/target-5.c: Likewise.
+	* testsuite/libgomp.c/target-6.c: Likewise.
+	* testsuite/libgomp.c/target-teams-1.c: Likewise.
+	* testsuite/libgomp.c/teams-1.c: Likewise.
+	* testsuite/libgomp.c/thread-limit-2.c: Likewise.
+	* testsuite/libgomp.c/thread-limit-3.c: Likewise.
+	* testsuite/libgomp.c/thread-limit-4.c: Likewise.
+	* testsuite/libgomp.c/thread-limit-5.c: Likewise.
+	* testsuite/libgomp.fortran/icv-3.f90: Likewise.
+	* testsuite/libgomp.fortran/icv-4.f90: Likewise.
+	* testsuite/libgomp.fortran/teams1.f90: Likewise.
+
+2021-10-29  Aldy Hernandez  <aldyh@redhat.com>
+
+	* testsuite/libgomp.graphite/force-parallel-4.c: Adjust for threading changes.
+	* testsuite/libgomp.graphite/force-parallel-8.c: Same.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	* libgomp.texi (OpenMP 5.0): Mention that Non-rectangular loop nests
+	aren't implemented for Fortran yet.
+
+2021-10-27  Jakub Jelinek  <jakub@redhat.com>
+
+	* testsuite/libgomp.c/loop-26.c: New test.
+	* testsuite/libgomp.c/loop-27.c: New test.
+
+2021-10-25  Tobias Burnus  <tobias@codesourcery.com>
+
+	PR testsuite/102910
+	* testsuite/libgomp.oacc-c-c++-common/loop-gwv-2.c: Use __builtin_alloca
+	instead of #include <alloca.h> + alloca.
+
+2021-10-21  Chung-Lin Tang  <cltang@codesourcery.com>
+
+	* libgomp.texi (Support of strictly structured blocks in Fortran):
+	Adjust to 'Y'.
+	* testsuite/libgomp.fortran/task-reduction-16.f90: Adjust testcase.
+
 2021-10-20  Chung-Lin Tang  <cltang@codesourcery.com>
 
 	* testsuite/libgomp.fortran/target-in-reduction-1.f90: New test.
diff --git a/libgomp/config/gcn/icv-device.c b/libgomp/config/gcn/icv-device.c
index 34e0f83..2de46a1f 100644
--- a/libgomp/config/gcn/icv-device.c
+++ b/libgomp/config/gcn/icv-device.c
@@ -52,18 +52,6 @@ omp_get_num_devices (void)
 }
 
 int
-omp_get_num_teams (void)
-{
-  return gomp_num_teams_var + 1;
-}
-
-int __attribute__ ((__optimize__ ("O2")))
-omp_get_team_num (void)
-{
-  return __builtin_gcn_dim_pos (0);
-}
-
-int
 omp_is_initial_device (void)
 {
   /* AMD GCN is an accelerator-only target.  */
@@ -84,7 +72,5 @@ ialias (omp_set_default_device)
 ialias (omp_get_default_device)
 ialias (omp_get_initial_device)
 ialias (omp_get_num_devices)
-ialias (omp_get_num_teams)
-ialias (omp_get_team_num)
 ialias (omp_is_initial_device)
 ialias (omp_get_device_num)
diff --git a/libgomp/config/gcn/target.c b/libgomp/config/gcn/target.c
index a93ecc9..ba83000 100644
--- a/libgomp/config/gcn/target.c
+++ b/libgomp/config/gcn/target.c
@@ -26,9 +26,12 @@
 #include "libgomp.h"
 #include <limits.h>
 
-void
-GOMP_teams (unsigned int num_teams, unsigned int thread_limit)
+bool
+GOMP_teams4 (unsigned int num_teams_lower, unsigned int num_teams_upper,
+	     unsigned int thread_limit, bool first)
 {
+  if (!first)
+    return false;
   if (thread_limit)
     {
       struct gomp_task_icv *icv = gomp_icv (true);
@@ -38,14 +41,15 @@ GOMP_teams (unsigned int num_teams, unsigned int thread_limit)
   unsigned int num_workgroups, workgroup_id;
   num_workgroups = __builtin_gcn_dim_size (0);
   workgroup_id = __builtin_gcn_dim_pos (0);
-  if (!num_teams || num_teams >= num_workgroups)
-    num_teams = num_workgroups;
-  else if (workgroup_id >= num_teams)
-    {
-      gomp_free_thread (gcn_thrs ());
-      exit (0);
-    }
-  gomp_num_teams_var = num_teams - 1;
+  /* FIXME: If num_teams_lower > num_workgroups, we want to loop
+     multiple times at least for some workgroups.  */
+  (void) num_teams_lower;
+  if (!num_teams_upper || num_teams_upper >= num_workgroups)
+    num_teams_upper = num_workgroups;
+  else if (workgroup_id >= num_teams_upper)
+    return false;
+  gomp_num_teams_var = num_teams_upper - 1;
+  return true;
 }
 
 int
diff --git a/libgomp/config/gcn/teams.c b/libgomp/config/gcn/teams.c
new file mode 100644
index 0000000..40d50d8
--- /dev/null
+++ b/libgomp/config/gcn/teams.c
@@ -0,0 +1,54 @@
+/* Copyright (C) 2015-2021 Free Software Foundation, Inc.
+
+   This file is part of the GNU Offloading and Multi Processing Library
+   (libgomp).
+
+   Libgomp is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+   more details.
+
+   Under Section 7 of GPL version 3, you are granted additional
+   permissions described in the GCC Runtime Library Exception, version
+   3.1, as published by the Free Software Foundation.
+
+   You should have received a copy of the GNU General Public License and
+   a copy of the GCC Runtime Library Exception along with this program;
+   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* This file defines OpenMP API entry points that accelerator targets are
+   expected to replace.  */
+
+#include "libgomp.h"
+
+void
+GOMP_teams_reg (void (*fn) (void *), void *data, unsigned int num_teams,
+		unsigned int thread_limit, unsigned int flags)
+{
+  (void) fn;
+  (void) data;
+  (void) flags;
+  (void) num_teams;
+  (void) thread_limit;
+}
+
+int
+omp_get_num_teams (void)
+{
+  return gomp_num_teams_var + 1;
+}
+
+int __attribute__ ((__optimize__ ("O2")))
+omp_get_team_num (void)
+{
+  return __builtin_gcn_dim_pos (0);
+}
+
+ialias (omp_get_num_teams)
+ialias (omp_get_team_num)
diff --git a/libgomp/config/nvptx/target.c b/libgomp/config/nvptx/target.c
index e4140e4..2eb545f 100644
--- a/libgomp/config/nvptx/target.c
+++ b/libgomp/config/nvptx/target.c
@@ -26,26 +26,43 @@
 #include "libgomp.h"
 #include <limits.h>
 
-void
-GOMP_teams (unsigned int num_teams, unsigned int thread_limit)
+extern int __gomp_team_num __attribute__((shared));
+
+bool
+GOMP_teams4 (unsigned int num_teams_lower, unsigned int num_teams_upper,
+	     unsigned int thread_limit, bool first)
 {
+  unsigned int num_blocks, block_id;
+  asm ("mov.u32 %0, %%nctaid.x;" : "=r" (num_blocks));
+  if (!first)
+    {
+      unsigned int team_num;
+      if (num_blocks > gomp_num_teams_var)
+	return false;
+      team_num = __gomp_team_num;
+      if (team_num > gomp_num_teams_var - num_blocks)
+	return false;
+      __gomp_team_num = team_num + num_blocks;
+      return true;
+    }
   if (thread_limit)
     {
       struct gomp_task_icv *icv = gomp_icv (true);
       icv->thread_limit_var
 	= thread_limit > INT_MAX ? UINT_MAX : thread_limit;
     }
-  unsigned int num_blocks, block_id;
-  asm ("mov.u32 %0, %%nctaid.x;" : "=r" (num_blocks));
+  if (!num_teams_upper)
+    num_teams_upper = num_blocks;
+  else if (num_blocks < num_teams_lower)
+    num_teams_upper = num_teams_lower;
+  else if (num_blocks < num_teams_upper)
+    num_teams_upper = num_blocks;
   asm ("mov.u32 %0, %%ctaid.x;" : "=r" (block_id));
-  if (!num_teams || num_teams >= num_blocks)
-    num_teams = num_blocks;
-  else if (block_id >= num_teams)
-    {
-      gomp_free_thread (nvptx_thrs);
-      asm ("exit;");
-    }
-  gomp_num_teams_var = num_teams - 1;
+  if (block_id >= num_teams_upper)
+    return false;
+  __gomp_team_num = block_id;
+  gomp_num_teams_var = num_teams_upper - 1;
+  return true;
 }
 
 int
diff --git a/libgomp/config/nvptx/team.c b/libgomp/config/nvptx/team.c
index e235964..310eb28 100644
--- a/libgomp/config/nvptx/team.c
+++ b/libgomp/config/nvptx/team.c
@@ -32,6 +32,7 @@
 #include <string.h>
 
 struct gomp_thread *nvptx_thrs __attribute__((shared,nocommon));
+int __gomp_team_num __attribute__((shared,nocommon));
 
 static void gomp_thread_start (struct gomp_thread_pool *);
 
@@ -54,9 +55,11 @@ gomp_nvptx_main (void (*fn) (void *), void *fn_data)
   if (tid == 0)
     {
       gomp_global_icv.nthreads_var = ntids;
+      gomp_global_icv.thread_limit_var = ntids;
       /* Starting additional threads is not supported.  */
       gomp_global_icv.dyn_var = true;
 
+      __gomp_team_num = 0;
       nvptx_thrs = alloca (ntids * sizeof (*nvptx_thrs));
       memset (nvptx_thrs, 0, ntids * sizeof (*nvptx_thrs));
 
diff --git a/libgomp/config/nvptx/teams.c b/libgomp/config/nvptx/teams.c
index 631cb1c..60fb11d 100644
--- a/libgomp/config/nvptx/teams.c
+++ b/libgomp/config/nvptx/teams.c
@@ -28,6 +28,8 @@
 
 #include "libgomp.h"
 
+extern int __gomp_team_num __attribute__((shared));
+
 void
 GOMP_teams_reg (void (*fn) (void *), void *data, unsigned int num_teams,
 		unsigned int thread_limit, unsigned int flags)
@@ -48,9 +50,7 @@ omp_get_num_teams (void)
 int
 omp_get_team_num (void)
 {
-  int ctaid;
-  asm ("mov.u32 %0, %%ctaid.x;" : "=r" (ctaid));
-  return ctaid;
+  return __gomp_team_num;
 }
 
 ialias (omp_get_num_teams)
diff --git a/libgomp/env.c b/libgomp/env.c
index df10ff6..75018e8 100644
--- a/libgomp/env.c
+++ b/libgomp/env.c
@@ -1243,7 +1243,7 @@ parse_gomp_openacc_dim (void)
 
       errno = 0;
       val = strtol (pos, &eptr, 10);
-      if (errno || eptr != pos || val < 0 || (unsigned)val != val)
+      if (errno || eptr == pos || val < 0 || (unsigned)val != val)
 	break;
 
       goacc_default_dims[i] = (int)val;
diff --git a/libgomp/libgomp.h b/libgomp/libgomp.h
index 023d8a8..ceef643 100644
--- a/libgomp/libgomp.h
+++ b/libgomp/libgomp.h
@@ -768,6 +768,14 @@ struct gomp_thread
   /* User pthread thread pool */
   struct gomp_thread_pool *thread_pool;
 
+#ifdef LIBGOMP_USE_PTHREADS
+  /* omp_get_num_teams () - 1.  */
+  unsigned int num_teams;
+
+  /* omp_get_team_num ().  */
+  unsigned int team_num;
+#endif
+
 #if defined(LIBGOMP_USE_PTHREADS) \
     && (!defined(HAVE_TLS) \
 	|| !defined(__GLIBC__) \
diff --git a/libgomp/libgomp.map b/libgomp/libgomp.map
index 4e5397a..2ac5809 100644
--- a/libgomp/libgomp.map
+++ b/libgomp/libgomp.map
@@ -399,6 +399,7 @@ GOMP_5.1 {
 	GOMP_error;
 	GOMP_scope_start;
 	GOMP_warning;
+	GOMP_teams4;
 } GOMP_5.0.1;
 
 OACC_2.0 {
diff --git a/libgomp/libgomp.texi b/libgomp/libgomp.texi
index e711707..9fc4545 100644
--- a/libgomp/libgomp.texi
+++ b/libgomp/libgomp.texi
@@ -189,7 +189,7 @@ The OpenMP 4.5 specification is fully supported.
 @item @code{requires} directive @tab P
       @tab Only fulfillable requirement is @code{atomic_default_mem_order}
 @item @code{teams} construct outside an enclosing target region @tab Y @tab
-@item Non-rectangular loop nests @tab Y @tab
+@item Non-rectangular loop nests @tab P @tab Only C/C++
 @item @code{!=} as relational-op in canonical loop form for C/C++ @tab Y @tab
 @item @code{nonmonotonic} as default loop schedule modifier for worksharing-loop
       constructs @tab Y @tab
@@ -292,7 +292,7 @@ The OpenMP 4.5 specification is fully supported.
       clauses of the taskloop construct @tab Y @tab
 @item @code{align} clause/modifier in @code{allocate} directive/clause
       and @code{allocator} directive @tab P @tab C/C++ on clause only
-@item @code{thread_limit} clause to @code{target} construct @tab N @tab
+@item @code{thread_limit} clause to @code{target} construct @tab Y @tab
 @item @code{has_device_addr} clause to @code{target} construct @tab N @tab
 @item iterators in @code{target update} motion clauses and @code{map}
       clauses @tab N @tab
diff --git a/libgomp/libgomp_g.h b/libgomp/libgomp_g.h
index 40e5cf0..ab50f65 100644
--- a/libgomp/libgomp_g.h
+++ b/libgomp/libgomp_g.h
@@ -355,6 +355,7 @@ extern void GOMP_target_enter_exit_data (int, size_t, void **, size_t *,
 					 unsigned short *, unsigned int,
 					 void **);
 extern void GOMP_teams (unsigned int, unsigned int);
+extern bool GOMP_teams4 (unsigned int, unsigned int, unsigned int, bool);
 
 /* teams.c */
 
diff --git a/libgomp/target.c b/libgomp/target.c
index 196dba4..5d3103a 100644
--- a/libgomp/target.c
+++ b/libgomp/target.c
@@ -539,7 +539,7 @@ static inline void
 gomp_map_vars_existing (struct gomp_device_descr *devicep,
 			struct goacc_asyncqueue *aq, splay_tree_key oldn,
 			splay_tree_key newn, struct target_var_desc *tgt_var,
-			unsigned char kind, bool always_to_flag,
+			unsigned char kind, bool always_to_flag, bool implicit,
 			struct gomp_coalesce_buf *cbuf,
 			htab_t *refcount_set)
 {
@@ -550,11 +550,22 @@ gomp_map_vars_existing (struct gomp_device_descr *devicep,
   tgt_var->always_copy_from = GOMP_MAP_ALWAYS_FROM_P (kind);
   tgt_var->is_attach = false;
   tgt_var->offset = newn->host_start - oldn->host_start;
-  tgt_var->length = newn->host_end - newn->host_start;
+
+  /* For implicit maps, old contained in new is valid.  */
+  bool implicit_subset = (implicit
+			  && newn->host_start <= oldn->host_start
+			  && oldn->host_end <= newn->host_end);
+  if (implicit_subset)
+    tgt_var->length = oldn->host_end - oldn->host_start;
+  else
+    tgt_var->length = newn->host_end - newn->host_start;
 
   if ((kind & GOMP_MAP_FLAG_FORCE)
-      || oldn->host_start > newn->host_start
-      || oldn->host_end < newn->host_end)
+      /* For implicit maps, old contained in new is valid.  */
+      || !(implicit_subset
+	   /* Otherwise, new contained inside old is considered valid.  */
+	   || (oldn->host_start <= newn->host_start
+	       && newn->host_end <= oldn->host_end)))
     {
       gomp_mutex_unlock (&devicep->lock);
       gomp_fatal ("Trying to map into device [%p..%p) object when "
@@ -564,11 +575,17 @@ gomp_map_vars_existing (struct gomp_device_descr *devicep,
     }
 
   if (GOMP_MAP_ALWAYS_TO_P (kind) || always_to_flag)
-    gomp_copy_host2dev (devicep, aq,
-			(void *) (oldn->tgt->tgt_start + oldn->tgt_offset
-				  + newn->host_start - oldn->host_start),
-			(void *) newn->host_start,
-			newn->host_end - newn->host_start, false, cbuf);
+    {
+      /* Implicit + always should not happen. If this does occur, below
+	 address/length adjustment is a TODO.  */
+      assert (!implicit_subset);
+
+      gomp_copy_host2dev (devicep, aq,
+			  (void *) (oldn->tgt->tgt_start + oldn->tgt_offset
+				    + newn->host_start - oldn->host_start),
+			  (void *) newn->host_start,
+			  newn->host_end - newn->host_start, false, cbuf);
+    }
 
   gomp_increment_refcount (oldn, refcount_set);
 }
@@ -576,8 +593,24 @@ gomp_map_vars_existing (struct gomp_device_descr *devicep,
 static int
 get_kind (bool short_mapkind, void *kinds, int idx)
 {
-  return short_mapkind ? ((unsigned short *) kinds)[idx]
-		       : ((unsigned char *) kinds)[idx];
+  if (!short_mapkind)
+    return ((unsigned char *) kinds)[idx];
+
+  int val = ((unsigned short *) kinds)[idx];
+  if (GOMP_MAP_IMPLICIT_P (val))
+    val &= ~GOMP_MAP_IMPLICIT;
+  return val;
+}
+
+
+static bool
+get_implicit (bool short_mapkind, void *kinds, int idx)
+{
+  if (!short_mapkind)
+    return false;
+
+  int val = ((unsigned short *) kinds)[idx];
+  return GOMP_MAP_IMPLICIT_P (val);
 }
 
 static void
@@ -631,6 +664,7 @@ gomp_map_fields_existing (struct target_mem_desc *tgt,
   struct splay_tree_s *mem_map = &devicep->mem_map;
   struct splay_tree_key_s cur_node;
   int kind;
+  bool implicit;
   const bool short_mapkind = true;
   const int typemask = short_mapkind ? 0xff : 0x7;
 
@@ -638,12 +672,14 @@ gomp_map_fields_existing (struct target_mem_desc *tgt,
   cur_node.host_end = cur_node.host_start + sizes[i];
   splay_tree_key n2 = splay_tree_lookup (mem_map, &cur_node);
   kind = get_kind (short_mapkind, kinds, i);
+  implicit = get_implicit (short_mapkind, kinds, i);
   if (n2
       && n2->tgt == n->tgt
       && n2->host_start - n->host_start == n2->tgt_offset - n->tgt_offset)
     {
       gomp_map_vars_existing (devicep, aq, n2, &cur_node, &tgt->list[i],
-			      kind & typemask, false, cbuf, refcount_set);
+			      kind & typemask, false, implicit, cbuf,
+			      refcount_set);
       return;
     }
   if (sizes[i] == 0)
@@ -659,7 +695,8 @@ gomp_map_fields_existing (struct target_mem_desc *tgt,
 		 == n2->tgt_offset - n->tgt_offset)
 	    {
 	      gomp_map_vars_existing (devicep, aq, n2, &cur_node, &tgt->list[i],
-				      kind & typemask, false, cbuf, refcount_set);
+				      kind & typemask, false, implicit, cbuf,
+				      refcount_set);
 	      return;
 	    }
 	}
@@ -671,7 +708,8 @@ gomp_map_fields_existing (struct target_mem_desc *tgt,
 	  && n2->host_start - n->host_start == n2->tgt_offset - n->tgt_offset)
 	{
 	  gomp_map_vars_existing (devicep, aq, n2, &cur_node, &tgt->list[i],
-				  kind & typemask, false, cbuf, refcount_set);
+				  kind & typemask, false, implicit, cbuf,
+				  refcount_set);
 	  return;
 	}
     }
@@ -903,6 +941,7 @@ gomp_map_vars_internal (struct gomp_device_descr *devicep,
   for (i = 0; i < mapnum; i++)
     {
       int kind = get_kind (short_mapkind, kinds, i);
+      bool implicit = get_implicit (short_mapkind, kinds, i);
       if (hostaddrs[i] == NULL
 	  || (kind & typemask) == GOMP_MAP_FIRSTPRIVATE_INT)
 	{
@@ -1085,8 +1124,8 @@ gomp_map_vars_internal (struct gomp_device_descr *devicep,
 		}
 	    }
 	  gomp_map_vars_existing (devicep, aq, n, &cur_node, &tgt->list[i],
-				  kind & typemask, always_to_cnt > 0, NULL,
-				  refcount_set);
+				  kind & typemask, always_to_cnt > 0, implicit,
+				  NULL, refcount_set);
 	  i += always_to_cnt;
 	}
       else
@@ -1256,6 +1295,7 @@ gomp_map_vars_internal (struct gomp_device_descr *devicep,
 	else if (tgt->list[i].key == NULL)
 	  {
 	    int kind = get_kind (short_mapkind, kinds, i);
+	    bool implicit = get_implicit (short_mapkind, kinds, i);
 	    if (hostaddrs[i] == NULL)
 	      continue;
 	    switch (kind & typemask)
@@ -1415,7 +1455,7 @@ gomp_map_vars_internal (struct gomp_device_descr *devicep,
 	    splay_tree_key n = splay_tree_lookup (mem_map, k);
 	    if (n && n->refcount != REFCOUNT_LINK)
 	      gomp_map_vars_existing (devicep, aq, n, k, &tgt->list[i],
-				      kind & typemask, false, cbufp,
+				      kind & typemask, false, implicit, cbufp,
 				      refcount_set);
 	    else
 	      {
@@ -2322,7 +2362,7 @@ gomp_unload_device (struct gomp_device_descr *devicep)
 
 static void
 gomp_target_fallback (void (*fn) (void *), void **hostaddrs,
-		      struct gomp_device_descr *devicep)
+		      struct gomp_device_descr *devicep, void **args)
 {
   struct gomp_thread old_thr, *thr = gomp_thread ();
 
@@ -2338,6 +2378,25 @@ gomp_target_fallback (void (*fn) (void *), void **hostaddrs,
       thr->place = old_thr.place;
       thr->ts.place_partition_len = gomp_places_list_len;
     }
+  if (args)
+    while (*args)
+      {
+	intptr_t id = (intptr_t) *args++, val;
+	if (id & GOMP_TARGET_ARG_SUBSEQUENT_PARAM)
+	  val = (intptr_t) *args++;
+	else
+	  val = id >> GOMP_TARGET_ARG_VALUE_SHIFT;
+	if ((id & GOMP_TARGET_ARG_DEVICE_MASK) != GOMP_TARGET_ARG_DEVICE_ALL)
+	  continue;
+	id &= GOMP_TARGET_ARG_ID_MASK;
+	if (id != GOMP_TARGET_ARG_THREAD_LIMIT)
+	  continue;
+	val = val > INT_MAX ? INT_MAX : val;
+	if (val)
+	  gomp_icv (true)->thread_limit_var = val;
+	break;
+      }
+
   fn (hostaddrs);
   gomp_free_thread (thr);
   *thr = old_thr;
@@ -2438,7 +2497,7 @@ GOMP_target (int device, void (*fn) (void *), const void *unused,
       /* All shared memory devices should use the GOMP_target_ext function.  */
       || devicep->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM
       || !(fn_addr = gomp_get_target_fn_addr (devicep, fn)))
-    return gomp_target_fallback (fn, hostaddrs, devicep);
+    return gomp_target_fallback (fn, hostaddrs, devicep, NULL);
 
   htab_t refcount_set = htab_create (mapnum);
   struct target_mem_desc *tgt_vars
@@ -2577,7 +2636,7 @@ GOMP_target_ext (int device, void (*fn) (void *), size_t mapnum,
 				      tgt_align, tgt_size);
 	    }
 	}
-      gomp_target_fallback (fn, hostaddrs, devicep);
+      gomp_target_fallback (fn, hostaddrs, devicep, args);
       return;
     }
 
@@ -3012,7 +3071,8 @@ gomp_target_task_fn (void *data)
 	  || (devicep->can_run_func && !devicep->can_run_func (fn_addr)))
 	{
 	  ttask->state = GOMP_TARGET_TASK_FALLBACK;
-	  gomp_target_fallback (ttask->fn, ttask->hostaddrs, devicep);
+	  gomp_target_fallback (ttask->fn, ttask->hostaddrs, devicep,
+				ttask->args);
 	  return false;
 	}
 
@@ -3088,6 +3148,32 @@ GOMP_teams (unsigned int num_teams, unsigned int thread_limit)
   (void) num_teams;
 }
 
+bool
+GOMP_teams4 (unsigned int num_teams_low, unsigned int num_teams_high,
+	     unsigned int thread_limit, bool first)
+{
+  struct gomp_thread *thr = gomp_thread ();
+  if (first)
+    {
+      if (thread_limit)
+	{
+	  struct gomp_task_icv *icv = gomp_icv (true);
+	  icv->thread_limit_var
+	    = thread_limit > INT_MAX ? UINT_MAX : thread_limit;
+	}
+      (void) num_teams_high;
+      if (num_teams_low == 0)
+	num_teams_low = 1;
+      thr->num_teams = num_teams_low - 1;
+      thr->team_num = 0;
+    }
+  else if (thr->team_num == thr->num_teams)
+    return false;
+  else
+    ++thr->team_num;
+  return true;
+}
+
 void *
 omp_target_alloc (size_t size, int device_num)
 {
diff --git a/libgomp/task.c b/libgomp/task.c
index feb4796..414ca6e 100644
--- a/libgomp/task.c
+++ b/libgomp/task.c
@@ -745,6 +745,7 @@ gomp_create_target_task (struct gomp_device_descr *devicep,
   size_t depend_size = 0;
   uintptr_t depend_cnt = 0;
   size_t tgt_align = 0, tgt_size = 0;
+  uintptr_t args_cnt = 0;
 
   if (depend != NULL)
     {
@@ -769,10 +770,22 @@ gomp_create_target_task (struct gomp_device_descr *devicep,
 	tgt_size += tgt_align - 1;
       else
 	tgt_size = 0;
+      if (args)
+	{
+	  void **cargs = args;
+	  while (*cargs)
+	    {
+	      intptr_t id = (intptr_t) *cargs++;
+	      if (id & GOMP_TARGET_ARG_SUBSEQUENT_PARAM)
+		cargs++;
+	    }
+	  args_cnt = cargs + 1 - args;
+	}
     }
 
   task = gomp_malloc (sizeof (*task) + depend_size
 		      + sizeof (*ttask)
+		      + args_cnt * sizeof (void *)
 		      + mapnum * (sizeof (void *) + sizeof (size_t)
 				  + sizeof (unsigned short))
 		      + tgt_size);
@@ -785,9 +798,18 @@ gomp_create_target_task (struct gomp_device_descr *devicep,
   ttask->devicep = devicep;
   ttask->fn = fn;
   ttask->mapnum = mapnum;
-  ttask->args = args;
   memcpy (ttask->hostaddrs, hostaddrs, mapnum * sizeof (void *));
-  ttask->sizes = (size_t *) &ttask->hostaddrs[mapnum];
+  if (args_cnt)
+    {
+      ttask->args = (void **) &ttask->hostaddrs[mapnum];
+      memcpy (ttask->args, args, args_cnt * sizeof (void *));
+      ttask->sizes = (size_t *) &ttask->args[args_cnt];
+    }
+  else
+    {
+      ttask->args = args;
+      ttask->sizes = (size_t *) &ttask->hostaddrs[mapnum];
+    }
   memcpy (ttask->sizes, sizes, mapnum * sizeof (size_t));
   ttask->kinds = (unsigned short *) &ttask->sizes[mapnum];
   memcpy (ttask->kinds, kinds, mapnum * sizeof (unsigned short));
diff --git a/libgomp/team.c b/libgomp/team.c
index 11a2521..3bcc817 100644
--- a/libgomp/team.c
+++ b/libgomp/team.c
@@ -56,6 +56,8 @@ struct gomp_thread_start_data
   struct gomp_task *task;
   struct gomp_thread_pool *thread_pool;
   unsigned int place;
+  unsigned int num_teams;
+  unsigned int team_num;
   bool nested;
   pthread_t handle;
 };
@@ -88,6 +90,8 @@ gomp_thread_start (void *xdata)
   thr->ts = data->ts;
   thr->task = data->task;
   thr->place = data->place;
+  thr->num_teams = data->num_teams;
+  thr->team_num = data->team_num;
 #ifdef GOMP_NEEDS_THREAD_HANDLE
   thr->handle = data->handle;
 #endif
@@ -645,6 +649,8 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
 	  nthr->ts.single_count = 0;
 #endif
 	  nthr->ts.static_trip = 0;
+	  nthr->num_teams = thr->num_teams;
+	  nthr->team_num = thr->team_num;
 	  nthr->task = &team->implicit_task[i];
 	  nthr->place = place;
 	  gomp_init_task (nthr->task, task, icv);
@@ -833,6 +839,8 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
       start_data->ts.single_count = 0;
 #endif
       start_data->ts.static_trip = 0;
+      start_data->num_teams = thr->num_teams;
+      start_data->team_num = thr->team_num;
       start_data->task = &team->implicit_task[i];
       gomp_init_task (start_data->task, task, icv);
       team->implicit_task[i].icv.nthreads_var = nthreads_var;
diff --git a/libgomp/teams.c b/libgomp/teams.c
index 9409f8e..a93fb7c 100644
--- a/libgomp/teams.c
+++ b/libgomp/teams.c
@@ -28,14 +28,12 @@
 #include "libgomp.h"
 #include <limits.h>
 
-static unsigned gomp_num_teams = 1, gomp_team_num = 0;
-
 void
 GOMP_teams_reg (void (*fn) (void *), void *data, unsigned int num_teams,
 		unsigned int thread_limit, unsigned int flags)
 {
+  struct gomp_thread *thr = gomp_thread ();
   (void) flags;
-  (void) num_teams;
   unsigned old_thread_limit_var = 0;
   if (thread_limit == 0)
     thread_limit = gomp_teams_thread_limit_var;
@@ -48,11 +46,11 @@ GOMP_teams_reg (void (*fn) (void *), void *data, unsigned int num_teams,
     }
   if (num_teams == 0)
     num_teams = gomp_nteams_var ? gomp_nteams_var : 3;
-  gomp_num_teams = num_teams;
-  for (gomp_team_num = 0; gomp_team_num < num_teams; gomp_team_num++)
+  thr->num_teams = num_teams - 1;
+  for (thr->team_num = 0; thr->team_num < num_teams; thr->team_num++)
     fn (data);
-  gomp_num_teams = 1;
-  gomp_team_num = 0;
+  thr->num_teams = 0;
+  thr->team_num = 0;
   if (thread_limit)
     {
       struct gomp_task_icv *icv = gomp_icv (true);
@@ -63,13 +61,15 @@ GOMP_teams_reg (void (*fn) (void *), void *data, unsigned int num_teams,
 int
 omp_get_num_teams (void)
 {
-  return gomp_num_teams;
+  struct gomp_thread *thr = gomp_thread ();
+  return thr->num_teams + 1;
 }
 
 int
 omp_get_team_num (void)
 {
-  return gomp_team_num;
+  struct gomp_thread *thr = gomp_thread ();
+  return thr->team_num;
 }
 
 ialias (omp_get_num_teams)
diff --git a/libgomp/testsuite/libgomp.c-c++-common/icv-3.c b/libgomp/testsuite/libgomp.c-c++-common/icv-3.c
index 54cbf99..8bf24d9 100644
--- a/libgomp/testsuite/libgomp.c-c++-common/icv-3.c
+++ b/libgomp/testsuite/libgomp.c-c++-common/icv-3.c
@@ -18,6 +18,7 @@ main ()
     abort ();
   #pragma omp teams
   {
+    #pragma omp parallel if(0)
     if (omp_get_max_teams () != 7
 	|| omp_get_teams_thread_limit () != 15
 	|| omp_get_num_teams () < 1
@@ -30,6 +31,7 @@ main ()
   }
   #pragma omp teams num_teams(5) thread_limit (13)
   {
+    #pragma omp parallel if(0)
     if (omp_get_max_teams () != 7
 	|| omp_get_teams_thread_limit () != 15
 	|| omp_get_num_teams () != 5
@@ -41,6 +43,7 @@ main ()
   }
   #pragma omp teams num_teams(8) thread_limit (16)
   {
+    #pragma omp parallel if(0)
     if (omp_get_max_teams () != 7
 	|| omp_get_teams_thread_limit () != 15
 	|| omp_get_num_teams () != 8
diff --git a/libgomp/testsuite/libgomp.c-c++-common/icv-4.c b/libgomp/testsuite/libgomp.c-c++-common/icv-4.c
index 6cb671d..b987a33 100644
--- a/libgomp/testsuite/libgomp.c-c++-common/icv-4.c
+++ b/libgomp/testsuite/libgomp.c-c++-common/icv-4.c
@@ -26,6 +26,7 @@ main ()
     omp_set_teams_thread_limit (12);
   #pragma omp teams
   {
+    #pragma omp parallel if(0)
     if (omp_get_max_teams () != 6
 	|| omp_get_teams_thread_limit () != 12
 	|| omp_get_num_teams () < 1
diff --git a/libgomp/testsuite/libgomp.c-c++-common/target-implicit-map-1.c b/libgomp/testsuite/libgomp.c-c++-common/target-implicit-map-1.c
new file mode 100644
index 0000000..f2e7293
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c-c++-common/target-implicit-map-1.c
@@ -0,0 +1,31 @@
+#ifdef __cplusplus
+extern "C"
+#else
+extern
+#endif
+void abort (void);
+
+int
+main (void)
+{
+  #define N 5
+  int array[N][N];
+
+  for (int i = 0; i < N; i++)
+    {
+      #pragma omp target enter data map(alloc: array[i:1][0:N])
+
+      #pragma omp target
+      for (int j = 0; j < N; j++)
+	array[i][j] = i + j;
+
+      #pragma omp target exit data map(from: array[i:1][0:N])
+    }
+
+  for (int i = 0; i < N; i++)
+    for (int j = 0; j < N; j++)
+      if (array[i][j] != i + j)
+	abort ();
+
+  return 0;
+}
diff --git a/libgomp/testsuite/libgomp.c-c++-common/teams-1.c b/libgomp/testsuite/libgomp.c-c++-common/teams-1.c
new file mode 100644
index 0000000..76189ef
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c-c++-common/teams-1.c
@@ -0,0 +1,26 @@
+#include <omp.h>
+#include <stdlib.h>
+
+int
+main ()
+{
+  #pragma omp teams num_teams (5)
+  {
+    if (omp_get_num_teams () != 5)
+      abort ();
+    #pragma omp distribute dist_schedule(static,1)
+    for (int i = 0; i < 5; ++i)
+      if (omp_get_team_num () != i)
+	abort ();
+  }
+  #pragma omp teams num_teams (7 : 9)
+  {
+    if (omp_get_num_teams () < 7 || omp_get_num_teams () > 9)
+      abort ();
+    #pragma omp distribute dist_schedule(static,1)
+    for (int i = 0; i < omp_get_num_teams (); ++i)
+      if (omp_get_team_num () != i)
+	abort ();
+  }
+  return 0;
+}
diff --git a/libgomp/testsuite/libgomp.c-c++-common/teams-2.c b/libgomp/testsuite/libgomp.c-c++-common/teams-2.c
new file mode 100644
index 0000000..316bcfe
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c-c++-common/teams-2.c
@@ -0,0 +1,70 @@
+#include <omp.h>
+#include <stdlib.h>
+
+int
+foo ()
+{
+  return 934;
+}
+
+int
+main ()
+{
+  int a[934] = {};
+  int k, e;
+  #pragma omp target map(a)
+  #pragma omp teams num_teams (foo ())
+  {
+    int i = omp_get_team_num ();
+    if (omp_get_num_teams () != 934
+	|| (unsigned) i >= 934U
+	|| a[i] != 0)
+      abort ();
+    ++a[i];
+  }
+  #pragma omp target map(a)
+  #pragma omp teams num_teams (foo () - 50 : foo ())
+  {
+    int i = omp_get_team_num ();
+    int j = omp_get_num_teams ();
+    if (j < 884
+	|| j > 934
+	|| (unsigned) i >= (unsigned) j
+	|| a[i] != 1)
+      abort ();
+    ++a[i];
+  }
+  #pragma omp target teams map(a) num_teams (foo () / 2)
+  {
+    int i = omp_get_team_num ();
+    if (omp_get_num_teams () != 467
+	|| (unsigned) i >= 467U
+	|| a[i] != 2)
+      abort ();
+    ++a[i];
+  }
+  #pragma omp target teams map(a) num_teams (foo () / 2 - 50 : foo () / 2)
+  {
+    int i = omp_get_team_num ();
+    int j = omp_get_num_teams ();
+    if (j < 417
+	|| j > 467
+	|| (unsigned) i >= (unsigned) j
+	|| a[i] != 3)
+      abort ();
+    ++a[i];
+  }
+  e = 4;
+  for (k = 0; k < 934; k++)
+    {
+      if (k >= 417 && k < 467 && a[k] == 3)
+	e = 3;
+      else if (k == 467)
+	e = 2;
+      else if (k >= 884 && a[k] == 1)
+	e = 1;
+      if (a[k] != e)
+	abort ();
+    }
+  return 0;
+}
diff --git a/libgomp/testsuite/libgomp.c-c++-common/thread-limit-1.c b/libgomp/testsuite/libgomp.c-c++-common/thread-limit-1.c
new file mode 100644
index 0000000..cac2202
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c-c++-common/thread-limit-1.c
@@ -0,0 +1,23 @@
+#include <omp.h>
+#include <stdlib.h>
+
+void
+foo ()
+{
+  {
+    #pragma omp target parallel nowait thread_limit (4) num_threads (1)
+    if (omp_get_thread_limit () > 4)
+      abort ();
+  }
+  #pragma omp taskwait
+}
+
+int
+main ()
+{
+  #pragma omp target thread_limit (6)
+  if (omp_get_thread_limit () > 6)
+    abort ();
+  foo ();
+  return 0;
+}
diff --git a/libgomp/testsuite/libgomp.c/loop-26.c b/libgomp/testsuite/libgomp.c/loop-26.c
new file mode 100644
index 0000000..5b30213
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c/loop-26.c
@@ -0,0 +1,196 @@
+/* { dg-do run } */
+
+extern void abort (void);
+
+signed char v[5][7][9][21][4][42][3];
+int a[84];
+int *volatile zero = &a[42];
+int *volatile two = &a[42 + 2];
+int *volatile three = &a[42 + 3];
+int *volatile five = &a[42 + 5];
+int *volatile seven = &a[42 + 7];
+int *volatile nine = &a[42 + 9];
+int *volatile eleven = &a[42 + 11];
+int *volatile minusone = &a[42 - 1];
+volatile int zeroi = 0, onei = 1, twoi = 2, threei = 3, fivei = 5;
+
+int
+main ()
+{
+  for (int i = 0; i < 5; i++)
+  for (int j = 0; j < 7; j++)
+  for (int k = 0; k < 9; k++)
+  for (int l = j; l < 5 + j; l++)
+  for (int m = 7; m < 11; m++)
+  for (int n = 0; n < l - 2; n++)
+  for (int o = 0; o < 3; o++)
+    v[i][j][k][l][m - 7][n][o] = 1;
+
+  int niters = 0;
+  #pragma omp parallel
+  #pragma omp for collapse(7) reduction(+:niters)
+  for (int i = 0; i < 5; i++)
+  for (int *j = &a[42]; j < &a[42 + 7]; j++)
+  for (int *k = &a[42]; k < &a[42 + 9]; k++)
+  for (int *l = j; l < 5 + j; l++)
+  for (int *m = &a[42 + 7]; m < &a[42 + 11]; m++)
+  for (int *n = &a[42]; n < l - 2; n++)
+  for (int *o = &a[42]; o < &a[42 + 3]; o++)
+    {
+      niters++;
+      if (i < 0 || i >= 5
+	  || j - &a[42] < 0 || j - &a[42] >= 7
+	  || k - &a[42] < 0 || k - &a[42] >= 9
+	  || l - &a[42] < 0 || l >= j + 5
+	  || m - &a[42] < 7 || m - &a[42] >= 11
+	  || n - &a[42] < 0 || n >= l - 2
+	  || o - &a[42] < 0 || o - &a[42] >= 3)
+	abort ();
+      if (v[i][j - &a[42]][k - &a[42]][l - &a[42]][m - &a[42 + 7]][n - &a[42]][o - &a[42]] != 1)
+	abort ();
+      v[i][j - &a[42]][k - &a[42]][l - &a[42]][m - &a[42 + 7]][n - &a[42]][o - &a[42]]++;
+    }
+
+  if (niters != 58860)
+    abort ();
+  int niters2 = 0;
+  #pragma omp parallel
+  #pragma omp for collapse(7) reduction(+:niters2)
+  for (int *i = zero; i < five; i += onei)
+  for (int *j = seven - onei; j >= zero; j -= onei)
+  for (int *k = nine - onei; k >= zero; k += -onei)
+  for (int *l = j + zeroi; l < fivei + j; l += onei)
+  for (int *m = eleven - onei; m >= seven; m -= onei)
+  for (int *n = l - threei; n >= zero; n -= onei)
+  for (int *o = zero; o < three; o += onei)
+    {
+      niters2++;
+      if (i - &a[42] < 0 || i - &a[42] >= 5
+	  || j - &a[42] < 0 || j - &a[42] >= 7
+	  || k - &a[42] < 0 || k - &a[42] >= 9
+	  || l < j || l >= j + 5
+	  || m - &a[42] < 7 || m - &a[42] >= 11
+	  || n - &a[42] < 0 || n >= l - 2
+	  || o - &a[42] < 0 || o - &a[42] >= 3)
+	abort ();
+      if (v[i - &a[42]][j - &a[42]][k - &a[42]][l - &a[42]][m - &a[42 + 7]][n - &a[42]][o - &a[42]] != 2)
+	abort ();
+      v[i - &a[42]][j - &a[42]][k - &a[42]][l - &a[42]][m - &a[42 + 7]][n - &a[42]][o - &a[42]]++;
+    }
+
+  if (niters2 != 58860)
+    abort ();
+
+  for (int i = 0; i < 5; i++)
+  for (int j = 0; j < 7; j++)
+  for (int k = 0; k < 9; k++)
+  for (int l = j; l < 5 + j; l++)
+  for (int m = 7; m < 11; m++)
+  for (int n = 0; n < l - 2; n++)
+  for (int o = 0; o < 3; o++)
+    if (v[i][j][k][l][m - 7][n][o] != 3)
+      abort ();
+
+  int niters3 = 0;
+  #pragma omp parallel
+  #pragma omp for collapse(5) reduction(+:niters3)
+  for (int *i = &a[42 + 4]; i >= &a[42 + 0]; i--)
+  for (int *j = &a[42 + 6]; j >= &a[42 + 0]; --j)
+  for (int *l = j + 4; l >= j; l--)
+  for (int *n = l - 3; n >= &a[42]; --n)
+  for (int *o = &a[42 + 2]; o >= &a[42 + 0]; o--)
+    {
+      niters3++;
+      if (i - &a[42] < 0 || i - &a[42] >= 5
+	  || j - &a[42] < 0 || j - &a[42] >= 7
+	  || l < j || l >= j + 5
+	  || n - &a[42] < 0 || n >= l - 2
+	  || o - &a[42] < 0 || o - &a[42] >= 3)
+	abort ();
+      if (v[i - &a[42]][j - &a[42]][0][l - &a[42]][0][n - &a[42]][o - &a[42]] != 3)
+	abort ();
+      v[i - &a[42]][j - &a[42]][0][l - &a[42]][0][n - &a[42]][o - &a[42]]++;
+    }
+
+  if (niters3 != 1635)
+    abort ();
+
+  int niters4 = 0;
+  #pragma omp parallel
+  #pragma omp for collapse(5) reduction(+:niters4)
+  for (int *i = zero; i < five; i += onei)
+  for (int *j = zero; j <= seven - onei; j += onei)
+  for (int *l = zeroi + j; l < j + fivei; l += onei)
+  for (int *n = zero; n <= l - threei; n += onei)
+  for (int o = zeroi; o < threei; o += onei)
+    {
+      niters4++;
+      if (i - &a[42] < 0 || i - &a[42] >= 5
+	  || j - &a[42] < 0 || j - &a[42] >= 7
+	  || l < j || l >= j + 5
+	  || n - &a[42] < 0 || n >= l - 2
+	  || o < 0 || o >= 3)
+	abort ();
+      if (v[i - &a[42]][j - &a[42]][0][l - &a[42]][0][n - &a[42]][o] != 4)
+	abort ();
+      v[i - &a[42]][j - &a[42]][0][l - &a[42]][0][n - &a[42]][o]++;
+    }
+
+  if (niters4 != 1635)
+    abort ();
+
+  for (int i = 0; i < 5; i++)
+  for (int j = 0; j < 7; j++)
+  for (int l = j; l < j + 5; l++)
+  for (int n = 0; n < l - 2; n++)
+  for (int o = 0; o < 3; o++)
+    if (v[i][j][0][l][0][n][o] != 5)
+      abort ();
+
+  int niters5 = 0;
+  #pragma omp parallel
+  #pragma omp for collapse(3) reduction(+:niters5)
+  for (int *j = &a[42 + 6]; j >= &a[42]; --j)
+  for (int *l = j + 0; l <= j + 4; l++)
+  for (int *n = l - 3; n > &a[42 - 1]; --n)
+    {
+      niters5++;
+      if (j - &a[42] < 0 || j - &a[42] >= 7
+	  || l < j || l >= j + 5
+	  || n < &a[42] || n >= l - 2)
+	abort ();
+      if (v[0][j - &a[42]][0][l - &a[42]][0][n - &a[42]][0] != 5)
+	abort ();
+      v[0][j - &a[42]][0][l - &a[42]][0][n - &a[42]][0]++;
+    }
+
+  if (niters5 != 109)
+    abort ();
+
+  int niters6 = 0;
+  #pragma omp parallel
+  #pragma omp for collapse(3) reduction(+:niters6)
+  for (int *j = seven - onei; j > minusone; j -= onei)
+  for (int *l = j + threei + onei; l >= j; l += -onei)
+  for (int *n = l - threei; n > minusone; n -= onei)
+    {
+      niters6++;
+      if (j - &a[42] < 0 || j - &a[42] >= 7
+	  || l < j || l >= j + 5
+	  || n < &a[42] || n >= l - 2)
+	abort ();
+      if (v[0][j - &a[42]][0][l - &a[42]][0][n - &a[42]][0] != 6)
+	abort ();
+      v[0][j - &a[42]][0][l - &a[42]][0][n - &a[42]][0]++;
+    }
+
+  if (niters6 != 109)
+    abort ();
+
+  for (int j = 0; j < 7; j++)
+  for (int l = j; l < j + 5; l++)
+  for (int n = 0; n < l - 2; n++)
+    if (v[0][j][0][l][0][n][0] != 7)
+      abort ();
+  return 0;
+}
diff --git a/libgomp/testsuite/libgomp.c/loop-27.c b/libgomp/testsuite/libgomp.c/loop-27.c
new file mode 100644
index 0000000..3353b87
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c/loop-27.c
@@ -0,0 +1,180 @@
+/* { dg-do run } */
+
+extern void abort (void);
+
+int x;
+short *i, *j;
+int ii, jj;
+short *volatile a;
+short *volatile b;
+volatile int c, e, f, g;
+short *volatile d;
+int k[11][20];
+short v[84];
+
+int
+main ()
+{
+  int niters;
+  for (ii = 1; ii <= 10; ii++)
+    for (jj = 1; jj <= ii + 5; jj++)
+      k[ii][jj] = 1;
+  a = &v[42 + 1]; b = &v[42 + 11]; c = 1; d = &v[42 + 1]; e = 6; f = 1;
+  niters = 0; i = &v[0]; j = &v[0]; x = -100;
+  #pragma omp parallel for collapse(2) lastprivate (i, j, x) reduction(+:niters)
+  for (i = &v[42 + 1]; i <= &v[42 + 10]; i++)
+    for (j = &v[42 + 1]; j <= i + 5; j++)
+      {
+	if (i < &v[42 + 1] || i > &v[42 + 10] || j < &v[42 + 1] || j > i + 5 || k[i - &v[42]][j - &v[42]] != 1)
+	  abort ();
+	k[i - &v[42]][j - &v[42]]++;
+	x = (i - &v[42]) * 1024 + ((j - &v[42]) & 1023);
+	niters++;
+      }
+  if (i != &v[42 + 11] || j != &v[42 + 16] || x != 10255 || niters != 105)
+    abort ();
+  niters = 0; i = &v[0]; j = &v[0]; x = -100;
+  #pragma omp parallel for collapse(2) lastprivate (i, j, x) reduction(+:niters)
+  for (i = a; i < b; i += c)
+    for (j = d; j < e + i; j += f)
+      {
+	if (i < &v[42 + 1] || i > &v[42 + 10] || j < &v[42 + 1] || j > i + 5 || k[i - &v[42]][j - &v[42]] != 2)
+	  abort ();
+	k[i - &v[42]][j - &v[42]]++;
+	x = (i - &v[42]) * 1024 + ((j - &v[42]) & 1023);
+	niters++;
+      }
+  if (i != &v[42 + 11] || j != &v[42 + 16] || x != 10255 || niters != 105)
+    abort ();
+  for (ii = 1; ii <= 10; ii++)
+    for (jj = 1; jj <= ii + 5; jj++)
+      if (k[ii][jj] == 3)
+	k[ii][jj] = 0;
+      else
+	abort ();
+  for (ii = 0; ii < 11; ii++)
+    for (jj = 0; jj < 20; jj++)
+      if (k[ii][jj] != 0)
+	abort ();
+  for (ii = 0; ii < 10; ii++)
+    for (jj = 0; jj < ii; jj++)
+      k[ii][jj] = 1;
+  a = &v[42]; b = &v[42 + 10]; c = 1; d = &v[42]; e = 0; f = 1;
+  niters = 0; i = &v[0]; j = &v[0]; x = -100;
+  #pragma omp parallel for collapse(2) lastprivate (i, j, x) reduction(+:niters)
+  for (i = &v[42]; i < &v[42 + 10]; i++)
+    for (j = &v[42]; j < i; j++)
+      {
+	if (i < &v[42] || i >= &v[42 + 10] || j < &v[42] || j >= i || k[i - &v[42]][j - &v[42]] != 1)
+	  abort ();
+	k[i - &v[42]][j - &v[42]]++;
+	x = (i - &v[42]) * 1024 + ((j - &v[42]) & 1023);
+	niters++;
+      }
+  if (i != &v[42 + 10] || j != &v[42 + 9] || x != 9224 || niters != 45)
+    abort ();
+  niters = 0; i = &v[0]; j = &v[0]; x = -100;
+  #pragma omp parallel for collapse(2) lastprivate (i, j, x) reduction(+:niters)
+  for (i = a; i < b; i += c)
+    for (j = d; j < i - e; j += f)
+      {
+	if (i < &v[42] || i >= &v[42 + 10] || j < &v[42] || j >= i || k[i - &v[42]][j - &v[42]] != 2)
+	  abort ();
+	k[i - &v[42]][j - &v[42]]++;
+	x = (i - &v[42]) * 1024 + ((j - &v[42]) & 1023);
+	niters++;
+      }
+  if (i != &v[42 + 10] || j != &v[42 + 9] || x != 9224 || niters != 45)
+    abort ();
+  for (ii = 0; ii < 10; ii++)
+    for (jj = 0; jj < ii; jj++)
+      if (k[ii][jj] == 3)
+	k[ii][jj] = 0;
+      else
+	abort ();
+  for (ii = 0; ii < 11; ii++)
+    for (jj = 0; jj < 20; jj++)
+      if (k[ii][jj] != 0)
+	abort ();
+  for (ii = 0; ii < 10; ii++)
+    for (jj = ii + 1; jj < ii + 4; jj++)
+      k[ii][jj] = 1;
+  a = &v[42]; b = &v[42 + 10]; c = 1; e = 1; f = 1; g = -4;
+  niters = 0; i = &v[0]; j = &v[0]; x = -100;
+  #pragma omp parallel for collapse(2) lastprivate (i, j, x) reduction(+:niters)
+  for (i = &v[42]; i < &v[42 + 10]; i++)
+    for (j = i + 1; j < i + 4; j++)
+      {
+	if (i < &v[42] || i >= &v[42 + 10] || j < i + 1 || j >= i + 4 || k[i - &v[42]][j - &v[42]] != 1)
+	  abort ();
+	k[i - &v[42]][j - &v[42]]++;
+	x = (i - &v[42]) * 1024 + ((j - &v[42]) & 1023);
+	niters++;
+      }
+  if (i != &v[42 + 10] || j != &v[42 + 13] || x != 9228 || niters != 30)
+    abort ();
+  niters = 0; i = &v[0]; j = &v[0]; x = -100;
+  #pragma omp parallel for collapse(2) lastprivate (i, j, x) reduction(+:niters)
+  for (i = a; i < b; i += c)
+    for (j = i + e; j < i - g; j += f)
+      {
+	if (i < &v[42] || i >= &v[42 + 10] || j < i + 1 || j >= i + 4 || k[i - &v[42]][j - &v[42]] != 2)
+	  abort ();
+	k[i - &v[42]][j - &v[42]]++;
+	x = (i - &v[42]) * 1024 + ((j - &v[42]) & 1023);
+	niters++;
+      }
+  if (i != &v[42 + 10] || j != &v[42 + 13] || x != 9228 || niters != 30)
+    abort ();
+  for (ii = 0; ii < 10; ii++)
+    for (jj = ii + 1; jj < ii + 4; jj++)
+      if (k[ii][jj] == 3)
+	k[ii][jj] = 0;
+      else
+	abort ();
+  for (ii = 0; ii < 11; ii++)
+    for (jj = 0; jj < 20; jj++)
+      if (k[ii][jj] != 0)
+	abort ();
+  for (ii = 1; ii < 10; ii += 2)
+    for (jj = 1; jj < ii + 1; jj++)
+      k[ii][jj] = 1;
+  a = &v[42 + 1]; b = &v[42 + 10]; c = 2; d = &v[42 + 1]; e = 1; f = 1;
+  niters = 0; i = &v[0]; j = &v[0]; x = -100;
+  #pragma omp parallel for collapse(2) lastprivate (i, j, x) reduction(+:niters)
+  for (i = &v[42 + 1]; i < &v[42 + 10]; i += 2)
+    for (j = &v[42 + 1]; j < i + 1; j++)
+      {
+	if (i < &v[42 + 1] || i >= &v[42 + 10] || j < &v[42 + 1] || j >= i + 1 || k[i - &v[42]][j - &v[42]] != 1)
+	  abort ();
+	k[i - &v[42]][j - &v[42]]++;
+	x = (i - &v[42]) * 1024 + ((j - &v[42]) & 1023);
+	niters++;
+      }
+  if (i != &v[42 + 11] || j != &v[42 + 10] || x != 9225 || niters != 25)
+    abort ();
+  niters = 0; i = &v[0]; j = &v[0]; x = -100;
+  #pragma omp parallel for collapse(2) lastprivate (i, j, x) reduction(+:niters)
+  for (i = a; i < b; i += c)
+    for (j = d; j < i + e; j += f)
+      {
+	if (i < &v[42 + 1] || i >= &v[42 + 10] || j < &v[42 + 1] || j >= i + 1 || k[i - &v[42]][j - &v[42]] != 2)
+	  abort ();
+	k[i - &v[42]][j - &v[42]]++;
+	x = (i - &v[42]) * 1024 + ((j - &v[42]) & 1023);
+	niters++;
+      }
+  if (i != &v[42 + 11] || j != &v[42 + 10] || x != 9225 || niters != 25)
+    abort ();
+  for (ii = 1; ii < 10; ii += 2)
+    for (jj = 1; jj < ii + 1; jj++)
+      if (k[ii][jj] == 3)
+	k[ii][jj] = 0;
+      else
+	abort ();
+  for (ii = 0; ii < 11; ii++)
+    for (jj = 0; jj < 20; jj++)
+      if (k[ii][jj] != 0)
+	abort ();
+  return 0;
+}
diff --git a/libgomp/testsuite/libgomp.c/target-3.c b/libgomp/testsuite/libgomp.c/target-3.c
index 7002cf2..ec818d2 100644
--- a/libgomp/testsuite/libgomp.c/target-3.c
+++ b/libgomp/testsuite/libgomp.c/target-3.c
@@ -11,7 +11,9 @@ main ()
     abort ();
   #pragma omp target if (0)
   #pragma omp teams
-  if (omp_get_level ())
-    abort ();
+  #pragma omp distribute dist_schedule(static,1)
+  for (int i = 0; i < 1; ++i)
+    if (omp_get_level ())
+      abort ();
   return 0;
 }
diff --git a/libgomp/testsuite/libgomp.c/target-5.c b/libgomp/testsuite/libgomp.c/target-5.c
index 21a69ea..20b1b4d 100644
--- a/libgomp/testsuite/libgomp.c/target-5.c
+++ b/libgomp/testsuite/libgomp.c/target-5.c
@@ -55,6 +55,7 @@ main ()
     abort ();
   #pragma omp target if (0)
   #pragma omp teams
+  #pragma omp parallel if(0)
   {
     omp_sched_t s_c;
     int c_c;
diff --git a/libgomp/testsuite/libgomp.c/target-6.c b/libgomp/testsuite/libgomp.c/target-6.c
index 8ffcb5b..1c4d8ca 100644
--- a/libgomp/testsuite/libgomp.c/target-6.c
+++ b/libgomp/testsuite/libgomp.c/target-6.c
@@ -47,11 +47,13 @@ main ()
 		{
 		  #pragma omp teams thread_limit (2)
 		    {
-		      if (omp_in_parallel ()
-			  || omp_get_level () != 0
-			  || omp_get_ancestor_thread_num (0) != 0
-			  || omp_get_ancestor_thread_num (1) != -1)
-			abort ();
+		      #pragma omp distribute dist_schedule(static,1)
+		      for (int i = 0; i < 1; ++i)
+			if (omp_in_parallel ()
+			    || omp_get_level () != 0
+			    || omp_get_ancestor_thread_num (0) != 0
+			    || omp_get_ancestor_thread_num (1) != -1)
+			  abort ();
 		      #pragma omp parallel num_threads (2)
 		      {
 			if (!omp_in_parallel ()
diff --git a/libgomp/testsuite/libgomp.c/target-teams-1.c b/libgomp/testsuite/libgomp.c/target-teams-1.c
index 8f591e0..adbabef 100644
--- a/libgomp/testsuite/libgomp.c/target-teams-1.c
+++ b/libgomp/testsuite/libgomp.c/target-teams-1.c
@@ -35,76 +35,115 @@ foo (int a, int b, long c, long d)
     abort ();
   #pragma omp target map(from: err)
   #pragma omp teams
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1;
+  {
+    err = omp_get_num_teams () < 1;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1;
+  }
   if (err)
     abort ();
   #pragma omp target teams map(from: err)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1;
+  {
+    err = omp_get_num_teams () < 1;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1;
+  }
   if (err)
     abort ();
   #pragma omp target map(from: err)
   #pragma omp teams num_teams (4)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_num_teams () > 4;
+  {
+    err = omp_get_num_teams () < 1 || omp_get_num_teams () > 4;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1;
+  }
   if (err)
     abort ();
   #pragma omp target teams num_teams (4) map(from: err)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_num_teams () > 4;
+  {
+    err = omp_get_num_teams () < 1 || omp_get_num_teams () > 4;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1;
+  }
   if (err)
     abort ();
   #pragma omp target map(from: err)
   #pragma omp teams thread_limit (7)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_thread_limit () > 7;
+  {
+    err = omp_get_num_teams () < 1;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > 7;
+  }
   if (err)
     abort ();
   #pragma omp target teams thread_limit (7) map(from: err)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_thread_limit () > 7;
+  {
+    err = omp_get_num_teams () < 1;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > 7;
+  }
   if (err)
     abort ();
   #pragma omp target map(from: err)
   #pragma omp teams num_teams (4) thread_limit (8)
   {
     {
-      err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	    || omp_get_num_teams () > 4 || omp_get_thread_limit () > 8;
+      err = omp_get_num_teams () < 1 || omp_get_num_teams () > 4;
     }
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > 8;
   }
   if (err)
     abort ();
   #pragma omp target teams num_teams (4) thread_limit (8) map(from: err)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_num_teams () > 4 || omp_get_thread_limit () > 8;
+  {
+    err = omp_get_num_teams () < 1 || omp_get_num_teams () > 4;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > 8;
+  }
   if (err)
     abort ();
   #pragma omp target map(from: err)
   #pragma omp teams num_teams (a) thread_limit (b)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_num_teams () > a || omp_get_thread_limit () > b;
+  {
+    err = omp_get_num_teams () < 1 || omp_get_num_teams () > a;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > b;
+  }
   if (err)
     abort ();
   #pragma omp target teams num_teams (a) thread_limit (b) map(from: err)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_num_teams () > a || omp_get_thread_limit () > b;
+  {
+    err = omp_get_num_teams () < 1 || omp_get_num_teams () > a;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > b;
+  }
   if (err)
     abort ();
   #pragma omp target map(from: err)
   #pragma omp teams num_teams (c + 1) thread_limit (d - 1)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_num_teams () > c + 1 || omp_get_thread_limit () > d - 1;
+  {
+    err = omp_get_num_teams () < 1 || omp_get_num_teams () > c + 1;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > d - 1;
+  }
   if (err)
     abort ();
   #pragma omp target teams num_teams (c + 1) thread_limit (d - 1) map(from: err)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_num_teams () > c + 1 || omp_get_thread_limit () > d - 1;
+  {
+    err = omp_get_num_teams () < 1 || omp_get_num_teams () > c + 1;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > d - 1;
+  }
   if (err)
     abort ();
   #pragma omp target map (always, to: c, d) map(from: err)
   #pragma omp teams num_teams (c + 1) thread_limit (d - 1)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_num_teams () > c + 1 || omp_get_thread_limit () > d - 1;
+  {
+    err = omp_get_num_teams () < 1 || omp_get_num_teams () > c + 1;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > d - 1;
+  }
   if (err)
     abort ();
   #pragma omp target data map (to: c, d)
@@ -116,8 +155,11 @@ foo (int a, int b, long c, long d)
        their device and original values match is unclear.  */
     #pragma omp target map (to: c, d) map(from: err)
     #pragma omp teams num_teams (c + 1) thread_limit (d - 1)
-    err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	  || omp_get_num_teams () > c + 1 || omp_get_thread_limit () > d - 1;
+    {
+      err = omp_get_num_teams () < 1 || omp_get_num_teams () > c + 1;
+      #pragma omp parallel if(0)
+      err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > d - 1;
+    }
     if (err)
       abort ();
   }
@@ -125,21 +167,30 @@ foo (int a, int b, long c, long d)
      target involved.  */
   #pragma omp target map(from: err)
   #pragma omp teams num_teams (baz () + 1) thread_limit (baz () - 1)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_num_teams () > baz () + 1 || omp_get_thread_limit () > baz () - 1;
+  {
+    err = omp_get_num_teams () < 1 || omp_get_num_teams () > baz () + 1;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > baz () - 1;
+  }
   if (err)
     abort ();
   #pragma omp target teams num_teams (baz () + 1) thread_limit (baz () - 1) map(from: err)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_num_teams () > baz () + 1 || omp_get_thread_limit () > baz () - 1;
+  {
+    err = omp_get_num_teams () < 1 || omp_get_num_teams () > baz () + 1;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > baz () - 1;
+  }
   if (err)
     abort ();
   /* This one can't be optimized, as v might have different value between
      host and target.  */
   #pragma omp target map(from: err)
   #pragma omp teams num_teams (v + 1) thread_limit (v - 1)
-  err = omp_get_num_teams () < 1 || omp_get_thread_limit () < 1
-	|| omp_get_num_teams () > v + 1 || omp_get_thread_limit () > v - 1;
+  {
+    err = omp_get_num_teams () < 1 || omp_get_num_teams () > v + 1;
+    #pragma omp parallel if(0)
+    err |= omp_get_thread_limit () < 1 || omp_get_thread_limit () > v - 1;
+  }
   if (err)
     abort ();
 }
diff --git a/libgomp/testsuite/libgomp.c/teams-1.c b/libgomp/testsuite/libgomp.c/teams-1.c
index 977e5fc..09b3570 100644
--- a/libgomp/testsuite/libgomp.c/teams-1.c
+++ b/libgomp/testsuite/libgomp.c/teams-1.c
@@ -6,15 +6,17 @@
 int
 main ()
 {
+  omp_set_dynamic (0);
+  omp_set_nested (1);
   #pragma omp teams thread_limit (2)
   {
+    #pragma omp distribute dist_schedule(static,1)
+    for (int i = 0; i < 1; ++i)
     if (omp_in_parallel ()
 	|| omp_get_level () != 0
 	|| omp_get_ancestor_thread_num (0) != 0
 	|| omp_get_ancestor_thread_num (1) != -1)
       abort ();
-    omp_set_dynamic (0);
-    omp_set_nested (1);
     #pragma omp parallel num_threads (2)
     {
       if (!omp_in_parallel ()
diff --git a/libgomp/testsuite/libgomp.c/teams-4.c b/libgomp/testsuite/libgomp.c/teams-4.c
new file mode 100644
index 0000000..6e065c5
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c/teams-4.c
@@ -0,0 +1,30 @@
+#include <omp.h>
+#include <stdlib.h>
+
+int
+main ()
+{
+  if (omp_get_num_teams () != 1 || omp_get_team_num () != 0)
+    abort ();
+  #pragma omp parallel num_threads (2)
+  if (omp_get_num_teams () != 1 || omp_get_team_num () != 0)
+    abort ();
+  #pragma omp teams num_teams (4)
+  {
+    int team = omp_get_team_num ();
+    if (omp_get_num_teams () != 4 || (unsigned) team >= 4U)
+      abort ();
+    #pragma omp parallel num_threads (3)
+    if (omp_get_num_teams () != 4 || omp_get_team_num () != team)
+      abort ();
+    #pragma omp parallel if (0)
+    #pragma omp target
+    #pragma omp teams num_teams (2)
+    if (omp_get_num_teams () != 2
+	|| (unsigned) omp_get_team_num () >= 2U)
+      abort ();
+    if (omp_get_num_teams () != 4 || (unsigned) team >= 4U)
+      abort ();
+  }
+  return 0;
+}
diff --git a/libgomp/testsuite/libgomp.c/teams-5.c b/libgomp/testsuite/libgomp.c/teams-5.c
new file mode 100644
index 0000000..ad42e05
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c/teams-5.c
@@ -0,0 +1,17 @@
+#include <omp.h>
+#include <stdlib.h>
+
+int
+main ()
+{
+  #pragma omp teams num_teams (2)
+  {
+    if (omp_get_num_teams () != 2)
+      abort ();
+    #pragma omp parallel if (0)
+    #pragma omp target
+    if (omp_get_num_teams () != 1 || omp_get_team_num () != 0)
+      abort ();
+  }
+  return 0;
+}
diff --git a/libgomp/testsuite/libgomp.c/thread-limit-2.c b/libgomp/testsuite/libgomp.c/thread-limit-2.c
index 2cff1fd..7997a70 100644
--- a/libgomp/testsuite/libgomp.c/thread-limit-2.c
+++ b/libgomp/testsuite/libgomp.c/thread-limit-2.c
@@ -20,25 +20,26 @@ main ()
   if (omp_get_num_threads () > 9)
     abort ();
   #pragma omp target if (0)
-  #pragma omp teams thread_limit (6)
   {
-    if (omp_get_thread_limit () > 6)
-      abort ();
-    if (omp_get_thread_limit () == 6)
+    omp_set_dynamic (0);
+    omp_set_nested (1);
+    #pragma omp teams thread_limit (6)
       {
-	omp_set_dynamic (0);
-	omp_set_nested (1);
 	#pragma omp parallel num_threads (3)
-	if (omp_get_num_threads () != 3)
+	if (omp_get_thread_limit () > 6
+	    || (omp_get_thread_limit () == 6 && omp_get_num_threads () != 3))
 	  abort ();
 	#pragma omp parallel num_threads (3)
-	if (omp_get_num_threads () != 3)
+	if (omp_get_thread_limit () > 6
+	    || (omp_get_thread_limit () == 6 && omp_get_num_threads () != 3))
 	  abort ();
 	#pragma omp parallel num_threads (8)
-	if (omp_get_num_threads () > 6)
+	if (omp_get_thread_limit () > 6
+	    || (omp_get_thread_limit () == 6 && omp_get_num_threads () > 6))
 	  abort ();
 	#pragma omp parallel num_threads (6)
-	if (omp_get_num_threads () != 6)
+	if (omp_get_thread_limit () > 6
+	    || (omp_get_thread_limit () == 6 && omp_get_num_threads () != 6))
 	  abort ();
 	int cnt = 0;
 	#pragma omp parallel num_threads (5)
diff --git a/libgomp/testsuite/libgomp.c/thread-limit-3.c b/libgomp/testsuite/libgomp.c/thread-limit-3.c
index af9bd78..01b3396 100644
--- a/libgomp/testsuite/libgomp.c/thread-limit-3.c
+++ b/libgomp/testsuite/libgomp.c/thread-limit-3.c
@@ -6,6 +6,7 @@ main ()
 {
   #pragma omp target if (0)
   #pragma omp teams thread_limit (1)
+  #pragma omp parallel if (0)
   if (omp_get_thread_limit () != 1)
     abort ();
   return 0;
diff --git a/libgomp/testsuite/libgomp.c/thread-limit-4.c b/libgomp/testsuite/libgomp.c/thread-limit-4.c
index 351423c..c6ccc7d 100644
--- a/libgomp/testsuite/libgomp.c/thread-limit-4.c
+++ b/libgomp/testsuite/libgomp.c/thread-limit-4.c
@@ -18,25 +18,25 @@ main ()
   #pragma omp parallel num_threads (16)
   if (omp_get_num_threads () > 9)
     abort ();
+  omp_set_dynamic (0);
+  omp_set_nested (1);
   #pragma omp teams thread_limit (6)
-  {
-    if (omp_get_thread_limit () > 6)
-      abort ();
-    if (omp_get_thread_limit () == 6)
-      {
-	omp_set_dynamic (0);
-	omp_set_nested (1);
+    {
 	#pragma omp parallel num_threads (3)
-	if (omp_get_num_threads () != 3)
+	if (omp_get_thread_limit () > 6
+	    || (omp_get_thread_limit () == 6 && omp_get_num_threads () != 3))
 	  abort ();
 	#pragma omp parallel num_threads (3)
-	if (omp_get_num_threads () != 3)
+	if (omp_get_thread_limit () > 6
+	    || (omp_get_thread_limit () == 6 && omp_get_num_threads () != 3))
 	  abort ();
 	#pragma omp parallel num_threads (8)
-	if (omp_get_num_threads () > 6)
+	if (omp_get_thread_limit () > 6
+	    || (omp_get_thread_limit () == 6 && omp_get_num_threads () > 6))
 	  abort ();
 	#pragma omp parallel num_threads (6)
-	if (omp_get_num_threads () != 6)
+	if (omp_get_thread_limit () > 6
+	    || (omp_get_thread_limit () == 6 && omp_get_num_threads () != 6))
 	  abort ();
 	int cnt = 0;
 	#pragma omp parallel num_threads (5)
@@ -52,7 +52,6 @@ main ()
 	  #pragma omp atomic
 	  --cnt;
 	}
-      }
-  }
+    }
   return 0;
 }
diff --git a/libgomp/testsuite/libgomp.c/thread-limit-5.c b/libgomp/testsuite/libgomp.c/thread-limit-5.c
index d3d22b1..d72988d 100644
--- a/libgomp/testsuite/libgomp.c/thread-limit-5.c
+++ b/libgomp/testsuite/libgomp.c/thread-limit-5.c
@@ -5,6 +5,7 @@ int
 main ()
 {
   #pragma omp teams thread_limit (1)
+  #pragma omp parallel if(0)
   if (omp_get_thread_limit () != 1)
     abort ();
   return 0;
diff --git a/libgomp/testsuite/libgomp.fortran/icv-3.f90 b/libgomp/testsuite/libgomp.fortran/icv-3.f90
index b2ccd77..4f81a63 100644
--- a/libgomp/testsuite/libgomp.fortran/icv-3.f90
+++ b/libgomp/testsuite/libgomp.fortran/icv-3.f90
@@ -13,6 +13,7 @@ implicit none (type, external)
   if (omp_get_teams_thread_limit () /= 15) &
     error stop 4
   !$omp teams
+   !$omp parallel if(.false.)
     if (omp_get_max_teams () /= 7 &
         .or. omp_get_teams_thread_limit () /= 15 &
         .or. omp_get_num_teams () < 1 &
@@ -22,8 +23,10 @@ implicit none (type, external)
         .or. omp_get_thread_limit () < 1 &
         .or. omp_get_thread_limit () > 15) &
       error stop 5
+   !$omp end parallel
   !$omp end teams
   !$omp teams num_teams(5) thread_limit (13)
+   !$omp parallel if(.false.)
     if (omp_get_max_teams () /= 7 &
         .or. omp_get_teams_thread_limit () /= 15 &
         .or. omp_get_num_teams () /= 5 &
@@ -32,8 +35,10 @@ implicit none (type, external)
         .or. omp_get_thread_limit () < 1 &
         .or. omp_get_thread_limit () > 13) &
       error stop 6
+   !$omp end parallel
   !$omp end teams
   !$omp teams num_teams(8) thread_limit (16)
+   !$omp parallel if(.false.)
     if (omp_get_max_teams () /= 7 &
         .or. omp_get_teams_thread_limit () /= 15 &
         .or. omp_get_num_teams () /= 8 &
@@ -42,6 +47,7 @@ implicit none (type, external)
         .or. omp_get_thread_limit () < 1 &
         .or. omp_get_thread_limit () > 16) &
       error stop 7
+   !$omp end parallel
   !$omp end teams
 contains
   logical function env_exists (name)
diff --git a/libgomp/testsuite/libgomp.fortran/icv-4.f90 b/libgomp/testsuite/libgomp.fortran/icv-4.f90
index f76c96d..2ae5d76 100644
--- a/libgomp/testsuite/libgomp.fortran/icv-4.f90
+++ b/libgomp/testsuite/libgomp.fortran/icv-4.f90
@@ -16,6 +16,7 @@ implicit none (type, external)
     call omp_set_teams_thread_limit (12)
   end if
   !$omp teams
+   !$omp parallel if(.false.)
     if (omp_get_max_teams () /= 6 &
         .or. omp_get_teams_thread_limit () /= 12 &
         .or. omp_get_num_teams () < 1 &
@@ -25,6 +26,7 @@ implicit none (type, external)
         .or. omp_get_thread_limit () < 1 &
         .or. omp_get_thread_limit () > 12) &
       error stop 3
+   !$omp end parallel
   !$omp end teams
 contains
   logical function env_is_set (name, val)
diff --git a/libgomp/testsuite/libgomp.fortran/teams-1.f90 b/libgomp/testsuite/libgomp.fortran/teams-1.f90
new file mode 100644
index 0000000..9969fe4
--- /dev/null
+++ b/libgomp/testsuite/libgomp.fortran/teams-1.f90
@@ -0,0 +1,22 @@
+program main
+  use omp_lib
+  implicit none (type, external)
+  integer :: i
+
+  !$omp teams num_teams (5)
+    if (omp_get_num_teams () /= 5) stop 1
+    !$omp distribute dist_schedule(static,1)
+    do i = 0, 4
+      if (omp_get_team_num () /= i) stop 2
+    end do
+  !$omp end teams
+
+  !$omp teams num_teams (7 : 9)
+    if (omp_get_num_teams () < 7 .or. omp_get_num_teams () > 9) &
+      stop 3
+    !$omp distribute dist_schedule(static,1)
+    do i = 0, omp_get_num_teams () - 1
+      if (omp_get_team_num () /= i) stop 4
+    end do
+  !$omp end teams
+end program main
diff --git a/libgomp/testsuite/libgomp.fortran/teams1.f90 b/libgomp/testsuite/libgomp.fortran/teams1.f90
index 0077a70..886765a 100644
--- a/libgomp/testsuite/libgomp.fortran/teams1.f90
+++ b/libgomp/testsuite/libgomp.fortran/teams1.f90
@@ -2,13 +2,17 @@
 
 program teams1
   use omp_lib
+  integer :: i
 !$omp teams thread_limit (2)
-  if (omp_in_parallel ()) stop 1
-  if (omp_get_level () .ne. 0) stop 2
-  if (omp_get_ancestor_thread_num (0) .ne. 0) stop 3
-  if (omp_get_ancestor_thread_num (1) .ne. -1) stop 4
-  call omp_set_dynamic (.false.)
-  call omp_set_nested (.true.)
+  !$omp distribute dist_schedule(static,1)
+  do i = 1, 1
+    if (omp_in_parallel ()) stop 1
+    if (omp_get_level () .ne. 0) stop 2
+    if (omp_get_ancestor_thread_num (0) .ne. 0) stop 3
+    if (omp_get_ancestor_thread_num (1) .ne. -1) stop 4
+    call omp_set_dynamic (.false.)
+    call omp_set_nested (.true.)
+  end do
 !$omp parallel num_threads (2)
   if (.not. omp_in_parallel ()) stop 5
   if (omp_get_level () .ne. 1) stop 6
diff --git a/libgomp/testsuite/libgomp.fortran/thread-limit-1.f90 b/libgomp/testsuite/libgomp.fortran/thread-limit-1.f90
new file mode 100644
index 0000000..bca69fb
--- /dev/null
+++ b/libgomp/testsuite/libgomp.fortran/thread-limit-1.f90
@@ -0,0 +1,41 @@
+! { dg-additional-options "-fdump-tree-original" }
+
+! { dg-final { scan-tree-dump-times "#pragma omp teams thread_limit\\(9\\)" 1 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp target thread_limit\\(9\\)" 1 "original" } }
+
+! { dg-final { scan-tree-dump-times "#pragma omp target nowait thread_limit\\(4\\)" 1 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma omp parallel num_threads\\(1\\)" 1 "original" } }
+
+! { dg-final { scan-tree-dump-times "#pragma omp target thread_limit\\(6\\)" 1 "original" } }
+
+
+module m
+  use omp_lib
+  implicit none
+contains
+
+subroutine uncalled()
+    !$omp target teams thread_limit (9)
+    !$omp end target teams
+end
+
+subroutine foo ()
+  block
+    !$omp target parallel nowait thread_limit (4) num_threads (1)
+    if (omp_get_thread_limit () > 4) &
+      stop 1
+    !$omp end target parallel
+  end block
+  !$omp taskwait
+end
+end module
+
+program main
+  use m
+  implicit none
+  !$omp target thread_limit (6)
+    if (omp_get_thread_limit () > 6) &
+      stop 2
+  !$omp end target
+  call foo ()
+end
diff --git a/libgomp/testsuite/libgomp.graphite/force-parallel-4.c b/libgomp/testsuite/libgomp.graphite/force-parallel-4.c
index 6caa934..587c0a1 100644
--- a/libgomp/testsuite/libgomp.graphite/force-parallel-4.c
+++ b/libgomp/testsuite/libgomp.graphite/force-parallel-4.c
@@ -1,5 +1,5 @@
 /* Autopar with IF conditions.  */
-/* { dg-additional-options "-fdisable-tree-thread1 -fdisable-tree-vrp-thread1" } */
+/* { dg-additional-options "-fno-thread-jumps" } */
 
 void abort();
 
diff --git a/libgomp/testsuite/libgomp.graphite/force-parallel-8.c b/libgomp/testsuite/libgomp.graphite/force-parallel-8.c
index 6868dd2..ee5a67a 100644
--- a/libgomp/testsuite/libgomp.graphite/force-parallel-8.c
+++ b/libgomp/testsuite/libgomp.graphite/force-parallel-8.c
@@ -1,4 +1,4 @@
-/* { dg-additional-options "-fdisable-tree-thread1 -fdisable-tree-vrp-thread1 --param max-stores-to-sink=0" } */
+/* { dg-additional-options "-fno-thread-jumps --param max-stores-to-sink=0" } */
 
 #define N 1500
 
diff --git a/libgomp/testsuite/libgomp.oacc-c-c++-common/loop-gwv-2.c b/libgomp/testsuite/libgomp.oacc-c-c++-common/loop-gwv-2.c
index cb3878b..e73ed60 100644
--- a/libgomp/testsuite/libgomp.oacc-c-c++-common/loop-gwv-2.c
+++ b/libgomp/testsuite/libgomp.oacc-c-c++-common/loop-gwv-2.c
@@ -6,7 +6,6 @@
 
 #include <stdio.h>
 #include <openacc.h>
-#include <alloca.h>
 #include <string.h>
 #include <gomp-constants.h>
 #include <stdlib.h>
@@ -78,9 +77,9 @@ int main ()
     vectorsize = __builtin_goacc_parlevel_size (GOMP_DIM_VECTOR);
   }
 
-  gangdist = (int *) alloca (gangsize * sizeof (int));
-  workerdist = (int *) alloca (workersize * sizeof (int));
-  vectordist = (int *) alloca (vectorsize * sizeof (int));
+  gangdist = (int *) __builtin_alloca (gangsize * sizeof (int));
+  workerdist = (int *) __builtin_alloca (workersize * sizeof (int));
+  vectordist = (int *) __builtin_alloca (vectorsize * sizeof (int));
   memset (gangdist, 0, gangsize * sizeof (int));
   memset (workerdist, 0, workersize * sizeof (int));
   memset (vectordist, 0, vectorsize * sizeof (int));
diff --git a/libiberty/ChangeLog b/libiberty/ChangeLog
index 7d33cb9..0bce6b2 100644
--- a/libiberty/ChangeLog
+++ b/libiberty/ChangeLog
@@ -1,3 +1,9 @@
+2021-10-22  Eric Gallager  <egallager@gcc.gnu.org>
+
+	PR other/102663
+	* Makefile.in: Allow dvi-formatted documentation
+	to be installed.
+
 2021-10-17  Luís Ferreira  <contact@lsferreira.net>
 
 	PR d/102618
diff --git a/libiberty/Makefile.in b/libiberty/Makefile.in
index 4f1213b..884cc6c 100644
--- a/libiberty/Makefile.in
+++ b/libiberty/Makefile.in
@@ -29,6 +29,7 @@ bindir = @bindir@
 libdir = @libdir@
 includedir = @includedir@
 target_header_dir = @target_header_dir@
+dvidir = @dvidir@
 objext = @OBJEXT@
 
 SHELL = @SHELL@
@@ -276,7 +277,24 @@ $(TESTLIB): $(REQUIRED_OFILES) $(CONFIGURED_OFILES)
 info: libiberty.info info-subdir
 install-info: install-info-subdir
 clean-info: clean-info-subdir
-dvi: libiberty.dvi dvi-subdir
+
+LIBIBERTY_DVIFILES = libiberty.dvi
+
+dvi: $(LIBIBERTY_DVIFILES) dvi-subdir
+
+.PHONY: install-dvi
+
+dvi__strip_dir = `echo $$p | sed -e 's|^.*/||'`;
+
+install-dvi: $(LIBIBERTY_DVIFILES)
+	@$(NORMAL_INSTALL)
+	test -z "$(dvidir)" || $(mkinstalldirs) "$(DESTDIR)$(dvidir)"
+	@list='$(LIBIBERTY_DVIFILES)'; for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  f=$(dvi__strip_dir) \
+	  echo " $(INSTALL_DATA) '$$d$$p' '$(DESTDIR)$(dvidir)/$$f'"; \
+	  $(INSTALL_DATA) "$$d$$p" "$(DESTDIR)$(dvidir)/$$f"; \
+	done
 
 LIBIBERTY_PDFFILES = libiberty.pdf
 
diff --git a/libphobos/ChangeLog b/libphobos/ChangeLog
index 0cc0b3d..0cae880 100644
--- a/libphobos/ChangeLog
+++ b/libphobos/ChangeLog
@@ -1,3 +1,9 @@
+2021-10-31  Iain Buclaw  <ibuclaw@gdcproject.org>
+
+	PR d/102837
+	* libdruntime/gcc/deh.d (ExceptionHeader.free): Use memset to reset
+	contents of internal EH storage.
+
 2021-09-30  Iain Buclaw  <ibuclaw@gdcproject.org>
 
 	* libdruntime/gcc/deh.d (ExceptionHeader.getClassInfo): Move to...
diff --git a/libphobos/libdruntime/gcc/deh.d b/libphobos/libdruntime/gcc/deh.d
index ba57fed..bbc351c 100644
--- a/libphobos/libdruntime/gcc/deh.d
+++ b/libphobos/libdruntime/gcc/deh.d
@@ -207,7 +207,7 @@ struct ExceptionHeader
      */
     static void free(ExceptionHeader* eh) @nogc
     {
-        *eh = ExceptionHeader.init;
+        __builtin_memset(eh, 0, ExceptionHeader.sizeof);
         if (eh != &ehstorage)
             __builtin_free(eh);
     }
diff --git a/libsanitizer/ChangeLog b/libsanitizer/ChangeLog
index 2b83bef..495245d 100644
--- a/libsanitizer/ChangeLog
+++ b/libsanitizer/ChangeLog
@@ -1,3 +1,11 @@
+2021-11-13  H.J. Lu  <hjl.tools@gmail.com>
+
+	* LOCAL_PATCHES: Update to the corresponding revision.
+
+2021-11-04  Martin Liska  <mliska@suse.cz>
+
+	* LOCAL_PATCHES: Update git revision.
+
 2021-10-08  H.J. Lu  <hjl.tools@gmail.com>
 
 	PR sanitizer/102632
diff --git a/libsanitizer/LOCAL_PATCHES b/libsanitizer/LOCAL_PATCHES
index 2f1b4f8..bcf13f5 100644
--- a/libsanitizer/LOCAL_PATCHES
+++ b/libsanitizer/LOCAL_PATCHES
@@ -1 +1 @@
-bb8adf080ef03288bccd9f1316ae2a1122ebcd85
+55b43a22ab96da3efc403fb933c0977b672b7930
diff --git a/libsanitizer/MERGE b/libsanitizer/MERGE
index 5ea083a..01913de 100644
--- a/libsanitizer/MERGE
+++ b/libsanitizer/MERGE
@@ -1,4 +1,4 @@
-fdf4c035225de52f596899931b1f6100e5e3e928
+82bc6a094e85014f1891ef9407496f44af8fe442
 
 The first line of this file holds the git revision number of the
 last merge done from the master library sources.
diff --git a/libsanitizer/asan/asan_allocator.cpp b/libsanitizer/asan/asan_allocator.cpp
index 268feac..3fa3674 100644
--- a/libsanitizer/asan/asan_allocator.cpp
+++ b/libsanitizer/asan/asan_allocator.cpp
@@ -102,19 +102,18 @@ class ChunkHeader {
 
  public:
   uptr UsedSize() const {
-    uptr R = user_requested_size_lo;
-    if (sizeof(uptr) > sizeof(user_requested_size_lo))
-      R += (uptr)user_requested_size_hi << (8 * sizeof(user_requested_size_lo));
-    return R;
+    static_assert(sizeof(user_requested_size_lo) == 4,
+                  "Expression below requires this");
+    return FIRST_32_SECOND_64(0, ((uptr)user_requested_size_hi << 32)) +
+           user_requested_size_lo;
   }
 
   void SetUsedSize(uptr size) {
     user_requested_size_lo = size;
-    if (sizeof(uptr) > sizeof(user_requested_size_lo)) {
-      size >>= (8 * sizeof(user_requested_size_lo));
-      user_requested_size_hi = size;
-      CHECK_EQ(user_requested_size_hi, size);
-    }
+    static_assert(sizeof(user_requested_size_lo) == 4,
+                  "Expression below requires this");
+    user_requested_size_hi = FIRST_32_SECOND_64(0, size >> 32);
+    CHECK_EQ(UsedSize(), size);
   }
 
   void SetAllocContext(u32 tid, u32 stack) {
@@ -522,7 +521,7 @@ struct Allocator {
         size > max_user_defined_malloc_size) {
       if (AllocatorMayReturnNull()) {
         Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
-               (void*)size);
+               size);
         return nullptr;
       }
       uptr malloc_limit =
diff --git a/libsanitizer/asan/asan_descriptions.cpp b/libsanitizer/asan/asan_descriptions.cpp
index 2ba8a02..d7d9616 100644
--- a/libsanitizer/asan/asan_descriptions.cpp
+++ b/libsanitizer/asan/asan_descriptions.cpp
@@ -251,7 +251,7 @@ static void PrintAccessAndVarIntersection(const StackVarDescr &var, uptr addr,
   }
   str.append("'");
   if (var.line > 0) {
-    str.append(" (line %d)", var.line);
+    str.append(" (line %zd)", var.line);
   }
   if (pos_descr) {
     Decorator d;
@@ -318,7 +318,8 @@ bool DescribeAddressIfGlobal(uptr addr, uptr access_size,
 }
 
 void ShadowAddressDescription::Print() const {
-  Printf("Address %p is located in the %s area.\n", addr, ShadowNames[kind]);
+  Printf("Address %p is located in the %s area.\n", (void *)addr,
+         ShadowNames[kind]);
 }
 
 void GlobalAddressDescription::Print(const char *bug_type) const {
@@ -356,7 +357,7 @@ bool GlobalAddressDescription::PointsInsideTheSameVariable(
 void StackAddressDescription::Print() const {
   Decorator d;
   Printf("%s", d.Location());
-  Printf("Address %p is located in stack of thread %s", addr,
+  Printf("Address %p is located in stack of thread %s", (void *)addr,
          AsanThreadIdAndName(tid).c_str());
 
   if (!frame_descr) {
@@ -469,7 +470,7 @@ AddressDescription::AddressDescription(uptr addr, uptr access_size,
 
 void WildAddressDescription::Print() const {
   Printf("Address %p is a wild pointer inside of access range of size %p.\n",
-         addr, access_size);
+         (void *)addr, (void *)access_size);
 }
 
 void PrintAddressDescription(uptr addr, uptr access_size,
diff --git a/libsanitizer/asan/asan_errors.cpp b/libsanitizer/asan/asan_errors.cpp
index 45166c0..7cd9fe9 100644
--- a/libsanitizer/asan/asan_errors.cpp
+++ b/libsanitizer/asan/asan_errors.cpp
@@ -46,10 +46,9 @@ void ErrorDeadlySignal::Print() {
 void ErrorDoubleFree::Print() {
   Decorator d;
   Printf("%s", d.Error());
-  Report(
-      "ERROR: AddressSanitizer: attempting %s on %p in thread %s:\n",
-      scariness.GetDescription(), addr_description.addr,
-      AsanThreadIdAndName(tid).c_str());
+  Report("ERROR: AddressSanitizer: attempting %s on %p in thread %s:\n",
+         scariness.GetDescription(), (void *)addr_description.addr,
+         AsanThreadIdAndName(tid).c_str());
   Printf("%s", d.Default());
   scariness.Print();
   GET_STACK_TRACE_FATAL(second_free_stack->trace[0],
@@ -62,10 +61,9 @@ void ErrorDoubleFree::Print() {
 void ErrorNewDeleteTypeMismatch::Print() {
   Decorator d;
   Printf("%s", d.Error());
-  Report(
-      "ERROR: AddressSanitizer: %s on %p in thread %s:\n",
-      scariness.GetDescription(), addr_description.addr,
-      AsanThreadIdAndName(tid).c_str());
+  Report("ERROR: AddressSanitizer: %s on %p in thread %s:\n",
+         scariness.GetDescription(), (void *)addr_description.addr,
+         AsanThreadIdAndName(tid).c_str());
   Printf("%s  object passed to delete has wrong type:\n", d.Default());
   if (delete_size != 0) {
     Printf(
@@ -106,7 +104,7 @@ void ErrorFreeNotMalloced::Print() {
   Report(
       "ERROR: AddressSanitizer: attempting free on address "
       "which was not malloc()-ed: %p in thread %s\n",
-      addr_description.Address(), AsanThreadIdAndName(tid).c_str());
+      (void *)addr_description.Address(), AsanThreadIdAndName(tid).c_str());
   Printf("%s", d.Default());
   CHECK_GT(free_stack->size, 0);
   scariness.Print();
@@ -126,7 +124,7 @@ void ErrorAllocTypeMismatch::Print() {
   Printf("%s", d.Error());
   Report("ERROR: AddressSanitizer: %s (%s vs %s) on %p\n",
          scariness.GetDescription(), alloc_names[alloc_type],
-         dealloc_names[dealloc_type], addr_description.Address());
+         dealloc_names[dealloc_type], (void *)addr_description.Address());
   Printf("%s", d.Default());
   CHECK_GT(dealloc_stack->size, 0);
   scariness.Print();
@@ -145,7 +143,7 @@ void ErrorMallocUsableSizeNotOwned::Print() {
   Report(
       "ERROR: AddressSanitizer: attempting to call malloc_usable_size() for "
       "pointer which is not owned: %p\n",
-      addr_description.Address());
+      (void *)addr_description.Address());
   Printf("%s", d.Default());
   stack->Print();
   addr_description.Print();
@@ -158,7 +156,7 @@ void ErrorSanitizerGetAllocatedSizeNotOwned::Print() {
   Report(
       "ERROR: AddressSanitizer: attempting to call "
       "__sanitizer_get_allocated_size() for pointer which is not owned: %p\n",
-      addr_description.Address());
+      (void *)addr_description.Address());
   Printf("%s", d.Default());
   stack->Print();
   addr_description.Print();
@@ -298,9 +296,10 @@ void ErrorStringFunctionMemoryRangesOverlap::Print() {
   Report(
       "ERROR: AddressSanitizer: %s: memory ranges [%p,%p) and [%p, %p) "
       "overlap\n",
-      bug_type, addr1_description.Address(),
-      addr1_description.Address() + length1, addr2_description.Address(),
-      addr2_description.Address() + length2);
+      bug_type, (void *)addr1_description.Address(),
+      (void *)(addr1_description.Address() + length1),
+      (void *)addr2_description.Address(),
+      (void *)(addr2_description.Address() + length2));
   Printf("%s", d.Default());
   scariness.Print();
   stack->Print();
@@ -329,10 +328,10 @@ void ErrorBadParamsToAnnotateContiguousContainer::Print() {
       "      end     : %p\n"
       "      old_mid : %p\n"
       "      new_mid : %p\n",
-      beg, end, old_mid, new_mid);
+      (void *)beg, (void *)end, (void *)old_mid, (void *)new_mid);
   uptr granularity = SHADOW_GRANULARITY;
   if (!IsAligned(beg, granularity))
-    Report("ERROR: beg is not aligned by %d\n", granularity);
+    Report("ERROR: beg is not aligned by %zu\n", granularity);
   stack->Print();
   ReportErrorSummary(scariness.GetDescription(), stack);
 }
@@ -341,7 +340,7 @@ void ErrorODRViolation::Print() {
   Decorator d;
   Printf("%s", d.Error());
   Report("ERROR: AddressSanitizer: %s (%p):\n", scariness.GetDescription(),
-         global1.beg);
+         (void *)global1.beg);
   Printf("%s", d.Default());
   InternalScopedString g1_loc;
   InternalScopedString g2_loc;
@@ -371,7 +370,8 @@ void ErrorInvalidPointerPair::Print() {
   Decorator d;
   Printf("%s", d.Error());
   Report("ERROR: AddressSanitizer: %s: %p %p\n", scariness.GetDescription(),
-         addr1_description.Address(), addr2_description.Address());
+         (void *)addr1_description.Address(),
+         (void *)addr2_description.Address());
   Printf("%s", d.Default());
   GET_STACK_TRACE_FATAL(pc, bp);
   stack.Print();
@@ -538,7 +538,8 @@ static void PrintLegend(InternalScopedString *str) {
 static void PrintShadowBytes(InternalScopedString *str, const char *before,
                              u8 *bytes, u8 *guilty, uptr n) {
   Decorator d;
-  if (before) str->append("%s%p:", before, bytes);
+  if (before)
+    str->append("%s%p:", before, (void *)bytes);
   for (uptr i = 0; i < n; i++) {
     u8 *p = bytes + i;
     const char *before =
@@ -575,7 +576,7 @@ void ErrorGeneric::Print() {
   Printf("%s", d.Error());
   uptr addr = addr_description.Address();
   Report("ERROR: AddressSanitizer: %s on address %p at pc %p bp %p sp %p\n",
-         bug_descr, (void *)addr, pc, bp, sp);
+         bug_descr, (void *)addr, (void *)pc, (void *)bp, (void *)sp);
   Printf("%s", d.Default());
 
   Printf("%s%s of size %zu at %p thread %s%s\n", d.Access(),
diff --git a/libsanitizer/asan/asan_fake_stack.cpp b/libsanitizer/asan/asan_fake_stack.cpp
index bf5c342..07681c1 100644
--- a/libsanitizer/asan/asan_fake_stack.cpp
+++ b/libsanitizer/asan/asan_fake_stack.cpp
@@ -54,10 +54,11 @@ FakeStack *FakeStack::Create(uptr stack_size_log) {
                              : MmapOrDie(size, "FakeStack"));
   res->stack_size_log_ = stack_size_log;
   u8 *p = reinterpret_cast<u8 *>(res);
-  VReport(1, "T%d: FakeStack created: %p -- %p stack_size_log: %zd; "
+  VReport(1,
+          "T%d: FakeStack created: %p -- %p stack_size_log: %zd; "
           "mmapped %zdK, noreserve=%d \n",
-          GetCurrentTidOrInvalid(), p,
-          p + FakeStack::RequiredSize(stack_size_log), stack_size_log,
+          GetCurrentTidOrInvalid(), (void *)p,
+          (void *)(p + FakeStack::RequiredSize(stack_size_log)), stack_size_log,
           size >> 10, flags()->uar_noreserve);
   return res;
 }
diff --git a/libsanitizer/asan/asan_globals.cpp b/libsanitizer/asan/asan_globals.cpp
index 763d3c6..9400487 100644
--- a/libsanitizer/asan/asan_globals.cpp
+++ b/libsanitizer/asan/asan_globals.cpp
@@ -85,12 +85,12 @@ static void ReportGlobal(const Global &g, const char *prefix) {
   Report(
       "%s Global[%p]: beg=%p size=%zu/%zu name=%s module=%s dyn_init=%zu "
       "odr_indicator=%p\n",
-      prefix, &g, (void *)g.beg, g.size, g.size_with_redzone, g.name,
+      prefix, (void *)&g, (void *)g.beg, g.size, g.size_with_redzone, g.name,
       g.module_name, g.has_dynamic_init, (void *)g.odr_indicator);
   if (g.location) {
-    Report("  location (%p): name=%s[%p], %d %d\n", g.location,
-           g.location->filename, g.location->filename, g.location->line_no,
-           g.location->column_no);
+    Report("  location (%p): name=%s[%p], %d %d\n", (void *)g.location,
+           g.location->filename, (void *)g.location->filename,
+           g.location->line_no, g.location->column_no);
   }
 }
 
@@ -350,7 +350,8 @@ void __asan_register_globals(__asan_global *globals, uptr n) {
   global_registration_site_vector->push_back(site);
   if (flags()->report_globals >= 2) {
     PRINT_CURRENT_STACK();
-    Printf("=== ID %d; %p %p\n", stack_id, &globals[0], &globals[n - 1]);
+    Printf("=== ID %d; %p %p\n", stack_id, (void *)&globals[0],
+           (void *)&globals[n - 1]);
   }
   for (uptr i = 0; i < n; i++) {
     if (SANITIZER_WINDOWS && globals[i].beg == 0) {
diff --git a/libsanitizer/asan/asan_linux.cpp b/libsanitizer/asan/asan_linux.cpp
index 4bcbe5d..ad3693d 100644
--- a/libsanitizer/asan/asan_linux.cpp
+++ b/libsanitizer/asan/asan_linux.cpp
@@ -128,8 +128,8 @@ void AsanCheckIncompatibleRT() {}
 #else
 static int FindFirstDSOCallback(struct dl_phdr_info *info, size_t size,
                                 void *data) {
-  VReport(2, "info->dlpi_name = %s\tinfo->dlpi_addr = %p\n",
-          info->dlpi_name, info->dlpi_addr);
+  VReport(2, "info->dlpi_name = %s\tinfo->dlpi_addr = %p\n", info->dlpi_name,
+          (void *)info->dlpi_addr);
 
   // Continue until the first dynamic library is found
   if (!info->dlpi_name || info->dlpi_name[0] == 0)
diff --git a/libsanitizer/asan/asan_malloc_linux.cpp b/libsanitizer/asan/asan_malloc_linux.cpp
index c6bec85..bab80b9 100644
--- a/libsanitizer/asan/asan_malloc_linux.cpp
+++ b/libsanitizer/asan/asan_malloc_linux.cpp
@@ -21,129 +21,66 @@
 #  include "asan_interceptors.h"
 #  include "asan_internal.h"
 #  include "asan_stack.h"
+#  include "lsan/lsan_common.h"
 #  include "sanitizer_common/sanitizer_allocator_checks.h"
+#  include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #  include "sanitizer_common/sanitizer_errno.h"
 #  include "sanitizer_common/sanitizer_tls_get_addr.h"
 
 // ---------------------- Replacement functions ---------------- {{{1
 using namespace __asan;
 
-static uptr allocated_for_dlsym;
-static uptr last_dlsym_alloc_size_in_words;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static inline bool IsInDlsymAllocPool(const void *ptr) {
-  uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  return off < allocated_for_dlsym * sizeof(alloc_memory_for_dlsym[0]);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
-  uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
-  void *mem = (void*)&alloc_memory_for_dlsym[allocated_for_dlsym];
-  last_dlsym_alloc_size_in_words = size_in_words;
-  allocated_for_dlsym += size_in_words;
-  CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
-  return mem;
-}
-
-static void DeallocateFromLocalPool(const void *ptr) {
-  // Hack: since glibc 2.27 dlsym no longer uses stack-allocated memory to store
-  // error messages and instead uses malloc followed by free. To avoid pool
-  // exhaustion due to long object filenames, handle that special case here.
-  uptr prev_offset = allocated_for_dlsym - last_dlsym_alloc_size_in_words;
-  void *prev_mem = (void*)&alloc_memory_for_dlsym[prev_offset];
-  if (prev_mem == ptr) {
-    REAL(memset)(prev_mem, 0, last_dlsym_alloc_size_in_words * kWordSize);
-    allocated_for_dlsym = prev_offset;
-    last_dlsym_alloc_size_in_words = 0;
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+  static bool UseImpl() { return asan_init_is_running; }
+  static void OnAllocate(const void *ptr, uptr size) {
+#  if CAN_SANITIZE_LEAKS
+    // Suppress leaks from dlerror(). Previously dlsym hack on global array was
+    // used by leak sanitizer as a root region.
+    __lsan_register_root_region(ptr, size);
+#  endif
   }
-}
-
-static int PosixMemalignFromLocalPool(void **memptr, uptr alignment,
-                                      uptr size_in_bytes) {
-  if (UNLIKELY(!CheckPosixMemalignAlignment(alignment)))
-    return errno_EINVAL;
-
-  CHECK(alignment >= kWordSize);
-
-  uptr addr = (uptr)&alloc_memory_for_dlsym[allocated_for_dlsym];
-  uptr aligned_addr = RoundUpTo(addr, alignment);
-  uptr aligned_size = RoundUpTo(size_in_bytes, kWordSize);
-
-  uptr *end_mem = (uptr*)(aligned_addr + aligned_size);
-  uptr allocated = end_mem - alloc_memory_for_dlsym;
-  if (allocated >= kDlsymAllocPoolSize)
-    return errno_ENOMEM;
-
-  allocated_for_dlsym = allocated;
-  *memptr = (void*)aligned_addr;
-  return 0;
-}
-
-static inline bool MaybeInDlsym() {
-  // Fuchsia doesn't use dlsym-based interceptors.
-  return !SANITIZER_FUCHSIA && asan_init_is_running;
-}
-
-static inline bool UseLocalPool() { return MaybeInDlsym(); }
-
-static void *ReallocFromLocalPool(void *ptr, uptr size) {
-  const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  const uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
-  void *new_ptr;
-  if (UNLIKELY(UseLocalPool())) {
-    new_ptr = AllocateFromLocalPool(size);
-  } else {
-    ENSURE_ASAN_INITED();
-    GET_STACK_TRACE_MALLOC;
-    new_ptr = asan_malloc(size, &stack);
+  static void OnFree(const void *ptr, uptr size) {
+#  if CAN_SANITIZE_LEAKS
+    __lsan_unregister_root_region(ptr, size);
+#  endif
   }
-  internal_memcpy(new_ptr, ptr, copy_size);
-  return new_ptr;
-}
+};
 
 INTERCEPTOR(void, free, void *ptr) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
-    DeallocateFromLocalPool(ptr);
-    return;
-  }
+  if (DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Free(ptr);
   GET_STACK_TRACE_FREE;
   asan_free(ptr, &stack, FROM_MALLOC);
 }
 
 #if SANITIZER_INTERCEPT_CFREE
 INTERCEPTOR(void, cfree, void *ptr) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr)))
-    return;
+  if (DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Free(ptr);
   GET_STACK_TRACE_FREE;
   asan_free(ptr, &stack, FROM_MALLOC);
 }
 #endif // SANITIZER_INTERCEPT_CFREE
 
 INTERCEPTOR(void*, malloc, uptr size) {
-  if (UNLIKELY(UseLocalPool()))
-    // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(size);
+  if (DlsymAlloc::Use())
+    return DlsymAlloc::Allocate(size);
   ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_malloc(size, &stack);
 }
 
 INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
-  if (UNLIKELY(UseLocalPool()))
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(nmemb * size);
+  if (DlsymAlloc::Use())
+    return DlsymAlloc::Callocate(nmemb, size);
   ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_calloc(nmemb, size, &stack);
 }
 
 INTERCEPTOR(void*, realloc, void *ptr, uptr size) {
-  if (UNLIKELY(IsInDlsymAllocPool(ptr)))
-    return ReallocFromLocalPool(ptr, size);
-  if (UNLIKELY(UseLocalPool()))
-    return AllocateFromLocalPool(size);
+  if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Realloc(ptr, size);
   ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_realloc(ptr, size, &stack);
@@ -205,8 +142,6 @@ INTERCEPTOR(int, mallopt, int cmd, int value) {
 #endif // SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO
 
 INTERCEPTOR(int, posix_memalign, void **memptr, uptr alignment, uptr size) {
-  if (UNLIKELY(UseLocalPool()))
-    return PosixMemalignFromLocalPool(memptr, alignment, size);
   GET_STACK_TRACE_MALLOC;
   return asan_posix_memalign(memptr, alignment, size, &stack);
 }
diff --git a/libsanitizer/asan/asan_poisoning.cpp b/libsanitizer/asan/asan_poisoning.cpp
index 5f215fe..d97af91 100644
--- a/libsanitizer/asan/asan_poisoning.cpp
+++ b/libsanitizer/asan/asan_poisoning.cpp
@@ -66,7 +66,7 @@ void AsanPoisonOrUnpoisonIntraObjectRedzone(uptr ptr, uptr size, bool poison) {
   uptr end = ptr + size;
   if (Verbosity()) {
     Printf("__asan_%spoison_intra_object_redzone [%p,%p) %zd\n",
-           poison ? "" : "un", ptr, end, size);
+           poison ? "" : "un", (void *)ptr, (void *)end, size);
     if (Verbosity() >= 2)
       PRINT_CURRENT_STACK();
   }
diff --git a/libsanitizer/asan/asan_report.cpp b/libsanitizer/asan/asan_report.cpp
index 271d896..1f26633 100644
--- a/libsanitizer/asan/asan_report.cpp
+++ b/libsanitizer/asan/asan_report.cpp
@@ -67,14 +67,14 @@ static void PrintZoneForPointer(uptr ptr, uptr zone_ptr,
                                 const char *zone_name) {
   if (zone_ptr) {
     if (zone_name) {
-      Printf("malloc_zone_from_ptr(%p) = %p, which is %s\n",
-                 ptr, zone_ptr, zone_name);
+      Printf("malloc_zone_from_ptr(%p) = %p, which is %s\n", (void *)ptr,
+             (void *)zone_ptr, zone_name);
     } else {
       Printf("malloc_zone_from_ptr(%p) = %p, which doesn't have a name\n",
-                 ptr, zone_ptr);
+             (void *)ptr, (void *)zone_ptr);
     }
   } else {
-    Printf("malloc_zone_from_ptr(%p) = 0\n", ptr);
+    Printf("malloc_zone_from_ptr(%p) = 0\n", (void *)ptr);
   }
 }
 
@@ -435,9 +435,10 @@ static inline void CheckForInvalidPointerPair(void *p1, void *p2) {
 void ReportMacMzReallocUnknown(uptr addr, uptr zone_ptr, const char *zone_name,
                                BufferedStackTrace *stack) {
   ScopedInErrorReport in_report;
-  Printf("mz_realloc(%p) -- attempting to realloc unallocated memory.\n"
-             "This is an unrecoverable problem, exiting now.\n",
-             addr);
+  Printf(
+      "mz_realloc(%p) -- attempting to realloc unallocated memory.\n"
+      "This is an unrecoverable problem, exiting now.\n",
+      (void *)addr);
   PrintZoneForPointer(addr, zone_ptr, zone_name);
   stack->Print();
   DescribeAddressIfHeap(addr);
diff --git a/libsanitizer/asan/asan_rtl.cpp b/libsanitizer/asan/asan_rtl.cpp
index bfaa3bc..1b150b3 100644
--- a/libsanitizer/asan/asan_rtl.cpp
+++ b/libsanitizer/asan/asan_rtl.cpp
@@ -557,7 +557,8 @@ void UnpoisonStack(uptr bottom, uptr top, const char *type) {
         "False positive error reports may follow\n"
         "For details see "
         "https://github.com/google/sanitizers/issues/189\n",
-        type, top, bottom, top - bottom, top - bottom);
+        type, (void *)top, (void *)bottom, (void *)(top - bottom),
+        top - bottom);
     return;
   }
   PoisonShadow(bottom, RoundUpTo(top - bottom, SHADOW_GRANULARITY), 0);
diff --git a/libsanitizer/asan/asan_shadow_setup.cpp b/libsanitizer/asan/asan_shadow_setup.cpp
index 6e6260d..fc6de39 100644
--- a/libsanitizer/asan/asan_shadow_setup.cpp
+++ b/libsanitizer/asan/asan_shadow_setup.cpp
@@ -33,7 +33,7 @@ static void ProtectGap(uptr addr, uptr size) {
           "protect_shadow_gap=0:"
           " not protecting shadow gap, allocating gap's shadow\n"
           "|| `[%p, %p]` || ShadowGap's shadow ||\n",
-          GapShadowBeg, GapShadowEnd);
+          (void*)GapShadowBeg, (void*)GapShadowEnd);
     ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd,
                              "unprotected gap shadow");
     return;
@@ -113,7 +113,7 @@ void InitializeShadowMemory() {
         "Shadow memory range interleaves with an existing memory mapping. "
         "ASan cannot proceed correctly. ABORTING.\n");
     Report("ASan shadow was supposed to be located in the [%p-%p] range.\n",
-           shadow_start, kHighShadowEnd);
+           (void*)shadow_start, (void*)kHighShadowEnd);
     MaybeReportLinuxPIEBug();
     DumpProcessMap();
     Die();
diff --git a/libsanitizer/asan/asan_thread.cpp b/libsanitizer/asan/asan_thread.cpp
index d25e8ee..8af7425 100644
--- a/libsanitizer/asan/asan_thread.cpp
+++ b/libsanitizer/asan/asan_thread.cpp
@@ -254,7 +254,7 @@ void AsanThread::Init(const InitOptions *options) {
   int local = 0;
   VReport(1, "T%d: stack [%p,%p) size 0x%zx; local=%p\n", tid(),
           (void *)stack_bottom_, (void *)stack_top_, stack_top_ - stack_bottom_,
-          &local);
+          (void *)&local);
 }
 
 // Fuchsia doesn't use ThreadStart.
@@ -443,7 +443,7 @@ AsanThread *GetCurrentThread() {
 
 void SetCurrentThread(AsanThread *t) {
   CHECK(t->context());
-  VReport(2, "SetCurrentThread: %p for thread %p\n", t->context(),
+  VReport(2, "SetCurrentThread: %p for thread %p\n", (void *)t->context(),
           (void *)GetThreadSelf());
   // Make sure we do not reset the current AsanThread.
   CHECK_EQ(0, AsanTSDGet());
diff --git a/libsanitizer/configure b/libsanitizer/configure
index 1d9ae59..08bdbc6 100755
--- a/libsanitizer/configure
+++ b/libsanitizer/configure
@@ -12356,7 +12356,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 12369 "configure"
+#line 12359 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -12462,7 +12462,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 12475 "configure"
+#line 12465 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -15201,16 +15201,6 @@ freebsd* | dragonfly*)
   esac
   ;;
 
-gnu*)
-  version_type=linux
-  need_lib_prefix=no
-  need_version=no
-  library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}${major} ${libname}${shared_ext}'
-  soname_spec='${libname}${release}${shared_ext}$major'
-  shlibpath_var=LD_LIBRARY_PATH
-  hardcode_into_libs=yes
-  ;;
-
 haiku*)
   version_type=linux
   need_lib_prefix=no
@@ -15332,7 +15322,7 @@ linux*oldld* | linux*aout* | linux*coff*)
 # project, but have not yet been accepted: they are GCC-local changes
 # for the time being.  (See
 # https://lists.gnu.org/archive/html/libtool-patches/2018-05/msg00000.html)
-linux* | k*bsd*-gnu | kopensolaris*-gnu | uclinuxfdpiceabi)
+linux* | k*bsd*-gnu | kopensolaris*-gnu | gnu* | uclinuxfdpiceabi)
   version_type=linux
   need_lib_prefix=no
   need_version=no
diff --git a/libsanitizer/hwasan/hwasan.cpp b/libsanitizer/hwasan/hwasan.cpp
index e8ffbbd..6f0ea64 100644
--- a/libsanitizer/hwasan/hwasan.cpp
+++ b/libsanitizer/hwasan/hwasan.cpp
@@ -16,6 +16,7 @@
 #include "hwasan_checks.h"
 #include "hwasan_dynamic_shadow.h"
 #include "hwasan_globals.h"
+#include "hwasan_mapping.h"
 #include "hwasan_poisoning.h"
 #include "hwasan_report.h"
 #include "hwasan_thread.h"
@@ -344,7 +345,7 @@ __attribute__((constructor(0))) void __hwasan_init() {
 
   // Needs to be called here because flags()->random_tags might not have been
   // initialized when InitInstrumentation() was called.
-  GetCurrentThread()->InitRandomState();
+  GetCurrentThread()->EnsureRandomStateInited();
 
   SetPrintfAndReportCallback(AppendToErrorMessageBuffer);
   // This may call libc -> needs initialized shadow.
@@ -391,8 +392,15 @@ void __hwasan_print_shadow(const void *p, uptr sz) {
   uptr shadow_last = MemToShadow(ptr_raw + sz - 1);
   Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw,
          ptr_raw + sz, GetTagFromPointer((uptr)p));
-  for (uptr s = shadow_first; s <= shadow_last; ++s)
-    Printf("  %zx: %x\n", ShadowToMem(s), *(tag_t *)s);
+  for (uptr s = shadow_first; s <= shadow_last; ++s) {
+    tag_t mem_tag = *reinterpret_cast<tag_t *>(s);
+    uptr granule_addr = ShadowToMem(s);
+    if (mem_tag && mem_tag < kShadowAlignment)
+      Printf("  %zx: %02x(%02x)\n", granule_addr, mem_tag,
+             *reinterpret_cast<tag_t *>(granule_addr + kShadowAlignment - 1));
+    else
+      Printf("  %zx: %02x\n", granule_addr, mem_tag);
+  }
 }
 
 sptr __hwasan_test_shadow(const void *p, uptr sz) {
diff --git a/libsanitizer/hwasan/hwasan_allocation_functions.cpp b/libsanitizer/hwasan/hwasan_allocation_functions.cpp
index 850daed..9cd82db 100644
--- a/libsanitizer/hwasan/hwasan_allocation_functions.cpp
+++ b/libsanitizer/hwasan/hwasan_allocation_functions.cpp
@@ -14,6 +14,7 @@
 
 #include "hwasan.h"
 #include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 
@@ -21,22 +22,9 @@
 
 using namespace __hwasan;
 
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
-  uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  return off < sizeof(alloc_memory_for_dlsym);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
-  uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
-  void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
-  allocated_for_dlsym += size_in_words;
-  CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
-  return mem;
-}
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+  static bool UseImpl() { return !hwasan_inited; }
+};
 
 extern "C" {
 
@@ -83,17 +71,21 @@ void *__sanitizer_pvalloc(uptr size) {
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void __sanitizer_free(void *ptr) {
-  GET_MALLOC_STACK_TRACE;
-  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+  if (!ptr)
     return;
+  if (DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Free(ptr);
+  GET_MALLOC_STACK_TRACE;
   hwasan_free(ptr, &stack);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void __sanitizer_cfree(void *ptr) {
-  GET_MALLOC_STACK_TRACE;
-  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+  if (!ptr)
     return;
+  if (DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Free(ptr);
+  GET_MALLOC_STACK_TRACE;
   hwasan_free(ptr, &stack);
 }
 
@@ -119,29 +111,17 @@ void __sanitizer_malloc_stats(void) {
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_calloc(uptr nmemb, uptr size) {
+  if (DlsymAlloc::Use())
+    return DlsymAlloc::Callocate(nmemb, size);
   GET_MALLOC_STACK_TRACE;
-  if (UNLIKELY(!hwasan_inited))
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(nmemb * size);
   return hwasan_calloc(nmemb, size, &stack);
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_realloc(void *ptr, uptr size) {
+  if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Realloc(ptr, size);
   GET_MALLOC_STACK_TRACE;
-  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
-    uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-    uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
-    void *new_ptr;
-    if (UNLIKELY(!hwasan_inited)) {
-      new_ptr = AllocateFromLocalPool(copy_size);
-    } else {
-      copy_size = size;
-      new_ptr = hwasan_malloc(copy_size, &stack);
-    }
-    internal_memcpy(new_ptr, ptr, copy_size);
-    return new_ptr;
-  }
   return hwasan_realloc(ptr, size, &stack);
 }
 
@@ -153,12 +133,11 @@ void *__sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) {
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_malloc(uptr size) {
-  GET_MALLOC_STACK_TRACE;
   if (UNLIKELY(!hwasan_init_is_running))
     ENSURE_HWASAN_INITED();
-  if (UNLIKELY(!hwasan_inited))
-    // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(size);
+  if (DlsymAlloc::Use())
+    return DlsymAlloc::Allocate(size);
+  GET_MALLOC_STACK_TRACE;
   return hwasan_malloc(size, &stack);
 }
 
diff --git a/libsanitizer/hwasan/hwasan_exceptions.cpp b/libsanitizer/hwasan/hwasan_exceptions.cpp
index 169e787..6ed1da3 100644
--- a/libsanitizer/hwasan/hwasan_exceptions.cpp
+++ b/libsanitizer/hwasan/hwasan_exceptions.cpp
@@ -29,8 +29,8 @@ typedef _Unwind_Reason_Code PersonalityFn(int version, _Unwind_Action actions,
 // is statically linked and the sanitizer runtime and the program are linked
 // against different unwinders. The _Unwind_Context data structure is opaque so
 // it may be incompatible between unwinders.
-typedef _Unwind_Word GetGRFn(_Unwind_Context* context, int index);
-typedef _Unwind_Word GetCFAFn(_Unwind_Context* context);
+typedef uintptr_t GetGRFn(_Unwind_Context* context, int index);
+typedef uintptr_t GetCFAFn(_Unwind_Context* context);
 
 extern "C" SANITIZER_INTERFACE_ATTRIBUTE _Unwind_Reason_Code
 __hwasan_personality_wrapper(int version, _Unwind_Action actions,
diff --git a/libsanitizer/hwasan/hwasan_fuchsia.cpp b/libsanitizer/hwasan/hwasan_fuchsia.cpp
index f51e148..94e5c5f 100644
--- a/libsanitizer/hwasan/hwasan_fuchsia.cpp
+++ b/libsanitizer/hwasan/hwasan_fuchsia.cpp
@@ -130,7 +130,7 @@ static void ThreadCreateHook(void *hook, bool aborted) {
 static void ThreadStartHook(void *hook, thrd_t self) {
   Thread *thread = static_cast<Thread *>(hook);
   FinishThreadInitialization(thread);
-  thread->InitRandomState();
+  thread->EnsureRandomStateInited();
 }
 
 // This is the function that sets up the stack ring buffer and enables us to use
diff --git a/libsanitizer/hwasan/hwasan_linux.cpp b/libsanitizer/hwasan/hwasan_linux.cpp
index a86ec28..ba9e236 100644
--- a/libsanitizer/hwasan/hwasan_linux.cpp
+++ b/libsanitizer/hwasan/hwasan_linux.cpp
@@ -250,7 +250,7 @@ void InstallAtExitHandler() { atexit(HwasanAtExit); }
 // ---------------------- TSD ---------------- {{{1
 
 extern "C" void __hwasan_thread_enter() {
-  hwasanThreadList().CreateCurrentThread()->InitRandomState();
+  hwasanThreadList().CreateCurrentThread()->EnsureRandomStateInited();
 }
 
 extern "C" void __hwasan_thread_exit() {
diff --git a/libsanitizer/hwasan/hwasan_report.cpp b/libsanitizer/hwasan/hwasan_report.cpp
index 9b3b661..66d3d15 100644
--- a/libsanitizer/hwasan/hwasan_report.cpp
+++ b/libsanitizer/hwasan/hwasan_report.cpp
@@ -518,7 +518,7 @@ static void PrintTagInfoAroundAddr(tag_t *tag_ptr, uptr num_rows,
   InternalScopedString s;
   for (tag_t *row = beg_row; row < end_row; row += row_len) {
     s.append("%s", row == center_row_beg ? "=>" : "  ");
-    s.append("%p:", row);
+    s.append("%p:", (void *)row);
     for (uptr i = 0; i < row_len; i++) {
       s.append("%s", row + i == tag_ptr ? "[" : " ");
       print_tag(s, &row[i]);
@@ -702,12 +702,33 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size,
   tag_t mem_tag = *tag_ptr;
 
   Printf("%s", d.Access());
-  Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n",
-         is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag,
-         mem_tag, t->unique_id());
+  if (mem_tag && mem_tag < kShadowAlignment) {
+    tag_t *granule_ptr = reinterpret_cast<tag_t *>((untagged_addr + offset) &
+                                                   ~(kShadowAlignment - 1));
+    // If offset is 0, (untagged_addr + offset) is not aligned to granules.
+    // This is the offset of the leftmost accessed byte within the bad granule.
+    u8 in_granule_offset = (untagged_addr + offset) & (kShadowAlignment - 1);
+    tag_t short_tag = granule_ptr[kShadowAlignment - 1];
+    // The first mismatch was a short granule that matched the ptr_tag.
+    if (short_tag == ptr_tag) {
+      // If the access starts after the end of the short granule, then the first
+      // bad byte is the first byte of the access; otherwise it is the first
+      // byte past the end of the short granule
+      if (mem_tag > in_granule_offset) {
+        offset += mem_tag - in_granule_offset;
+      }
+    }
+    Printf(
+        "%s of size %zu at %p tags: %02x/%02x(%02x) (ptr/mem) in thread T%zd\n",
+        is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag,
+        mem_tag, short_tag, t->unique_id());
+  } else {
+    Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n",
+           is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag,
+           mem_tag, t->unique_id());
+  }
   if (offset != 0)
-    Printf("Invalid access starting at offset [%zu, %zu)\n", offset,
-           Min(access_size, static_cast<uptr>(offset) + (1 << kShadowScale)));
+    Printf("Invalid access starting at offset %zu\n", offset);
   Printf("%s", d.Default());
 
   stack->Print();
diff --git a/libsanitizer/hwasan/hwasan_thread.cpp b/libsanitizer/hwasan/hwasan_thread.cpp
index 5b65718..c776ae1 100644
--- a/libsanitizer/hwasan/hwasan_thread.cpp
+++ b/libsanitizer/hwasan/hwasan_thread.cpp
@@ -1,15 +1,15 @@
 
+#include "hwasan_thread.h"
+
 #include "hwasan.h"
+#include "hwasan_interface_internal.h"
 #include "hwasan_mapping.h"
-#include "hwasan_thread.h"
 #include "hwasan_poisoning.h"
-#include "hwasan_interface_internal.h"
-
+#include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_file.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
 #include "sanitizer_common/sanitizer_tls_get_addr.h"
 
-
 namespace __hwasan {
 
 static u32 RandomSeed() {
@@ -27,6 +27,7 @@ static u32 RandomSeed() {
 
 void Thread::InitRandomState() {
   random_state_ = flags()->random_tags ? RandomSeed() : unique_id_;
+  random_state_inited_ = true;
 
   // Push a random number of zeros onto the ring buffer so that the first stack
   // tag base will be random.
@@ -40,8 +41,9 @@ void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size,
   CHECK_EQ(0, stack_top_);
   CHECK_EQ(0, stack_bottom_);
 
-  static u64 unique_id;
-  unique_id_ = unique_id++;
+  static atomic_uint64_t unique_id;
+  unique_id_ = atomic_fetch_add(&unique_id, 1, memory_order_relaxed);
+
   if (auto sz = flags()->heap_history_size)
     heap_allocations_ = HeapAllocationsRingBuffer::New(sz);
 
@@ -108,10 +110,9 @@ void Thread::Destroy() {
 }
 
 void Thread::Print(const char *Prefix) {
-  Printf("%sT%zd %p stack: [%p,%p) sz: %zd tls: [%p,%p)\n", Prefix,
-         unique_id_, this, stack_bottom(), stack_top(),
-         stack_top() - stack_bottom(),
-         tls_begin(), tls_end());
+  Printf("%sT%zd %p stack: [%p,%p) sz: %zd tls: [%p,%p)\n", Prefix, unique_id_,
+         (void *)this, stack_bottom(), stack_top(),
+         stack_top() - stack_bottom(), tls_begin(), tls_end());
 }
 
 static u32 xorshift(u32 state) {
@@ -124,17 +125,21 @@ static u32 xorshift(u32 state) {
 // Generate a (pseudo-)random non-zero tag.
 tag_t Thread::GenerateRandomTag(uptr num_bits) {
   DCHECK_GT(num_bits, 0);
-  if (tagging_disabled_) return 0;
+  if (tagging_disabled_)
+    return 0;
   tag_t tag;
   const uptr tag_mask = (1ULL << num_bits) - 1;
   do {
     if (flags()->random_tags) {
-      if (!random_buffer_)
+      if (!random_buffer_) {
+        EnsureRandomStateInited();
         random_buffer_ = random_state_ = xorshift(random_state_);
+      }
       CHECK(random_buffer_);
       tag = random_buffer_ & tag_mask;
       random_buffer_ >>= num_bits;
     } else {
+      EnsureRandomStateInited();
       random_state_ += 1;
       tag = random_state_ & tag_mask;
     }
diff --git a/libsanitizer/hwasan/hwasan_thread.h b/libsanitizer/hwasan/hwasan_thread.h
index 9f20afe..3db7c1a 100644
--- a/libsanitizer/hwasan/hwasan_thread.h
+++ b/libsanitizer/hwasan/hwasan_thread.h
@@ -28,12 +28,17 @@ class Thread {
 
   void Init(uptr stack_buffer_start, uptr stack_buffer_size,
             const InitState *state = nullptr);
-  void InitRandomState();
+
   void InitStackAndTls(const InitState *state = nullptr);
 
   // Must be called from the thread itself.
   void InitStackRingBuffer(uptr stack_buffer_start, uptr stack_buffer_size);
 
+  inline void EnsureRandomStateInited() {
+    if (UNLIKELY(!random_state_inited_))
+      InitRandomState();
+  }
+
   void Destroy();
 
   uptr stack_top() { return stack_top_; }
@@ -70,6 +75,7 @@ class Thread {
   // via mmap() and *must* be valid in zero-initialized state.
   void ClearShadowForThreadStackAndTLS();
   void Print(const char *prefix);
+  void InitRandomState();
   uptr vfork_spill_;
   uptr stack_top_;
   uptr stack_bottom_;
@@ -89,6 +95,8 @@ class Thread {
 
   bool announced_;
 
+  bool random_state_inited_;  // Whether InitRandomState() has been called.
+
   friend struct ThreadListHead;
 };
 
diff --git a/libsanitizer/lsan/lsan_common.cpp b/libsanitizer/lsan/lsan_common.cpp
index 5f8fc5b..308dbb3 100644
--- a/libsanitizer/lsan/lsan_common.cpp
+++ b/libsanitizer/lsan/lsan_common.cpp
@@ -131,18 +131,13 @@ static LeakSuppressionContext *GetSuppressionContext() {
   return suppression_ctx;
 }
 
-static InternalMmapVector<RootRegion> *root_regions;
+static InternalMmapVectorNoCtor<RootRegion> root_regions;
 
-InternalMmapVector<RootRegion> const *GetRootRegions() { return root_regions; }
-
-void InitializeRootRegions() {
-  CHECK(!root_regions);
-  ALIGNED(64) static char placeholder[sizeof(InternalMmapVector<RootRegion>)];
-  root_regions = new (placeholder) InternalMmapVector<RootRegion>();
+InternalMmapVectorNoCtor<RootRegion> const *GetRootRegions() {
+  return &root_regions;
 }
 
 void InitCommonLsan() {
-  InitializeRootRegions();
   if (common_flags()->detect_leaks) {
     // Initialization which can fail or print warnings should only be done if
     // LSan is actually enabled.
@@ -188,7 +183,8 @@ void ScanRangeForPointers(uptr begin, uptr end,
                           const char *region_type, ChunkTag tag) {
   CHECK(tag == kReachable || tag == kIndirectlyLeaked);
   const uptr alignment = flags()->pointer_alignment();
-  LOG_POINTERS("Scanning %s range %p-%p.\n", region_type, begin, end);
+  LOG_POINTERS("Scanning %s range %p-%p.\n", region_type, (void *)begin,
+               (void *)end);
   uptr pp = begin;
   if (pp % alignment)
     pp = pp + alignment - pp % alignment;
@@ -207,13 +203,15 @@ void ScanRangeForPointers(uptr begin, uptr end,
       LOG_POINTERS(
           "%p is poisoned: ignoring %p pointing into chunk %p-%p of size "
           "%zu.\n",
-          pp, p, chunk, chunk + m.requested_size(), m.requested_size());
+          (void *)pp, p, (void *)chunk, (void *)(chunk + m.requested_size()),
+          m.requested_size());
       continue;
     }
 
     m.set_tag(tag);
-    LOG_POINTERS("%p: found %p pointing into chunk %p-%p of size %zu.\n", pp, p,
-                 chunk, chunk + m.requested_size(), m.requested_size());
+    LOG_POINTERS("%p: found %p pointing into chunk %p-%p of size %zu.\n",
+                 (void *)pp, p, (void *)chunk,
+                 (void *)(chunk + m.requested_size()), m.requested_size());
     if (frontier)
       frontier->push_back(chunk);
   }
@@ -281,7 +279,7 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
   InternalMmapVector<uptr> registers;
   for (uptr i = 0; i < suspended_threads.ThreadCount(); i++) {
     tid_t os_id = static_cast<tid_t>(suspended_threads.GetThreadID(i));
-    LOG_THREADS("Processing thread %d.\n", os_id);
+    LOG_THREADS("Processing thread %llu.\n", os_id);
     uptr stack_begin, stack_end, tls_begin, tls_end, cache_begin, cache_end;
     DTLS *dtls;
     bool thread_found = GetThreadRangesLocked(os_id, &stack_begin, &stack_end,
@@ -290,14 +288,14 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
     if (!thread_found) {
       // If a thread can't be found in the thread registry, it's probably in the
       // process of destruction. Log this event and move on.
-      LOG_THREADS("Thread %d not found in registry.\n", os_id);
+      LOG_THREADS("Thread %llu not found in registry.\n", os_id);
       continue;
     }
     uptr sp;
     PtraceRegistersStatus have_registers =
         suspended_threads.GetRegistersAndSP(i, &registers, &sp);
     if (have_registers != REGISTERS_AVAILABLE) {
-      Report("Unable to get registers from thread %d.\n", os_id);
+      Report("Unable to get registers from thread %llu.\n", os_id);
       // If unable to get SP, consider the entire stack to be reachable unless
       // GetRegistersAndSP failed with ESRCH.
       if (have_registers == REGISTERS_UNAVAILABLE_FATAL) continue;
@@ -313,7 +311,8 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
     }
 
     if (flags()->use_stacks) {
-      LOG_THREADS("Stack at %p-%p (SP = %p).\n", stack_begin, stack_end, sp);
+      LOG_THREADS("Stack at %p-%p (SP = %p).\n", (void *)stack_begin,
+                  (void *)stack_end, (void *)sp);
       if (sp < stack_begin || sp >= stack_end) {
         // SP is outside the recorded stack range (e.g. the thread is running a
         // signal handler on alternate stack, or swapcontext was used).
@@ -327,7 +326,7 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
           stack_begin += page_size;
         }
         LOG_THREADS("Skipped %d guard page(s) to obtain stack %p-%p.\n",
-                    skipped, stack_begin, stack_end);
+                    skipped, (void *)stack_begin, (void *)stack_end);
       } else {
         // Shrink the stack range to ignore out-of-scope values.
         stack_begin = sp;
@@ -339,7 +338,7 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
 
     if (flags()->use_tls) {
       if (tls_begin) {
-        LOG_THREADS("TLS at %p-%p.\n", tls_begin, tls_end);
+        LOG_THREADS("TLS at %p-%p.\n", (void *)tls_begin, (void *)tls_end);
         // If the tls and cache ranges don't overlap, scan full tls range,
         // otherwise, only scan the non-overlapping portions
         if (cache_begin == cache_end || tls_end < cache_begin ||
@@ -373,7 +372,8 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
           uptr dtls_beg = dtv.beg;
           uptr dtls_end = dtls_beg + dtv.size;
           if (dtls_beg < dtls_end) {
-            LOG_THREADS("DTLS %zu at %p-%p.\n", id, dtls_beg, dtls_end);
+            LOG_THREADS("DTLS %d at %p-%p.\n", id, (void *)dtls_beg,
+                        (void *)dtls_end);
             ScanRangeForPointers(dtls_beg, dtls_end, frontier, "DTLS",
                                  kReachable);
           }
@@ -381,7 +381,7 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
       } else {
         // We are handling a thread with DTLS under destruction. Log about
         // this and continue.
-        LOG_THREADS("Thread %d has DTLS under destruction.\n", os_id);
+        LOG_THREADS("Thread %llu has DTLS under destruction.\n", os_id);
       }
 #endif
     }
@@ -399,8 +399,9 @@ void ScanRootRegion(Frontier *frontier, const RootRegion &root_region,
   uptr intersection_end = Min(region_end, root_region.begin + root_region.size);
   if (intersection_begin >= intersection_end) return;
   LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n",
-               root_region.begin, root_region.begin + root_region.size,
-               region_begin, region_end,
+               (void *)root_region.begin,
+               (void *)(root_region.begin + root_region.size),
+               (void *)region_begin, (void *)region_end,
                is_readable ? "readable" : "unreadable");
   if (is_readable)
     ScanRangeForPointers(intersection_begin, intersection_end, frontier, "ROOT",
@@ -420,10 +421,8 @@ static void ProcessRootRegion(Frontier *frontier,
 // Scans root regions for heap pointers.
 static void ProcessRootRegions(Frontier *frontier) {
   if (!flags()->use_root_regions) return;
-  CHECK(root_regions);
-  for (uptr i = 0; i < root_regions->size(); i++) {
-    ProcessRootRegion(frontier, (*root_regions)[i]);
-  }
+  for (uptr i = 0; i < root_regions.size(); i++)
+    ProcessRootRegion(frontier, root_regions[i]);
 }
 
 static void FloodFillTag(Frontier *frontier, ChunkTag tag) {
@@ -460,8 +459,8 @@ static void IgnoredSuppressedCb(uptr chunk, void *arg) {
   if (idx >= suppressed.size() || m.stack_trace_id() != suppressed[idx])
     return;
 
-  LOG_POINTERS("Suppressed: chunk %p-%p of size %zu.\n", chunk,
-               chunk + m.requested_size(), m.requested_size());
+  LOG_POINTERS("Suppressed: chunk %p-%p of size %zu.\n", (void *)chunk,
+               (void *)(chunk + m.requested_size()), m.requested_size());
   m.set_tag(kIgnored);
 }
 
@@ -472,8 +471,8 @@ static void CollectIgnoredCb(uptr chunk, void *arg) {
   chunk = GetUserBegin(chunk);
   LsanMetadata m(chunk);
   if (m.allocated() && m.tag() == kIgnored) {
-    LOG_POINTERS("Ignored: chunk %p-%p of size %zu.\n",
-                 chunk, chunk + m.requested_size(), m.requested_size());
+    LOG_POINTERS("Ignored: chunk %p-%p of size %zu.\n", (void *)chunk,
+                 (void *)(chunk + m.requested_size()), m.requested_size());
     reinterpret_cast<Frontier *>(arg)->push_back(chunk);
   }
 }
@@ -487,7 +486,6 @@ static uptr GetCallerPC(const StackTrace &stack) {
 
 struct InvalidPCParam {
   Frontier *frontier;
-  const StackDepotReverseMap *stack_depot;
   bool skip_linker_allocations;
 };
 
@@ -502,7 +500,7 @@ static void MarkInvalidPCCb(uptr chunk, void *arg) {
     u32 stack_id = m.stack_trace_id();
     uptr caller_pc = 0;
     if (stack_id > 0)
-      caller_pc = GetCallerPC(param->stack_depot->Get(stack_id));
+      caller_pc = GetCallerPC(StackDepotGet(stack_id));
     // If caller_pc is unknown, this chunk may be allocated in a coroutine. Mark
     // it as reachable, as we can't properly report its allocation stack anyway.
     if (caller_pc == 0 || (param->skip_linker_allocations &&
@@ -533,11 +531,9 @@ static void MarkInvalidPCCb(uptr chunk, void *arg) {
 // which we don't care about).
 // On all other platforms, this simply checks to ensure that the caller pc is
 // valid before reporting chunks as leaked.
-static void ProcessPC(Frontier *frontier,
-                      const StackDepotReverseMap &stack_depot) {
+static void ProcessPC(Frontier *frontier) {
   InvalidPCParam arg;
   arg.frontier = frontier;
-  arg.stack_depot = &stack_depot;
   arg.skip_linker_allocations =
       flags()->use_tls && flags()->use_ld_allocations && GetLinker() != nullptr;
   ForEachChunk(MarkInvalidPCCb, &arg);
@@ -545,7 +541,6 @@ static void ProcessPC(Frontier *frontier,
 
 // Sets the appropriate tag on each chunk.
 static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads,
-                              const StackDepotReverseMap &stack_depot,
                               Frontier *frontier) {
   const InternalMmapVector<u32> &suppressed_stacks =
       GetSuppressionContext()->GetSortedSuppressedStacks();
@@ -560,7 +555,7 @@ static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads,
   FloodFillTag(frontier, kReachable);
 
   CHECK_EQ(0, frontier->size());
-  ProcessPC(frontier, stack_depot);
+  ProcessPC(frontier);
 
   // The check here is relatively expensive, so we do this in a separate flood
   // fill. That way we can skip the check for chunks that are reachable
@@ -621,8 +616,9 @@ static void ReportIfNotSuspended(ThreadContextBase *tctx, void *arg) {
   if (tctx->status == ThreadStatusRunning) {
     uptr i = InternalLowerBound(suspended_threads, tctx->os_id);
     if (i >= suspended_threads.size() || suspended_threads[i] != tctx->os_id)
-      Report("Running thread %d was not suspended. False leaks are possible.\n",
-             tctx->os_id);
+      Report(
+          "Running thread %llu was not suspended. False leaks are possible.\n",
+          tctx->os_id);
   }
 }
 
@@ -654,8 +650,7 @@ static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads,
   CHECK(param);
   CHECK(!param->success);
   ReportUnsuspendedThreads(suspended_threads);
-  ClassifyAllChunks(suspended_threads, param->leak_report.stack_depot(),
-                    &param->frontier);
+  ClassifyAllChunks(suspended_threads, &param->frontier);
   ForEachChunk(CollectLeaksCb, &param->leak_report);
   // Clean up for subsequent leak checks. This assumes we did not overwrite any
   // kIgnored tags.
@@ -795,7 +790,7 @@ void LeakReport::AddLeakedChunk(uptr chunk, u32 stack_trace_id,
   CHECK(tag == kDirectlyLeaked || tag == kIndirectlyLeaked);
 
   if (u32 resolution = flags()->resolution) {
-    StackTrace stack = stack_depot_.Get(stack_trace_id);
+    StackTrace stack = StackDepotGet(stack_trace_id);
     stack.size = Min(stack.size, resolution);
     stack_trace_id = StackDepotPut(stack);
   }
@@ -863,7 +858,7 @@ void LeakReport::PrintReportForLeak(uptr index) {
   Printf("%s", d.Default());
 
   CHECK(leaks_[index].stack_trace_id);
-  stack_depot_.Get(leaks_[index].stack_trace_id).Print();
+  StackDepotGet(leaks_[index].stack_trace_id).Print();
 
   if (flags()->report_objects) {
     Printf("Objects leaked above:\n");
@@ -876,7 +871,7 @@ void LeakReport::PrintLeakedObjectsForLeak(uptr index) {
   u32 leak_id = leaks_[index].id;
   for (uptr j = 0; j < leaked_objects_.size(); j++) {
     if (leaked_objects_[j].leak_id == leak_id)
-      Printf("%p (%zu bytes)\n", leaked_objects_[j].addr,
+      Printf("%p (%zu bytes)\n", (void *)leaked_objects_[j].addr,
              leaked_objects_[j].size);
   }
 }
@@ -900,7 +895,7 @@ uptr LeakReport::ApplySuppressions() {
   uptr new_suppressions = false;
   for (uptr i = 0; i < leaks_.size(); i++) {
     Suppression *s = suppressions->GetSuppressionForStack(
-        leaks_[i].stack_trace_id, stack_depot_.Get(leaks_[i].stack_trace_id));
+        leaks_[i].stack_trace_id, StackDepotGet(leaks_[i].stack_trace_id));
     if (s) {
       s->weight += leaks_[i].total_size;
       atomic_store_relaxed(&s->hit_count, atomic_load_relaxed(&s->hit_count) +
@@ -964,10 +959,9 @@ SANITIZER_INTERFACE_ATTRIBUTE
 void __lsan_register_root_region(const void *begin, uptr size) {
 #if CAN_SANITIZE_LEAKS
   Lock l(&global_mutex);
-  CHECK(root_regions);
   RootRegion region = {reinterpret_cast<uptr>(begin), size};
-  root_regions->push_back(region);
-  VReport(1, "Registered root region at %p of size %llu\n", begin, size);
+  root_regions.push_back(region);
+  VReport(1, "Registered root region at %p of size %zu\n", begin, size);
 #endif // CAN_SANITIZE_LEAKS
 }
 
@@ -975,22 +969,21 @@ SANITIZER_INTERFACE_ATTRIBUTE
 void __lsan_unregister_root_region(const void *begin, uptr size) {
 #if CAN_SANITIZE_LEAKS
   Lock l(&global_mutex);
-  CHECK(root_regions);
   bool removed = false;
-  for (uptr i = 0; i < root_regions->size(); i++) {
-    RootRegion region = (*root_regions)[i];
+  for (uptr i = 0; i < root_regions.size(); i++) {
+    RootRegion region = root_regions[i];
     if (region.begin == reinterpret_cast<uptr>(begin) && region.size == size) {
       removed = true;
-      uptr last_index = root_regions->size() - 1;
-      (*root_regions)[i] = (*root_regions)[last_index];
-      root_regions->pop_back();
-      VReport(1, "Unregistered root region at %p of size %llu\n", begin, size);
+      uptr last_index = root_regions.size() - 1;
+      root_regions[i] = root_regions[last_index];
+      root_regions.pop_back();
+      VReport(1, "Unregistered root region at %p of size %zu\n", begin, size);
       break;
     }
   }
   if (!removed) {
     Report(
-        "__lsan_unregister_root_region(): region at %p of size %llu has not "
+        "__lsan_unregister_root_region(): region at %p of size %zu has not "
         "been registered.\n",
         begin, size);
     Die();
diff --git a/libsanitizer/lsan/lsan_common.h b/libsanitizer/lsan/lsan_common.h
index c15df1b..f9b55e4 100644
--- a/libsanitizer/lsan/lsan_common.h
+++ b/libsanitizer/lsan/lsan_common.h
@@ -108,14 +108,12 @@ class LeakReport {
   uptr ApplySuppressions();
   uptr UnsuppressedLeakCount();
   uptr IndirectUnsuppressedLeakCount();
-  const StackDepotReverseMap &stack_depot() { return stack_depot_; }
 
  private:
   void PrintReportForLeak(uptr index);
   void PrintLeakedObjectsForLeak(uptr index);
 
   u32 next_id_ = 0;
-  StackDepotReverseMap stack_depot_;
   InternalMmapVector<Leak> leaks_;
   InternalMmapVector<LeakedObject> leaked_objects_;
 };
@@ -142,7 +140,7 @@ struct CheckForLeaksParam {
   bool success = false;
 };
 
-InternalMmapVector<RootRegion> const *GetRootRegions();
+InternalMmapVectorNoCtor<RootRegion> const *GetRootRegions();
 void ScanRootRegion(Frontier *frontier, RootRegion const &region,
                     uptr region_begin, uptr region_end, bool is_readable);
 void ForEachExtraStackRangeCb(uptr begin, uptr end, void* arg);
@@ -282,6 +280,13 @@ int __lsan_is_turned_off();
 
 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
 const char *__lsan_default_suppressions();
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __lsan_register_root_region(const void *p, __lsan::uptr size);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __lsan_unregister_root_region(const void *p, __lsan::uptr size);
+
 }  // extern "C"
 
 #endif  // LSAN_COMMON_H
diff --git a/libsanitizer/lsan/lsan_common_mac.cpp b/libsanitizer/lsan/lsan_common_mac.cpp
index 8516a17..4301dcc 100644
--- a/libsanitizer/lsan/lsan_common_mac.cpp
+++ b/libsanitizer/lsan/lsan_common_mac.cpp
@@ -149,7 +149,7 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) {
   kern_return_t err = KERN_SUCCESS;
   mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
 
-  InternalMmapVector<RootRegion> const *root_regions = GetRootRegions();
+  InternalMmapVectorNoCtor<RootRegion> const *root_regions = GetRootRegions();
 
   while (err == KERN_SUCCESS) {
     struct vm_region_submap_info_64 info;
diff --git a/libsanitizer/lsan/lsan_interceptors.cpp b/libsanitizer/lsan/lsan_interceptors.cpp
index 90a90a5..22999d5 100644
--- a/libsanitizer/lsan/lsan_interceptors.cpp
+++ b/libsanitizer/lsan/lsan_interceptors.cpp
@@ -13,6 +13,7 @@
 
 #include "interception/interception.h"
 #include "sanitizer_common/sanitizer_allocator.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_allocator_report.h"
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
@@ -43,6 +44,22 @@ int pthread_key_create(unsigned *key, void (*destructor)(void* v));
 int pthread_setspecific(unsigned key, const void *v);
 }
 
+struct DlsymAlloc : DlSymAllocator<DlsymAlloc> {
+  static bool UseImpl() { return lsan_init_is_running; }
+  static void OnAllocate(const void *ptr, uptr size) {
+#if CAN_SANITIZE_LEAKS
+    // Suppress leaks from dlerror(). Previously dlsym hack on global array was
+    // used by leak sanitizer as a root region.
+    __lsan_register_root_region(ptr, size);
+#endif
+  }
+  static void OnFree(const void *ptr, uptr size) {
+#if CAN_SANITIZE_LEAKS
+    __lsan_unregister_root_region(ptr, size);
+#endif
+  }
+};
+
 ///// Malloc/free interceptors. /////
 
 namespace std {
@@ -52,41 +69,34 @@ namespace std {
 
 #if !SANITIZER_MAC
 INTERCEPTOR(void*, malloc, uptr size) {
+  if (DlsymAlloc::Use())
+    return DlsymAlloc::Allocate(size);
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
   return lsan_malloc(size, stack);
 }
 
 INTERCEPTOR(void, free, void *p) {
+  if (DlsymAlloc::PointerIsMine(p))
+    return DlsymAlloc::Free(p);
   ENSURE_LSAN_INITED;
   lsan_free(p);
 }
 
 INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
-  // This hack is not required for Fuchsia because there are no dlsym calls
-  // involved in setting up interceptors.
-#if !SANITIZER_FUCHSIA
-  if (lsan_init_is_running) {
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    const uptr kCallocPoolSize = 1024;
-    static uptr calloc_memory_for_dlsym[kCallocPoolSize];
-    static uptr allocated;
-    uptr size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
-    void *mem = (void*)&calloc_memory_for_dlsym[allocated];
-    allocated += size_in_words;
-    CHECK(allocated < kCallocPoolSize);
-    return mem;
-  }
-#endif  // !SANITIZER_FUCHSIA
+  if (DlsymAlloc::Use())
+    return DlsymAlloc::Callocate(nmemb, size);
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
   return lsan_calloc(nmemb, size, stack);
 }
 
-INTERCEPTOR(void*, realloc, void *q, uptr size) {
+INTERCEPTOR(void *, realloc, void *ptr, uptr size) {
+  if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Realloc(ptr, size);
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
-  return lsan_realloc(q, size, stack);
+  return lsan_realloc(ptr, size, stack);
 }
 
 INTERCEPTOR(void*, reallocarray, void *q, uptr nmemb, uptr size) {
diff --git a/libsanitizer/sanitizer_common/Makefile.am b/libsanitizer/sanitizer_common/Makefile.am
index d04f2d8..57d7f02 100644
--- a/libsanitizer/sanitizer_common/Makefile.am
+++ b/libsanitizer/sanitizer_common/Makefile.am
@@ -44,7 +44,6 @@ sanitizer_common_files = \
 	sanitizer_mutex.cpp \
 	sanitizer_netbsd.cpp \
 	sanitizer_openbsd.cpp \
-	sanitizer_persistent_allocator.cpp \
 	sanitizer_platform_limits_freebsd.cpp \
 	sanitizer_platform_limits_linux.cpp \
 	sanitizer_platform_limits_openbsd.cpp \
diff --git a/libsanitizer/sanitizer_common/Makefile.in b/libsanitizer/sanitizer_common/Makefile.in
index 2856894..ce24f6e 100644
--- a/libsanitizer/sanitizer_common/Makefile.in
+++ b/libsanitizer/sanitizer_common/Makefile.in
@@ -130,7 +130,6 @@ am__objects_1 = sancov_flags.lo sanitizer_allocator.lo \
 	sanitizer_linux_libcdep.lo sanitizer_linux_s390.lo \
 	sanitizer_mac.lo sanitizer_mac_libcdep.lo sanitizer_mutex.lo \
 	sanitizer_netbsd.lo sanitizer_openbsd.lo \
-	sanitizer_persistent_allocator.lo \
 	sanitizer_platform_limits_freebsd.lo \
 	sanitizer_platform_limits_linux.lo \
 	sanitizer_platform_limits_openbsd.lo \
@@ -404,7 +403,6 @@ sanitizer_common_files = \
 	sanitizer_mutex.cpp \
 	sanitizer_netbsd.cpp \
 	sanitizer_openbsd.cpp \
-	sanitizer_persistent_allocator.cpp \
 	sanitizer_platform_limits_freebsd.cpp \
 	sanitizer_platform_limits_linux.cpp \
 	sanitizer_platform_limits_openbsd.cpp \
@@ -561,7 +559,6 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_mutex.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_netbsd.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_openbsd.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_persistent_allocator.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_platform_limits_freebsd.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_platform_limits_linux.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_platform_limits_openbsd.Plo@am__quote@
diff --git a/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h b/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h
index 73b48cb..7e2fa91 100644
--- a/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h
+++ b/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h
@@ -39,6 +39,11 @@ namespace __sanitizer {
 //   the current thread has exclusive access to the data
 //   if !h.exists() then the element never existed
 // }
+// {
+//   Map::Handle h(&m, addr, false, true);
+//   this will create a new element or return a handle to an existing element
+//   if !h.created() this thread does *not* have exclusive access to the data
+// }
 template<typename T, uptr kSize>
 class AddrHashMap {
  private:
@@ -89,6 +94,12 @@ class AddrHashMap {
     bool                   create_;
   };
 
+  typedef void (*ForEachCallback)(const uptr key, const T &val, void *arg);
+  // ForEach acquires a lock on each bucket while iterating over
+  // elements. Note that this only ensures that the structure of the hashmap is
+  // unchanged, there may be a data race to the element itself.
+  void ForEach(ForEachCallback cb, void *arg);
+
  private:
   friend class Handle;
   Bucket *table_;
@@ -98,6 +109,33 @@ class AddrHashMap {
   uptr calcHash(uptr addr);
 };
 
+template <typename T, uptr kSize>
+void AddrHashMap<T, kSize>::ForEach(ForEachCallback cb, void *arg) {
+  for (uptr n = 0; n < kSize; n++) {
+    Bucket *bucket = &table_[n];
+
+    ReadLock lock(&bucket->mtx);
+
+    for (uptr i = 0; i < kBucketSize; i++) {
+      Cell *c = &bucket->cells[i];
+      uptr addr1 = atomic_load(&c->addr, memory_order_acquire);
+      if (addr1 != 0)
+        cb(addr1, c->val, arg);
+    }
+
+    // Iterate over any additional cells.
+    if (AddBucket *add =
+            (AddBucket *)atomic_load(&bucket->add, memory_order_acquire)) {
+      for (uptr i = 0; i < add->size; i++) {
+        Cell *c = &add->cells[i];
+        uptr addr1 = atomic_load(&c->addr, memory_order_acquire);
+        if (addr1 != 0)
+          cb(addr1, c->val, arg);
+      }
+    }
+  }
+}
+
 template<typename T, uptr kSize>
 AddrHashMap<T, kSize>::Handle::Handle(AddrHashMap<T, kSize> *map, uptr addr) {
   map_ = map;
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator.h b/libsanitizer/sanitizer_common/sanitizer_allocator.h
index 5ec4741..ec23465 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator.h
@@ -14,6 +14,7 @@
 #define SANITIZER_ALLOCATOR_H
 
 #include "sanitizer_common.h"
+#include "sanitizer_flat_map.h"
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_lfstack.h"
 #include "sanitizer_libc.h"
@@ -43,12 +44,6 @@ void SetAllocatorOutOfMemory();
 
 void PrintHintAllocatorCannotReturnNull();
 
-// Allocators call these callbacks on mmap/munmap.
-struct NoOpMapUnmapCallback {
-  void OnMap(uptr p, uptr size) const { }
-  void OnUnmap(uptr p, uptr size) const { }
-};
-
 // Callback type for iterating over chunks.
 typedef void (*ForEachChunkCallback)(uptr chunk, void *arg);
 
@@ -70,7 +65,6 @@ inline void RandomShuffle(T *a, u32 n, u32 *rand_state) {
 #include "sanitizer_allocator_size_class_map.h"
 #include "sanitizer_allocator_stats.h"
 #include "sanitizer_allocator_primary64.h"
-#include "sanitizer_allocator_bytemap.h"
 #include "sanitizer_allocator_primary32.h"
 #include "sanitizer_allocator_local_cache.h"
 #include "sanitizer_allocator_secondary.h"
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_bytemap.h b/libsanitizer/sanitizer_common/sanitizer_allocator_bytemap.h
deleted file mode 100644
index 0084bb6..0000000
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_bytemap.h
+++ /dev/null
@@ -1,107 +0,0 @@
-//===-- sanitizer_allocator_bytemap.h ---------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Part of the Sanitizer Allocator.
-//
-//===----------------------------------------------------------------------===//
-#ifndef SANITIZER_ALLOCATOR_H
-#error This file must be included inside sanitizer_allocator.h
-#endif
-
-// Maps integers in rage [0, kSize) to u8 values.
-template <u64 kSize, typename AddressSpaceViewTy = LocalAddressSpaceView>
-class FlatByteMap {
- public:
-  using AddressSpaceView = AddressSpaceViewTy;
-  void Init() {
-    internal_memset(map_, 0, sizeof(map_));
-  }
-
-  void set(uptr idx, u8 val) {
-    CHECK_LT(idx, kSize);
-    CHECK_EQ(0U, map_[idx]);
-    map_[idx] = val;
-  }
-  u8 operator[] (uptr idx) {
-    CHECK_LT(idx, kSize);
-    // FIXME: CHECK may be too expensive here.
-    return map_[idx];
-  }
- private:
-  u8 map_[kSize];
-};
-
-// TwoLevelByteMap maps integers in range [0, kSize1*kSize2) to u8 values.
-// It is implemented as a two-dimensional array: array of kSize1 pointers
-// to kSize2-byte arrays. The secondary arrays are mmaped on demand.
-// Each value is initially zero and can be set to something else only once.
-// Setting and getting values from multiple threads is safe w/o extra locking.
-template <u64 kSize1, u64 kSize2,
-          typename AddressSpaceViewTy = LocalAddressSpaceView,
-          class MapUnmapCallback = NoOpMapUnmapCallback>
-class TwoLevelByteMap {
- public:
-  using AddressSpaceView = AddressSpaceViewTy;
-  void Init() {
-    internal_memset(map1_, 0, sizeof(map1_));
-    mu_.Init();
-  }
-
-  void TestOnlyUnmap() {
-    for (uptr i = 0; i < kSize1; i++) {
-      u8 *p = Get(i);
-      if (!p) continue;
-      MapUnmapCallback().OnUnmap(reinterpret_cast<uptr>(p), kSize2);
-      UnmapOrDie(p, kSize2);
-    }
-  }
-
-  uptr size() const { return kSize1 * kSize2; }
-  uptr size1() const { return kSize1; }
-  uptr size2() const { return kSize2; }
-
-  void set(uptr idx, u8 val) {
-    CHECK_LT(idx, kSize1 * kSize2);
-    u8 *map2 = GetOrCreate(idx / kSize2);
-    CHECK_EQ(0U, map2[idx % kSize2]);
-    map2[idx % kSize2] = val;
-  }
-
-  u8 operator[] (uptr idx) const {
-    CHECK_LT(idx, kSize1 * kSize2);
-    u8 *map2 = Get(idx / kSize2);
-    if (!map2) return 0;
-    auto value_ptr = AddressSpaceView::Load(&map2[idx % kSize2]);
-    return *value_ptr;
-  }
-
- private:
-  u8 *Get(uptr idx) const {
-    CHECK_LT(idx, kSize1);
-    return reinterpret_cast<u8 *>(
-        atomic_load(&map1_[idx], memory_order_acquire));
-  }
-
-  u8 *GetOrCreate(uptr idx) {
-    u8 *res = Get(idx);
-    if (!res) {
-      SpinMutexLock l(&mu_);
-      if (!(res = Get(idx))) {
-        res = (u8*)MmapOrDie(kSize2, "TwoLevelByteMap");
-        MapUnmapCallback().OnMap(reinterpret_cast<uptr>(res), kSize2);
-        atomic_store(&map1_[idx], reinterpret_cast<uptr>(res),
-                     memory_order_release);
-      }
-    }
-    return res;
-  }
-
-  atomic_uintptr_t map1_[kSize1];
-  StaticSpinMutex mu_;
-};
-
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h
index 0e81e67..9a3602f 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h
@@ -112,15 +112,13 @@ class CombinedAllocator {
     return new_p;
   }
 
-  bool PointerIsMine(void *p) {
+  bool PointerIsMine(const void *p) const {
     if (primary_.PointerIsMine(p))
       return true;
     return secondary_.PointerIsMine(p);
   }
 
-  bool FromPrimary(void *p) {
-    return primary_.PointerIsMine(p);
-  }
+  bool FromPrimary(const void *p) const { return primary_.PointerIsMine(p); }
 
   void *GetMetaData(const void *p) {
     if (primary_.PointerIsMine(p))
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_dlsym.h b/libsanitizer/sanitizer_common/sanitizer_allocator_dlsym.h
new file mode 100644
index 0000000..92b1373
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_dlsym.h
@@ -0,0 +1,79 @@
+//===-- sanitizer_allocator_dlsym.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Hack: Sanitizer initializer calls dlsym which may need to allocate and call
+// back into uninitialized sanitizer.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ALLOCATOR_DLSYM_H
+#define SANITIZER_ALLOCATOR_DLSYM_H
+
+#include "sanitizer_allocator_internal.h"
+
+namespace __sanitizer {
+
+template <typename Details>
+struct DlSymAllocator {
+  static bool Use() {
+    // Fuchsia doesn't use dlsym-based interceptors.
+    return !SANITIZER_FUCHSIA && UNLIKELY(Details::UseImpl());
+  }
+
+  static bool PointerIsMine(const void *ptr) {
+    // Fuchsia doesn't use dlsym-based interceptors.
+    return !SANITIZER_FUCHSIA &&
+           UNLIKELY(internal_allocator()->FromPrimary(ptr));
+  }
+
+  static void *Allocate(uptr size_in_bytes) {
+    void *ptr = InternalAlloc(size_in_bytes, nullptr, kWordSize);
+    CHECK(internal_allocator()->FromPrimary(ptr));
+    Details::OnAllocate(ptr,
+                        internal_allocator()->GetActuallyAllocatedSize(ptr));
+    return ptr;
+  }
+
+  static void *Callocate(SIZE_T nmemb, SIZE_T size) {
+    void *ptr = InternalCalloc(nmemb, size);
+    CHECK(internal_allocator()->FromPrimary(ptr));
+    Details::OnAllocate(ptr,
+                        internal_allocator()->GetActuallyAllocatedSize(ptr));
+    return ptr;
+  }
+
+  static void Free(void *ptr) {
+    uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
+    Details::OnFree(ptr, size);
+    InternalFree(ptr);
+  }
+
+  static void *Realloc(void *ptr, uptr new_size) {
+    if (!ptr)
+      return Allocate(new_size);
+    CHECK(internal_allocator()->FromPrimary(ptr));
+    if (!new_size) {
+      Free(ptr);
+      return nullptr;
+    }
+    uptr size = internal_allocator()->GetActuallyAllocatedSize(ptr);
+    uptr memcpy_size = Min(new_size, size);
+    void *new_ptr = Allocate(new_size);
+    if (new_ptr)
+      internal_memcpy(new_ptr, ptr, memcpy_size);
+    Free(ptr);
+    return new_ptr;
+  }
+
+  static void OnAllocate(const void *ptr, uptr size) {}
+  static void OnFree(const void *ptr, uptr size) {}
+};
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_ALLOCATOR_DLSYM_H
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h
index 38d2a7d..ae1b7e0 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h
@@ -189,7 +189,7 @@ class SizeClassAllocator32 {
     sci->free_list.push_front(b);
   }
 
-  bool PointerIsMine(const void *p) {
+  bool PointerIsMine(const void *p) const {
     uptr mem = reinterpret_cast<uptr>(p);
     if (SANITIZER_SIGN_EXTENDED_ADDRESSES)
       mem &= (kSpaceSize - 1);
@@ -198,8 +198,9 @@ class SizeClassAllocator32 {
     return GetSizeClass(p) != 0;
   }
 
-  uptr GetSizeClass(const void *p) {
-    return possible_regions[ComputeRegionId(reinterpret_cast<uptr>(p))];
+  uptr GetSizeClass(const void *p) const {
+    uptr id = ComputeRegionId(reinterpret_cast<uptr>(p));
+    return possible_regions.contains(id) ? possible_regions[id] : 0;
   }
 
   void *GetBlockBegin(const void *p) {
@@ -251,9 +252,9 @@ class SizeClassAllocator32 {
 
   // Iterate over all existing chunks.
   // The allocator must be locked when calling this function.
-  void ForEachChunk(ForEachChunkCallback callback, void *arg) {
+  void ForEachChunk(ForEachChunkCallback callback, void *arg) const {
     for (uptr region = 0; region < kNumPossibleRegions; region++)
-      if (possible_regions[region]) {
+      if (possible_regions.contains(region) && possible_regions[region]) {
         uptr chunk_size = ClassIdToSize(possible_regions[region]);
         uptr max_chunks_in_region = kRegionSize / (chunk_size + kMetadataSize);
         uptr region_beg = region * kRegionSize;
@@ -292,9 +293,7 @@ class SizeClassAllocator32 {
     return res;
   }
 
-  uptr ComputeRegionBeg(uptr mem) {
-    return mem & ~(kRegionSize - 1);
-  }
+  uptr ComputeRegionBeg(uptr mem) const { return mem & ~(kRegionSize - 1); }
 
   uptr AllocateRegion(AllocatorStats *stat, uptr class_id) {
     DCHECK_LT(class_id, kNumClasses);
@@ -305,7 +304,7 @@ class SizeClassAllocator32 {
     MapUnmapCallback().OnMap(res, kRegionSize);
     stat->Add(AllocatorStatMapped, kRegionSize);
     CHECK(IsAligned(res, kRegionSize));
-    possible_regions.set(ComputeRegionId(res), static_cast<u8>(class_id));
+    possible_regions[ComputeRegionId(res)] = class_id;
     return res;
   }
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h
index 3710947..f917310 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h
@@ -302,9 +302,8 @@ class SizeClassAllocator64 {
     UnmapWithCallbackOrDie((uptr)address_range.base(), address_range.size());
   }
 
-  static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats,
-                           uptr stats_size) {
-    for (uptr class_id = 0; class_id < stats_size; class_id++)
+  static void FillMemoryProfile(uptr start, uptr rss, bool file, uptr *stats) {
+    for (uptr class_id = 0; class_id < kNumClasses; class_id++)
       if (stats[class_id] == start)
         stats[class_id] = rss;
   }
@@ -330,7 +329,7 @@ class SizeClassAllocator64 {
     uptr rss_stats[kNumClasses];
     for (uptr class_id = 0; class_id < kNumClasses; class_id++)
       rss_stats[class_id] = SpaceBeg() + kRegionSize * class_id;
-    GetMemoryProfile(FillMemoryProfile, rss_stats, kNumClasses);
+    GetMemoryProfile(FillMemoryProfile, rss_stats);
 
     uptr total_mapped = 0;
     uptr total_rss = 0;
diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
index dd34fe8..c24354c 100644
--- a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
+++ b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h
@@ -161,7 +161,7 @@ class LargeMmapAllocator {
     return res;
   }
 
-  bool PointerIsMine(const void *p) {
+  bool PointerIsMine(const void *p) const {
     return GetBlockBegin(p) != nullptr;
   }
 
@@ -179,7 +179,7 @@ class LargeMmapAllocator {
     return GetHeader(p) + 1;
   }
 
-  void *GetBlockBegin(const void *ptr) {
+  void *GetBlockBegin(const void *ptr) const {
     uptr p = reinterpret_cast<uptr>(ptr);
     SpinMutexLock l(&mutex_);
     uptr nearest_chunk = 0;
@@ -301,7 +301,7 @@ class LargeMmapAllocator {
     return GetHeader(reinterpret_cast<uptr>(p));
   }
 
-  void *GetUser(const Header *h) {
+  void *GetUser(const Header *h) const {
     CHECK(IsAligned((uptr)h, page_size_));
     return reinterpret_cast<void*>(reinterpret_cast<uptr>(h) + page_size_);
   }
@@ -318,5 +318,5 @@ class LargeMmapAllocator {
   struct Stats {
     uptr n_allocs, n_frees, currently_allocated, max_allocated, by_size_log[64];
   } stats;
-  StaticSpinMutex mutex_;
+  mutable StaticSpinMutex mutex_;
 };
diff --git a/libsanitizer/sanitizer_common/sanitizer_asm.h b/libsanitizer/sanitizer_common/sanitizer_asm.h
index 6b86120..9ebba91 100644
--- a/libsanitizer/sanitizer_common/sanitizer_asm.h
+++ b/libsanitizer/sanitizer_common/sanitizer_asm.h
@@ -67,6 +67,9 @@
 #define NO_EXEC_STACK_DIRECTIVE
 #endif
 
-#if defined(__x86_64__) || defined(__i386__)
+#if (defined(__x86_64__) || defined(__i386__)) && defined(__has_include) && __has_include(<cet.h>)
 #include <cet.h>
 #endif
+#ifndef _CET_ENDBR
+#define _CET_ENDBR
+#endif
diff --git a/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.cpp b/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.cpp
index 7fe9cd7..626777d 100644
--- a/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.cpp
@@ -11,16 +11,58 @@
 
 #include "sanitizer_chained_origin_depot.h"
 
+#include "sanitizer_persistent_allocator.h"
+#include "sanitizer_stackdepotbase.h"
+
 namespace __sanitizer {
 
-bool ChainedOriginDepot::ChainedOriginDepotNode::eq(
-    hash_type hash, const args_type &args) const {
-  return here_id == args.here_id && prev_id == args.prev_id;
-}
+namespace {
+struct ChainedOriginDepotDesc {
+  u32 here_id;
+  u32 prev_id;
+};
 
-uptr ChainedOriginDepot::ChainedOriginDepotNode::storage_size(
-    const args_type &args) {
-  return sizeof(ChainedOriginDepotNode);
+struct ChainedOriginDepotNode {
+  using hash_type = u32;
+  u32 link;
+  u32 here_id;
+  u32 prev_id;
+
+  typedef ChainedOriginDepotDesc args_type;
+
+  bool eq(hash_type hash, const args_type &args) const;
+
+  static uptr allocated() { return 0; }
+
+  static hash_type hash(const args_type &args);
+
+  static bool is_valid(const args_type &args);
+
+  void store(u32 id, const args_type &args, hash_type other_hash);
+
+  args_type load(u32 id) const;
+
+  struct Handle {
+    const ChainedOriginDepotNode *node_ = nullptr;
+    u32 id_ = 0;
+    Handle(const ChainedOriginDepotNode *node, u32 id) : node_(node), id_(id) {}
+    bool valid() const { return node_; }
+    u32 id() const { return id_; }
+    int here_id() const { return node_->here_id; }
+    int prev_id() const { return node_->prev_id; }
+  };
+
+  static Handle get_handle(u32 id);
+
+  typedef Handle handle_type;
+};
+
+}  // namespace
+
+static StackDepotBase<ChainedOriginDepotNode, 4, 20> depot;
+
+bool ChainedOriginDepotNode::eq(hash_type hash, const args_type &args) const {
+  return here_id == args.here_id && prev_id == args.prev_id;
 }
 
 /* This is murmur2 hash for the 64->32 bit case.
@@ -36,8 +78,8 @@ uptr ChainedOriginDepot::ChainedOriginDepotNode::storage_size(
    split, or one of two reserved values (-1) or (-2). Either case can
    dominate depending on the workload.
 */
-ChainedOriginDepot::ChainedOriginDepotNode::hash_type
-ChainedOriginDepot::ChainedOriginDepotNode::hash(const args_type &args) {
+ChainedOriginDepotNode::hash_type ChainedOriginDepotNode::hash(
+    const args_type &args) {
   const u32 m = 0x5bd1e995;
   const u32 seed = 0x9747b28c;
   const u32 r = 24;
@@ -62,26 +104,21 @@ ChainedOriginDepot::ChainedOriginDepotNode::hash(const args_type &args) {
   return h;
 }
 
-bool ChainedOriginDepot::ChainedOriginDepotNode::is_valid(
-    const args_type &args) {
-  return true;
-}
+bool ChainedOriginDepotNode::is_valid(const args_type &args) { return true; }
 
-void ChainedOriginDepot::ChainedOriginDepotNode::store(const args_type &args,
-                                                       hash_type other_hash) {
+void ChainedOriginDepotNode::store(u32 id, const args_type &args,
+                                   hash_type other_hash) {
   here_id = args.here_id;
   prev_id = args.prev_id;
 }
 
-ChainedOriginDepot::ChainedOriginDepotNode::args_type
-ChainedOriginDepot::ChainedOriginDepotNode::load() const {
+ChainedOriginDepotNode::args_type ChainedOriginDepotNode::load(u32 id) const {
   args_type ret = {here_id, prev_id};
   return ret;
 }
 
-ChainedOriginDepot::ChainedOriginDepotNode::Handle
-ChainedOriginDepot::ChainedOriginDepotNode::get_handle() {
-  return Handle(this);
+ChainedOriginDepotNode::Handle ChainedOriginDepotNode::get_handle(u32 id) {
+  return Handle(&depot.nodes[id], id);
 }
 
 ChainedOriginDepot::ChainedOriginDepot() {}
@@ -93,8 +130,7 @@ StackDepotStats ChainedOriginDepot::GetStats() const {
 bool ChainedOriginDepot::Put(u32 here_id, u32 prev_id, u32 *new_id) {
   ChainedOriginDepotDesc desc = {here_id, prev_id};
   bool inserted;
-  ChainedOriginDepotNode::Handle h = depot.Put(desc, &inserted);
-  *new_id = h.valid() ? h.id() : 0;
+  *new_id = depot.Put(desc, &inserted);
   return inserted;
 }
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.h b/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.h
index 73a10e1..2e800964 100644
--- a/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.h
+++ b/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.h
@@ -13,7 +13,6 @@
 #define SANITIZER_CHAINED_ORIGIN_DEPOT_H
 
 #include "sanitizer_common.h"
-#include "sanitizer_stackdepotbase.h"
 
 namespace __sanitizer {
 
@@ -37,49 +36,6 @@ class ChainedOriginDepot {
   void UnlockAll();
 
  private:
-  struct ChainedOriginDepotDesc {
-    u32 here_id;
-    u32 prev_id;
-  };
-
-  struct ChainedOriginDepotNode {
-    using hash_type = u32;
-    ChainedOriginDepotNode *link;
-    u32 id;
-    u32 here_id;
-    u32 prev_id;
-
-    typedef ChainedOriginDepotDesc args_type;
-
-    bool eq(hash_type hash, const args_type &args) const;
-
-    static uptr storage_size(const args_type &args);
-
-    static hash_type hash(const args_type &args);
-
-    static bool is_valid(const args_type &args);
-
-    void store(const args_type &args, hash_type other_hash);
-
-    args_type load() const;
-
-    struct Handle {
-      ChainedOriginDepotNode *node_;
-      Handle() : node_(nullptr) {}
-      explicit Handle(ChainedOriginDepotNode *node) : node_(node) {}
-      bool valid() { return node_; }
-      u32 id() { return node_->id; }
-      int here_id() { return node_->here_id; }
-      int prev_id() { return node_->prev_id; }
-    };
-
-    Handle get_handle();
-
-    typedef Handle handle_type;
-  };
-
-  StackDepotBase<ChainedOriginDepotNode, 4, 20> depot;
-
   ChainedOriginDepot(const ChainedOriginDepot &) = delete;
   void operator=(const ChainedOriginDepot &) = delete;
 };
diff --git a/libsanitizer/sanitizer_common/sanitizer_common.h b/libsanitizer/sanitizer_common/sanitizer_common.h
index 17c29c7..0651544 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common.h
+++ b/libsanitizer/sanitizer_common/sanitizer_common.h
@@ -192,12 +192,13 @@ class ReservedAddressRange {
 };
 
 typedef void (*fill_profile_f)(uptr start, uptr rss, bool file,
-                               /*out*/uptr *stats, uptr stats_size);
+                               /*out*/ uptr *stats);
 
 // Parse the contents of /proc/self/smaps and generate a memory profile.
-// |cb| is a tool-specific callback that fills the |stats| array containing
-// |stats_size| elements.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size);
+// |cb| is a tool-specific callback that fills the |stats| array.
+void GetMemoryProfile(fill_profile_f cb, uptr *stats);
+void ParseUnixMemoryProfile(fill_profile_f cb, uptr *stats, char *smaps,
+                            uptr smaps_len);
 
 // Simple low-level (mmap-based) allocator for internal use. Doesn't have
 // constructor, so all instances of LowLevelAllocator should be
@@ -371,7 +372,7 @@ void ReportErrorSummary(const char *error_type, const AddressInfo &info,
 void ReportErrorSummary(const char *error_type, const StackTrace *trace,
                         const char *alt_tool_name = nullptr);
 
-void ReportMmapWriteExec(int prot);
+void ReportMmapWriteExec(int prot, int mflags);
 
 // Math
 #if SANITIZER_WINDOWS && !defined(__clang__) && !defined(__GNUC__)
@@ -419,9 +420,7 @@ inline uptr LeastSignificantSetBitIndex(uptr x) {
   return up;
 }
 
-inline bool IsPowerOfTwo(uptr x) {
-  return (x & (x - 1)) == 0;
-}
+inline constexpr bool IsPowerOfTwo(uptr x) { return (x & (x - 1)) == 0; }
 
 inline uptr RoundUpToPowerOfTwo(uptr size) {
   CHECK(size);
@@ -433,16 +432,16 @@ inline uptr RoundUpToPowerOfTwo(uptr size) {
   return 1ULL << (up + 1);
 }
 
-inline uptr RoundUpTo(uptr size, uptr boundary) {
+inline constexpr uptr RoundUpTo(uptr size, uptr boundary) {
   RAW_CHECK(IsPowerOfTwo(boundary));
   return (size + boundary - 1) & ~(boundary - 1);
 }
 
-inline uptr RoundDownTo(uptr x, uptr boundary) {
+inline constexpr uptr RoundDownTo(uptr x, uptr boundary) {
   return x & ~(boundary - 1);
 }
 
-inline bool IsAligned(uptr a, uptr alignment) {
+inline constexpr bool IsAligned(uptr a, uptr alignment) {
   return (a & (alignment - 1)) == 0;
 }
 
@@ -722,12 +721,15 @@ void SortAndDedup(Container &v, Compare comp = {}) {
   v.resize(last + 1);
 }
 
+constexpr uptr kDefaultFileMaxSize = FIRST_32_SECOND_64(1 << 26, 1 << 28);
+
 // Opens the file 'file_name" and reads up to 'max_len' bytes.
 // The resulting buffer is mmaped and stored in '*buff'.
 // Returns true if file was successfully opened and read.
 bool ReadFileToVector(const char *file_name,
                       InternalMmapVectorNoCtor<char> *buff,
-                      uptr max_len = 1 << 26, error_t *errno_p = nullptr);
+                      uptr max_len = kDefaultFileMaxSize,
+                      error_t *errno_p = nullptr);
 
 // Opens the file 'file_name" and reads up to 'max_len' bytes.
 // This function is less I/O efficient than ReadFileToVector as it may reread
@@ -738,7 +740,7 @@ bool ReadFileToVector(const char *file_name,
 // The total number of read bytes is stored in '*read_len'.
 // Returns true if file was successfully opened and read.
 bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
-                      uptr *read_len, uptr max_len = 1 << 26,
+                      uptr *read_len, uptr max_len = kDefaultFileMaxSize,
                       error_t *errno_p = nullptr);
 
 // When adding a new architecture, don't forget to also update
diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc
index 9511a3b..abb38cc 100644
--- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc
@@ -2422,6 +2422,60 @@ INTERCEPTOR(int, glob64, const char *pattern, int flags,
 #define INIT_GLOB64
 #endif  // SANITIZER_INTERCEPT_GLOB64
 
+#if SANITIZER_INTERCEPT_POSIX_SPAWN
+
+template <class RealSpawnPtr>
+static int PosixSpawnImpl(void *ctx, RealSpawnPtr *real_posix_spawn, pid_t *pid,
+                          const char *file_or_path, const void *file_actions,
+                          const void *attrp, char *const argv[],
+                          char *const envp[]) {
+  COMMON_INTERCEPTOR_READ_RANGE(ctx, file_or_path,
+                                internal_strlen(file_or_path) + 1);
+  if (argv) {
+    for (char *const *s = argv; ; ++s) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(*s));
+      if (!*s) break;
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, *s, internal_strlen(*s) + 1);
+    }
+  }
+  if (envp) {
+    for (char *const *s = envp; ; ++s) {
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, s, sizeof(*s));
+      if (!*s) break;
+      COMMON_INTERCEPTOR_READ_RANGE(ctx, *s, internal_strlen(*s) + 1);
+    }
+  }
+  int res =
+      real_posix_spawn(pid, file_or_path, file_actions, attrp, argv, envp);
+  if (res == 0)
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pid, sizeof(*pid));
+  return res;
+}
+INTERCEPTOR(int, posix_spawn, pid_t *pid, const char *path,
+            const void *file_actions, const void *attrp, char *const argv[],
+            char *const envp[]) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, posix_spawn, pid, path, file_actions, attrp,
+                           argv, envp);
+  return PosixSpawnImpl(ctx, REAL(posix_spawn), pid, path, file_actions, attrp,
+                        argv, envp);
+}
+INTERCEPTOR(int, posix_spawnp, pid_t *pid, const char *file,
+            const void *file_actions, const void *attrp, char *const argv[],
+            char *const envp[]) {
+  void *ctx;
+  COMMON_INTERCEPTOR_ENTER(ctx, posix_spawnp, pid, file, file_actions, attrp,
+                           argv, envp);
+  return PosixSpawnImpl(ctx, REAL(posix_spawnp), pid, file, file_actions, attrp,
+                        argv, envp);
+}
+#  define INIT_POSIX_SPAWN                  \
+    COMMON_INTERCEPT_FUNCTION(posix_spawn); \
+    COMMON_INTERCEPT_FUNCTION(posix_spawnp);
+#else  // SANITIZER_INTERCEPT_POSIX_SPAWN
+#  define INIT_POSIX_SPAWN
+#endif  // SANITIZER_INTERCEPT_POSIX_SPAWN
+
 #if SANITIZER_INTERCEPT_WAIT
 // According to sys/wait.h, wait(), waitid(), waitpid() may have symbol version
 // suffixes on Darwin. See the declaration of INTERCEPTOR_WITH_SUFFIX for
@@ -2658,17 +2712,20 @@ INTERCEPTOR(int, getnameinfo, void *sockaddr, unsigned salen, char *host,
 #endif
 
 #if SANITIZER_INTERCEPT_GETSOCKNAME
-INTERCEPTOR(int, getsockname, int sock_fd, void *addr, int *addrlen) {
+INTERCEPTOR(int, getsockname, int sock_fd, void *addr, unsigned *addrlen) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, getsockname, sock_fd, addr, addrlen);
-  COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
-  int addrlen_in = *addrlen;
+  unsigned addr_sz;
+  if (addrlen) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
+    addr_sz = *addrlen;
+  }
   // FIXME: under ASan the call below may write to freed memory and corrupt
   // its metadata. See
   // https://github.com/google/sanitizers/issues/321.
   int res = REAL(getsockname)(sock_fd, addr, addrlen);
-  if (res == 0) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addrlen_in, *addrlen));
+  if (!res && addr && addrlen) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen));
   }
   return res;
 }
@@ -3173,13 +3230,17 @@ INTERCEPTOR(int, getpeername, int sockfd, void *addr, unsigned *addrlen) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, getpeername, sockfd, addr, addrlen);
   unsigned addr_sz;
-  if (addrlen) addr_sz = *addrlen;
+  if (addrlen) {
+    COMMON_INTERCEPTOR_READ_RANGE(ctx, addrlen, sizeof(*addrlen));
+    addr_sz = *addrlen;
+  }
   // FIXME: under ASan the call below may write to freed memory and corrupt
   // its metadata. See
   // https://github.com/google/sanitizers/issues/321.
   int res = REAL(getpeername)(sockfd, addr, addrlen);
-  if (!res && addr && addrlen)
+  if (!res && addr && addrlen) {
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen));
+  }
   return res;
 }
 #define INIT_GETPEERNAME COMMON_INTERCEPT_FUNCTION(getpeername);
@@ -7418,7 +7479,7 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags, int fd,
             OFF_T off) {
   void *ctx;
   if (common_flags()->detect_write_exec)
-    ReportMmapWriteExec(prot);
+    ReportMmapWriteExec(prot, flags);
   if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
     return (void *)internal_mmap(addr, sz, prot, flags, fd, off);
   COMMON_INTERCEPTOR_ENTER(ctx, mmap, addr, sz, prot, flags, fd, off);
@@ -7428,7 +7489,7 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags, int fd,
 INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) {
   void *ctx;
   if (common_flags()->detect_write_exec)
-    ReportMmapWriteExec(prot);
+    ReportMmapWriteExec(prot, 0);
   if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
     return (int)internal_mprotect(addr, sz, prot);
   COMMON_INTERCEPTOR_ENTER(ctx, mprotect, addr, sz, prot);
@@ -7447,7 +7508,7 @@ INTERCEPTOR(void *, mmap64, void *addr, SIZE_T sz, int prot, int flags, int fd,
             OFF64_T off) {
   void *ctx;
   if (common_flags()->detect_write_exec)
-    ReportMmapWriteExec(prot);
+    ReportMmapWriteExec(prot, flags);
   if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
     return (void *)internal_mmap(addr, sz, prot, flags, fd, off);
   COMMON_INTERCEPTOR_ENTER(ctx, mmap64, addr, sz, prot, flags, fd, off);
@@ -9033,10 +9094,10 @@ INTERCEPTOR(char *, MD2Data, const unsigned char *data, unsigned int len,
     return ret; \
   }
 
-SHA2_INTERCEPTORS(224, u32);
-SHA2_INTERCEPTORS(256, u32);
-SHA2_INTERCEPTORS(384, u64);
-SHA2_INTERCEPTORS(512, u64);
+SHA2_INTERCEPTORS(224, u32)
+SHA2_INTERCEPTORS(256, u32)
+SHA2_INTERCEPTORS(384, u64)
+SHA2_INTERCEPTORS(512, u64)
 
 #define INIT_SHA2_INTECEPTORS(LEN) \
   COMMON_INTERCEPT_FUNCTION(SHA##LEN##_Init); \
@@ -10229,6 +10290,7 @@ static void InitializeCommonInterceptors() {
   INIT_TIME;
   INIT_GLOB;
   INIT_GLOB64;
+  INIT_POSIX_SPAWN;
   INIT_WAIT;
   INIT_WAIT4;
   INIT_INET;
diff --git a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector.h b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector.h
index b80cff4..0749f63 100644
--- a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector.h
+++ b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector.h
@@ -293,7 +293,7 @@ class DeadlockDetector {
   }
 
   // Returns true iff dtls is empty (no locks are currently held) and we can
-  // add the node to the currently held locks w/o chanding the global state.
+  // add the node to the currently held locks w/o changing the global state.
   // This operation is thread-safe as it only touches the dtls.
   bool onFirstLock(DeadlockDetectorTLS<BV> *dtls, uptr node, u32 stk = 0) {
     if (!dtls->empty()) return false;
diff --git a/libsanitizer/sanitizer_common/sanitizer_flat_map.h b/libsanitizer/sanitizer_common/sanitizer_flat_map.h
new file mode 100644
index 0000000..05fb554
--- /dev/null
+++ b/libsanitizer/sanitizer_common/sanitizer_flat_map.h
@@ -0,0 +1,173 @@
+//===-- sanitizer_flat_map.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Part of the Sanitizer Allocator.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_FLAT_MAP_H
+#define SANITIZER_FLAT_MAP_H
+
+#include "sanitizer_atomic.h"
+#include "sanitizer_common.h"
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_local_address_space_view.h"
+#include "sanitizer_mutex.h"
+
+namespace __sanitizer {
+
+// Call these callbacks on mmap/munmap.
+struct NoOpMapUnmapCallback {
+  void OnMap(uptr p, uptr size) const {}
+  void OnUnmap(uptr p, uptr size) const {}
+};
+
+// Maps integers in rage [0, kSize) to values.
+template <typename T, u64 kSize,
+          typename AddressSpaceViewTy = LocalAddressSpaceView>
+class FlatMap {
+ public:
+  using AddressSpaceView = AddressSpaceViewTy;
+  void Init() { internal_memset(map_, 0, sizeof(map_)); }
+
+  constexpr uptr size() const { return kSize; }
+
+  bool contains(uptr idx) const {
+    CHECK_LT(idx, kSize);
+    return true;
+  }
+
+  T &operator[](uptr idx) {
+    DCHECK_LT(idx, kSize);
+    return map_[idx];
+  }
+
+  const T &operator[](uptr idx) const {
+    DCHECK_LT(idx, kSize);
+    return map_[idx];
+  }
+
+ private:
+  T map_[kSize];
+};
+
+// TwoLevelMap maps integers in range [0, kSize1*kSize2) to values.
+// It is implemented as a two-dimensional array: array of kSize1 pointers
+// to kSize2-byte arrays. The secondary arrays are mmaped on demand.
+// Each value is initially zero and can be set to something else only once.
+// Setting and getting values from multiple threads is safe w/o extra locking.
+template <typename T, u64 kSize1, u64 kSize2,
+          typename AddressSpaceViewTy = LocalAddressSpaceView,
+          class MapUnmapCallback = NoOpMapUnmapCallback>
+class TwoLevelMap {
+  static_assert(IsPowerOfTwo(kSize2), "Use a power of two for performance.");
+
+ public:
+  using AddressSpaceView = AddressSpaceViewTy;
+  void Init() {
+    mu_.Init();
+    internal_memset(map1_, 0, sizeof(map1_));
+  }
+
+  void TestOnlyUnmap() {
+    for (uptr i = 0; i < kSize1; i++) {
+      T *p = Get(i);
+      if (!p)
+        continue;
+      MapUnmapCallback().OnUnmap(reinterpret_cast<uptr>(p), MmapSize());
+      UnmapOrDie(p, kSize2);
+    }
+    Init();
+  }
+
+  uptr MemoryUsage() const {
+    uptr res = 0;
+    for (uptr i = 0; i < kSize1; i++) {
+      T *p = Get(i);
+      if (!p)
+        continue;
+      res += MmapSize();
+    }
+    return res;
+  }
+
+  constexpr uptr size() const { return kSize1 * kSize2; }
+  constexpr uptr size1() const { return kSize1; }
+  constexpr uptr size2() const { return kSize2; }
+
+  bool contains(uptr idx) const {
+    CHECK_LT(idx, kSize1 * kSize2);
+    return Get(idx / kSize2);
+  }
+
+  const T &operator[](uptr idx) const {
+    DCHECK_LT(idx, kSize1 * kSize2);
+    T *map2 = GetOrCreate(idx / kSize2);
+    return *AddressSpaceView::Load(&map2[idx % kSize2]);
+  }
+
+  T &operator[](uptr idx) {
+    DCHECK_LT(idx, kSize1 * kSize2);
+    T *map2 = GetOrCreate(idx / kSize2);
+    return *AddressSpaceView::LoadWritable(&map2[idx % kSize2]);
+  }
+
+ private:
+  constexpr uptr MmapSize() const {
+    return RoundUpTo(kSize2 * sizeof(T), GetPageSizeCached());
+  }
+
+  T *Get(uptr idx) const {
+    DCHECK_LT(idx, kSize1);
+    return reinterpret_cast<T *>(
+        atomic_load(&map1_[idx], memory_order_acquire));
+  }
+
+  T *GetOrCreate(uptr idx) const {
+    DCHECK_LT(idx, kSize1);
+    // This code needs to use memory_order_acquire/consume, but we use
+    // memory_order_relaxed for performance reasons (matters for arm64). We
+    // expect memory_order_relaxed to be effectively equivalent to
+    // memory_order_consume in this case for all relevant architectures: all
+    // dependent data is reachable only by dereferencing the resulting pointer.
+    // If relaxed load fails to see stored ptr, the code will fall back to
+    // Create() and reload the value again with locked mutex as a memory
+    // barrier.
+    T *res = reinterpret_cast<T *>(atomic_load_relaxed(&map1_[idx]));
+    if (LIKELY(res))
+      return res;
+    return Create(idx);
+  }
+
+  NOINLINE T *Create(uptr idx) const {
+    SpinMutexLock l(&mu_);
+    T *res = Get(idx);
+    if (!res) {
+      res = reinterpret_cast<T *>(MmapOrDie(MmapSize(), "TwoLevelMap"));
+      MapUnmapCallback().OnMap(reinterpret_cast<uptr>(res), kSize2);
+      atomic_store(&map1_[idx], reinterpret_cast<uptr>(res),
+                   memory_order_release);
+    }
+    return res;
+  }
+
+  mutable StaticSpinMutex mu_;
+  mutable atomic_uintptr_t map1_[kSize1];
+};
+
+template <u64 kSize, typename AddressSpaceViewTy = LocalAddressSpaceView>
+using FlatByteMap = FlatMap<u8, kSize, AddressSpaceViewTy>;
+
+template <u64 kSize1, u64 kSize2,
+          typename AddressSpaceViewTy = LocalAddressSpaceView,
+          class MapUnmapCallback = NoOpMapUnmapCallback>
+using TwoLevelByteMap =
+    TwoLevelMap<u8, kSize1, kSize2, AddressSpaceViewTy, MapUnmapCallback>;
+}  // namespace __sanitizer
+
+#endif
diff --git a/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp
index de4c985..c7b30d9 100644
--- a/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp
@@ -372,7 +372,7 @@ bool IsAccessibleMemoryRange(uptr beg, uptr size) {
 }
 
 // FIXME implement on this platform.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {}
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
 
 bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size,
                       uptr *read_len, uptr max_len, error_t *errno_p) {
diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.cpp b/libsanitizer/sanitizer_common/sanitizer_linux.cpp
index ea3e5bd..aa59d97 100644
--- a/libsanitizer/sanitizer_common/sanitizer_linux.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_linux.cpp
@@ -150,14 +150,34 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
 
 namespace __sanitizer {
 
-#if SANITIZER_LINUX && defined(__x86_64__)
-#include "sanitizer_syscall_linux_x86_64.inc"
-#elif SANITIZER_LINUX && SANITIZER_RISCV64
-#include "sanitizer_syscall_linux_riscv64.inc"
-#elif SANITIZER_LINUX && defined(__aarch64__)
-#include "sanitizer_syscall_linux_aarch64.inc"
-#elif SANITIZER_LINUX && defined(__arm__)
-#include "sanitizer_syscall_linux_arm.inc"
+void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *old) {
+  CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, old));
+}
+
+ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) {
+  __sanitizer_sigset_t set;
+  internal_sigfillset(&set);
+#  if SANITIZER_LINUX && !SANITIZER_ANDROID
+  // Glibc uses SIGSETXID signal during setuid call. If this signal is blocked
+  // on any thread, setuid call hangs.
+  // See test/sanitizer_common/TestCases/Linux/setuid.c.
+  internal_sigdelset(&set, 33);
+#  endif
+  SetSigProcMask(&set, &saved_);
+  if (copy)
+    internal_memcpy(copy, &saved_, sizeof(saved_));
+}
+
+ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); }
+
+#  if SANITIZER_LINUX && defined(__x86_64__)
+#    include "sanitizer_syscall_linux_x86_64.inc"
+#  elif SANITIZER_LINUX && SANITIZER_RISCV64
+#    include "sanitizer_syscall_linux_riscv64.inc"
+#  elif SANITIZER_LINUX && defined(__aarch64__)
+#    include "sanitizer_syscall_linux_aarch64.inc"
+#  elif SANITIZER_LINUX && defined(__arm__)
+#    include "sanitizer_syscall_linux_arm.inc"
 #  elif SANITIZER_LINUX && defined(__hexagon__)
 #    include "sanitizer_syscall_linux_hexagon.inc"
 #  else
@@ -1741,17 +1761,9 @@ HandleSignalMode GetHandleSignalMode(int signum) {
 #if !SANITIZER_GO
 void *internal_start_thread(void *(*func)(void *arg), void *arg) {
   // Start the thread with signals blocked, otherwise it can steal user signals.
-  __sanitizer_sigset_t set, old;
-  internal_sigfillset(&set);
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
-  // Glibc uses SIGSETXID signal during setuid call. If this signal is blocked
-  // on any thread, setuid call hangs (see test/tsan/setuid.c).
-  internal_sigdelset(&set, 33);
-#endif
-  internal_sigprocmask(SIG_SETMASK, &set, &old);
+  ScopedBlockSignals block(nullptr);
   void *th;
   real_pthread_create(&th, nullptr, func, arg);
-  internal_sigprocmask(SIG_SETMASK, &old, nullptr);
   return th;
 }
 
@@ -1773,7 +1785,7 @@ struct __sanitizer_esr_context {
 
 static bool Aarch64GetESR(ucontext_t *ucontext, u64 *esr) {
   static const u32 kEsrMagic = 0x45535201;
-  u8 *aux = ucontext->uc_mcontext.__reserved;
+  u8 *aux = reinterpret_cast<u8 *>(ucontext->uc_mcontext.__reserved);
   while (true) {
     _aarch64_ctx *ctx = (_aarch64_ctx *)aux;
     if (ctx->size == 0) break;
diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.h b/libsanitizer/sanitizer_common/sanitizer_linux.h
index 9a23fcf..6a235db 100644
--- a/libsanitizer/sanitizer_common/sanitizer_linux.h
+++ b/libsanitizer/sanitizer_common/sanitizer_linux.h
@@ -49,7 +49,17 @@ uptr internal_getdents(fd_t fd, struct linux_dirent *dirp, unsigned int count);
 uptr internal_sigaltstack(const void* ss, void* oss);
 uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
     __sanitizer_sigset_t *oldset);
-#if SANITIZER_GLIBC
+
+void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset);
+struct ScopedBlockSignals {
+  explicit ScopedBlockSignals(__sanitizer_sigset_t *copy);
+  ~ScopedBlockSignals();
+
+ private:
+  __sanitizer_sigset_t saved_;
+};
+
+#  if SANITIZER_GLIBC
 uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp);
 #endif
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.cpp b/libsanitizer/sanitizer_common/sanitizer_mac.cpp
index fa077a1..a61cde8 100644
--- a/libsanitizer/sanitizer_common/sanitizer_mac.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_mac.cpp
@@ -551,6 +551,9 @@ uptr TlsBaseAddr() {
   asm("movq %%gs:0,%0" : "=r"(segbase));
 #elif defined(__i386__)
   asm("movl %%gs:0,%0" : "=r"(segbase));
+#elif defined(__aarch64__)
+  asm("mrs %x0, tpidrro_el0" : "=r"(segbase));
+  segbase &= 0x07ul;  // clearing lower bits, cpu id stored there
 #endif
   return segbase;
 }
@@ -1319,7 +1322,7 @@ uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
 }
 
 // FIXME implement on this platform.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
 
 void SignalContext::DumpAllRegisters(void *context) {
   Report("Register values:\n");
diff --git a/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc b/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc
index e3b664f..764e2ce 100644
--- a/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc
+++ b/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc
@@ -23,6 +23,7 @@
 #include <sys/mman.h>
 
 #include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_dlsym.h"
 #include "sanitizer_common/sanitizer_mac.h"
 
 // Similar code is used in Google Perftools,
@@ -192,20 +193,15 @@ void *__sanitizer_mz_malloc(malloc_zone_t *zone, uptr size) {
   return p;
 }
 
+struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
+  static bool UseImpl() { return !COMMON_MALLOC_SANITIZER_INITIALIZED; }
+};
+
 extern "C"
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__sanitizer_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) {
-  if (UNLIKELY(!COMMON_MALLOC_SANITIZER_INITIALIZED)) {
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    const size_t kCallocPoolSize = 1024;
-    static uptr calloc_memory_for_dlsym[kCallocPoolSize];
-    static size_t allocated;
-    size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
-    void *mem = (void*)&calloc_memory_for_dlsym[allocated];
-    allocated += size_in_words;
-    CHECK(allocated < kCallocPoolSize);
-    return mem;
-  }
+  if (DlsymAlloc::Use())
+    return DlsymAlloc::Callocate(nmemb, size);
   COMMON_MALLOC_CALLOC(nmemb, size);
   return p;
 }
@@ -223,6 +219,8 @@ extern "C"
 SANITIZER_INTERFACE_ATTRIBUTE
 void __sanitizer_mz_free(malloc_zone_t *zone, void *ptr) {
   if (!ptr) return;
+  if (DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Free(ptr);
   COMMON_MALLOC_FREE(ptr);
 }
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_mutex.cpp b/libsanitizer/sanitizer_common/sanitizer_mutex.cpp
index 1c177d8..40fe566 100644
--- a/libsanitizer/sanitizer_common/sanitizer_mutex.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_mutex.cpp
@@ -174,7 +174,7 @@ struct InternalDeadlockDetector {
     if (max_idx != MutexInvalid && !mutex_can_lock[max_idx][type]) {
       Printf("%s: internal deadlock: can't lock %s under %s mutex\n", SanitizerToolName,
              mutex_meta[type].name, mutex_meta[max_idx].name);
-      PrintMutexPC(pc);
+      PrintMutexPC(locked[max_idx].pc);
       CHECK(0);
     }
     locked[type].seq = ++sequence;
diff --git a/libsanitizer/sanitizer_common/sanitizer_mutex.h b/libsanitizer/sanitizer_common/sanitizer_mutex.h
index 7479d35..5ec6efa 100644
--- a/libsanitizer/sanitizer_common/sanitizer_mutex.h
+++ b/libsanitizer/sanitizer_common/sanitizer_mutex.h
@@ -95,8 +95,11 @@ enum {
 
 // Go linker does not support THREADLOCAL variables,
 // so we can't use per-thread state.
+// Disable checked locks on Darwin. Although Darwin platforms support
+// THREADLOCAL variables they are not usable early on during process init when
+// `__sanitizer::Mutex` is used.
 #define SANITIZER_CHECK_DEADLOCKS \
-  (SANITIZER_DEBUG && !SANITIZER_GO && SANITIZER_SUPPORTS_THREADLOCAL)
+  (SANITIZER_DEBUG && !SANITIZER_GO && SANITIZER_SUPPORTS_THREADLOCAL && !SANITIZER_MAC)
 
 #if SANITIZER_CHECK_DEADLOCKS
 struct MutexMeta {
diff --git a/libsanitizer/sanitizer_common/sanitizer_persistent_allocator.cpp b/libsanitizer/sanitizer_common/sanitizer_persistent_allocator.cpp
deleted file mode 100644
index 1ca0375..0000000
--- a/libsanitizer/sanitizer_common/sanitizer_persistent_allocator.cpp
+++ /dev/null
@@ -1,18 +0,0 @@
-//===-- sanitizer_persistent_allocator.cpp ----------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is shared between AddressSanitizer and ThreadSanitizer
-// run-time libraries.
-//===----------------------------------------------------------------------===//
-#include "sanitizer_persistent_allocator.h"
-
-namespace __sanitizer {
-
-PersistentAllocator thePersistentAllocator;
-
-}  // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_persistent_allocator.h b/libsanitizer/sanitizer_common/sanitizer_persistent_allocator.h
index de4fb6e..e18b003 100644
--- a/libsanitizer/sanitizer_common/sanitizer_persistent_allocator.h
+++ b/libsanitizer/sanitizer_common/sanitizer_persistent_allocator.h
@@ -20,50 +20,89 @@
 
 namespace __sanitizer {
 
+template <typename T>
 class PersistentAllocator {
  public:
-  void *alloc(uptr size);
+  T *alloc(uptr count = 1);
+  uptr allocated() const { return atomic_load_relaxed(&mapped_size); }
+
+  void TestOnlyUnmap();
 
  private:
-  void *tryAlloc(uptr size);
-  StaticSpinMutex mtx;  // Protects alloc of new blocks for region allocator.
+  T *tryAlloc(uptr count);
+  T *refillAndAlloc(uptr count);
+  mutable StaticSpinMutex mtx;  // Protects alloc of new blocks.
   atomic_uintptr_t region_pos;  // Region allocator for Node's.
   atomic_uintptr_t region_end;
+  atomic_uintptr_t mapped_size;
+
+  struct BlockInfo {
+    const BlockInfo *next;
+    uptr ptr;
+    uptr size;
+  };
+  const BlockInfo *curr;
 };
 
-inline void *PersistentAllocator::tryAlloc(uptr size) {
+template <typename T>
+inline T *PersistentAllocator<T>::tryAlloc(uptr count) {
   // Optimisic lock-free allocation, essentially try to bump the region ptr.
   for (;;) {
     uptr cmp = atomic_load(&region_pos, memory_order_acquire);
     uptr end = atomic_load(&region_end, memory_order_acquire);
-    if (cmp == 0 || cmp + size > end) return nullptr;
+    uptr size = count * sizeof(T);
+    if (cmp == 0 || cmp + size > end)
+      return nullptr;
     if (atomic_compare_exchange_weak(&region_pos, &cmp, cmp + size,
                                      memory_order_acquire))
-      return (void *)cmp;
+      return reinterpret_cast<T *>(cmp);
   }
 }
 
-inline void *PersistentAllocator::alloc(uptr size) {
+template <typename T>
+inline T *PersistentAllocator<T>::alloc(uptr count) {
   // First, try to allocate optimisitically.
-  void *s = tryAlloc(size);
-  if (s) return s;
+  T *s = tryAlloc(count);
+  if (LIKELY(s))
+    return s;
+  return refillAndAlloc(count);
+}
+
+template <typename T>
+inline T *PersistentAllocator<T>::refillAndAlloc(uptr count) {
   // If failed, lock, retry and alloc new superblock.
   SpinMutexLock l(&mtx);
   for (;;) {
-    s = tryAlloc(size);
-    if (s) return s;
+    T *s = tryAlloc(count);
+    if (s)
+      return s;
     atomic_store(&region_pos, 0, memory_order_relaxed);
-    uptr allocsz = 64 * 1024;
-    if (allocsz < size) allocsz = size;
+    uptr size = count * sizeof(T) + sizeof(BlockInfo);
+    uptr allocsz = RoundUpTo(Max<uptr>(size, 64u * 1024u), GetPageSizeCached());
     uptr mem = (uptr)MmapOrDie(allocsz, "stack depot");
+    BlockInfo *new_block = (BlockInfo *)(mem + allocsz) - 1;
+    new_block->next = curr;
+    new_block->ptr = mem;
+    new_block->size = allocsz;
+    curr = new_block;
+
+    atomic_fetch_add(&mapped_size, allocsz, memory_order_relaxed);
+
+    allocsz -= sizeof(BlockInfo);
     atomic_store(&region_end, mem + allocsz, memory_order_release);
     atomic_store(&region_pos, mem, memory_order_release);
   }
 }
 
-extern PersistentAllocator thePersistentAllocator;
-inline void *PersistentAlloc(uptr sz) {
-  return thePersistentAllocator.alloc(sz);
+template <typename T>
+void PersistentAllocator<T>::TestOnlyUnmap() {
+  while (curr) {
+    uptr mem = curr->ptr;
+    uptr allocsz = curr->size;
+    curr = curr->next;
+    UnmapOrDie((void *)mem, allocsz);
+  }
+  internal_memset(this, 0, sizeof(*this));
 }
 
 } // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h
index 4e6efca..14610f2 100644
--- a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h
@@ -235,6 +235,7 @@
 #define SANITIZER_INTERCEPT_TIME SI_POSIX
 #define SANITIZER_INTERCEPT_GLOB (SI_GLIBC || SI_SOLARIS)
 #define SANITIZER_INTERCEPT_GLOB64 SI_GLIBC
+#define SANITIZER_INTERCEPT_POSIX_SPAWN SI_POSIX
 #define SANITIZER_INTERCEPT_WAIT SI_POSIX
 #define SANITIZER_INTERCEPT_INET SI_POSIX
 #define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM SI_POSIX
@@ -460,10 +461,13 @@
 #define SANITIZER_INTERCEPT_SEND_SENDTO SI_POSIX
 #define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE SI_LINUX
 
-#define SANITIZER_INTERCEPT_STAT \
-  (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS)
-#define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD || SI_FREEBSD)
-#define SANITIZER_INTERCEPT___XSTAT (!SANITIZER_INTERCEPT_STAT && SI_POSIX)
+#define SI_STAT_LINUX (SI_LINUX && __GLIBC_PREREQ(2, 33))
+#define SANITIZER_INTERCEPT_STAT                                        \
+  (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS ||     \
+   SI_STAT_LINUX)
+#define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD || SI_FREEBSD || SI_STAT_LINUX)
+#define SANITIZER_INTERCEPT___XSTAT \
+  ((!SANITIZER_INTERCEPT_STAT && SI_POSIX) || SI_STAT_LINUX)
 #define SANITIZER_INTERCEPT___XSTAT64 SI_LINUX_NOT_ANDROID
 #define SANITIZER_INTERCEPT___LXSTAT SANITIZER_INTERCEPT___XSTAT
 #define SANITIZER_INTERCEPT___LXSTAT64 SI_LINUX_NOT_ANDROID
@@ -477,7 +481,7 @@
   (SI_LINUX_NOT_ANDROID || SI_MAC || SI_FREEBSD || SI_NETBSD)
 
 #define SANITIZER_INTERCEPT_MMAP SI_POSIX
-#define SANITIZER_INTERCEPT_MMAP64 SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_MMAP64 SI_LINUX_NOT_ANDROID || SI_SOLARIS
 #define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO (SI_GLIBC || SI_ANDROID)
 #define SANITIZER_INTERCEPT_MEMALIGN (!SI_FREEBSD && !SI_MAC && !SI_NETBSD)
 #define SANITIZER_INTERCEPT___LIBC_MEMALIGN SI_GLIBC
diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps.h b/libsanitizer/sanitizer_common/sanitizer_procmaps.h
index a56640d..055af36 100644
--- a/libsanitizer/sanitizer_common/sanitizer_procmaps.h
+++ b/libsanitizer/sanitizer_common/sanitizer_procmaps.h
@@ -65,13 +65,23 @@ class MemoryMappedSegment {
   MemoryMappedSegmentData *data_;
 };
 
-class MemoryMappingLayout {
+class MemoryMappingLayoutBase {
+ public:
+  virtual bool Next(MemoryMappedSegment *segment) { UNIMPLEMENTED(); }
+  virtual bool Error() const { UNIMPLEMENTED(); };
+  virtual void Reset() { UNIMPLEMENTED(); }
+
+ protected:
+  ~MemoryMappingLayoutBase() {}
+};
+
+class MemoryMappingLayout final : public MemoryMappingLayoutBase {
  public:
   explicit MemoryMappingLayout(bool cache_enabled);
   ~MemoryMappingLayout();
-  bool Next(MemoryMappedSegment *segment);
-  bool Error() const;
-  void Reset();
+  virtual bool Next(MemoryMappedSegment *segment) override;
+  virtual bool Error() const override;
+  virtual void Reset() override;
   // In some cases, e.g. when running under a sandbox on Linux, ASan is unable
   // to obtain the memory mappings. It should fall back to pre-cached data
   // instead of aborting.
diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cpp b/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cpp
index 1b7dd46..eb351b0 100644
--- a/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_common.cpp
@@ -145,28 +145,44 @@ void MemoryMappingLayout::DumpListOfModules(
   }
 }
 
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) {
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {
   char *smaps = nullptr;
   uptr smaps_cap = 0;
   uptr smaps_len = 0;
   if (!ReadFileToBuffer("/proc/self/smaps", &smaps, &smaps_cap, &smaps_len))
     return;
+  ParseUnixMemoryProfile(cb, stats, smaps, smaps_len);
+  UnmapOrDie(smaps, smaps_cap);
+}
+
+void ParseUnixMemoryProfile(fill_profile_f cb, uptr *stats, char *smaps,
+                            uptr smaps_len) {
   uptr start = 0;
   bool file = false;
   const char *pos = smaps;
-  while (pos < smaps + smaps_len) {
+  char *end = smaps + smaps_len;
+  if (smaps_len < 2)
+    return;
+  // The following parsing can crash on almost every line
+  // in the case of malformed/truncated input.
+  // Fixing that is hard b/c e.g. ParseDecimal does not
+  // even accept end of the buffer and assumes well-formed input.
+  // So instead we patch end of the input a bit,
+  // it does not affect well-formed complete inputs.
+  *--end = 0;
+  *--end = '\n';
+  while (pos < end) {
     if (IsHex(pos[0])) {
       start = ParseHex(&pos);
       for (; *pos != '/' && *pos > '\n'; pos++) {}
       file = *pos == '/';
     } else if (internal_strncmp(pos, "Rss:", 4) == 0) {
-      while (!IsDecimal(*pos)) pos++;
+      while (pos < end && !IsDecimal(*pos)) pos++;
       uptr rss = ParseDecimal(&pos) * 1024;
-      cb(start, rss, file, stats, stats_size);
+      cb(start, rss, file, stats);
     }
     while (*pos++ != '\n') {}
   }
-  UnmapOrDie(smaps, smaps_cap);
 }
 
 } // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_solaris.cpp b/libsanitizer/sanitizer_common/sanitizer_procmaps_solaris.cpp
index bf813f23..e16c4e9 100644
--- a/libsanitizer/sanitizer_common/sanitizer_procmaps_solaris.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_solaris.cpp
@@ -55,7 +55,15 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
 
     internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s",
                       xmapentry->pr_mapname);
-    internal_readlink(proc_path, segment->filename, segment->filename_size);
+    ssize_t sz = internal_readlink(proc_path, segment->filename,
+                                   segment->filename_size - 1);
+
+    // If readlink failed, the map is anonymous.
+    if (sz == -1) {
+      segment->filename[0] = '\0';
+    } else if ((size_t)sz < segment->filename_size)
+      // readlink doesn't NUL-terminate.
+      segment->filename[sz] = '\0';
   }
 
   data_.current += sizeof(prxmap_t);
diff --git a/libsanitizer/sanitizer_common/sanitizer_stackdepot.cpp b/libsanitizer/sanitizer_common/sanitizer_stackdepot.cpp
index fc2ea2f..0285545 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stackdepot.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_stackdepot.cpp
@@ -14,93 +14,93 @@
 
 #include "sanitizer_common.h"
 #include "sanitizer_hash.h"
+#include "sanitizer_persistent_allocator.h"
 #include "sanitizer_stackdepotbase.h"
 
 namespace __sanitizer {
 
+static PersistentAllocator<uptr> traceAllocator;
+
 struct StackDepotNode {
-  using hash_type = u32;
-  StackDepotNode *link;
-  u32 id;
+  using hash_type = u64;
   hash_type stack_hash;
-  u32 size;
-  atomic_uint32_t tag_and_use_count;  // tag : 12 high bits; use_count : 20;
-  uptr stack[1];  // [size]
+  u32 link;
 
   static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20;
-  static const u32 kUseCountBits = 20;
-  static const u32 kMaxUseCount = 1 << kUseCountBits;
-  static const u32 kUseCountMask = (1 << kUseCountBits) - 1;
+  static const u32 kStackSizeBits = 16;
 
   typedef StackTrace args_type;
   bool eq(hash_type hash, const args_type &args) const {
-    u32 tag =
-        atomic_load(&tag_and_use_count, memory_order_relaxed) >> kUseCountBits;
-    if (stack_hash != hash || args.size != size || args.tag != tag)
-      return false;
-    uptr i = 0;
-    for (; i < size; i++) {
-      if (stack[i] != args.trace[i]) return false;
-    }
-    return true;
-  }
-  static uptr storage_size(const args_type &args) {
-    return sizeof(StackDepotNode) + (args.size - 1) * sizeof(uptr);
+    return hash == stack_hash;
   }
+  static uptr allocated();
   static hash_type hash(const args_type &args) {
-    MurMur2HashBuilder H(args.size * sizeof(uptr));
+    MurMur2Hash64Builder H(args.size * sizeof(uptr));
     for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]);
+    H.add(args.tag);
     return H.get();
   }
   static bool is_valid(const args_type &args) {
     return args.size > 0 && args.trace;
   }
-  void store(const args_type &args, hash_type hash) {
-    CHECK_EQ(args.tag & (~kUseCountMask >> kUseCountBits), args.tag);
-    atomic_store(&tag_and_use_count, args.tag << kUseCountBits,
-                 memory_order_relaxed);
-    stack_hash = hash;
-    size = args.size;
-    internal_memcpy(stack, args.trace, size * sizeof(uptr));
-  }
-  args_type load() const {
-    u32 tag =
-        atomic_load(&tag_and_use_count, memory_order_relaxed) >> kUseCountBits;
-    return args_type(&stack[0], size, tag);
-  }
-  StackDepotHandle get_handle() { return StackDepotHandle(this); }
+  void store(u32 id, const args_type &args, hash_type hash);
+  args_type load(u32 id) const;
+  static StackDepotHandle get_handle(u32 id);
 
   typedef StackDepotHandle handle_type;
 };
 
-COMPILER_CHECK(StackDepotNode::kMaxUseCount >= (u32)kStackDepotMaxUseCount);
-
-u32 StackDepotHandle::id() { return node_->id; }
-int StackDepotHandle::use_count() {
-  return atomic_load(&node_->tag_and_use_count, memory_order_relaxed) &
-         StackDepotNode::kUseCountMask;
-}
-void StackDepotHandle::inc_use_count_unsafe() {
-  u32 prev =
-      atomic_fetch_add(&node_->tag_and_use_count, 1, memory_order_relaxed) &
-      StackDepotNode::kUseCountMask;
-  CHECK_LT(prev + 1, StackDepotNode::kMaxUseCount);
-}
-
 // FIXME(dvyukov): this single reserved bit is used in TSan.
 typedef StackDepotBase<StackDepotNode, 1, StackDepotNode::kTabSizeLog>
     StackDepot;
 static StackDepot theDepot;
+// Keep rarely accessed stack traces out of frequently access nodes to improve
+// caching efficiency.
+static TwoLevelMap<uptr *, StackDepot::kNodesSize1, StackDepot::kNodesSize2>
+    tracePtrs;
+// Keep mutable data out of frequently access nodes to improve caching
+// efficiency.
+static TwoLevelMap<atomic_uint32_t, StackDepot::kNodesSize1,
+                   StackDepot::kNodesSize2>
+    useCounts;
+
+int StackDepotHandle::use_count() const {
+  return atomic_load_relaxed(&useCounts[id_]);
+}
 
-StackDepotStats StackDepotGetStats() { return theDepot.GetStats(); }
+void StackDepotHandle::inc_use_count_unsafe() {
+  atomic_fetch_add(&useCounts[id_], 1, memory_order_relaxed);
+}
+
+uptr StackDepotNode::allocated() {
+  return traceAllocator.allocated() + tracePtrs.MemoryUsage() +
+         useCounts.MemoryUsage();
+}
 
-u32 StackDepotPut(StackTrace stack) {
-  StackDepotHandle h = theDepot.Put(stack);
-  return h.valid() ? h.id() : 0;
+void StackDepotNode::store(u32 id, const args_type &args, hash_type hash) {
+  stack_hash = hash;
+  uptr *stack_trace = traceAllocator.alloc(args.size + 1);
+  CHECK_LT(args.size, 1 << kStackSizeBits);
+  *stack_trace = args.size + (args.tag << kStackSizeBits);
+  internal_memcpy(stack_trace + 1, args.trace, args.size * sizeof(uptr));
+  tracePtrs[id] = stack_trace;
 }
 
+StackDepotNode::args_type StackDepotNode::load(u32 id) const {
+  const uptr *stack_trace = tracePtrs[id];
+  if (!stack_trace)
+    return {};
+  uptr size = *stack_trace & ((1 << kStackSizeBits) - 1);
+  uptr tag = *stack_trace >> kStackSizeBits;
+  return args_type(stack_trace + 1, size, tag);
+}
+
+StackDepotStats StackDepotGetStats() { return theDepot.GetStats(); }
+
+u32 StackDepotPut(StackTrace stack) { return theDepot.Put(stack); }
+
 StackDepotHandle StackDepotPut_WithHandle(StackTrace stack) {
-  return theDepot.Put(stack);
+  return StackDepotNode::get_handle(theDepot.Put(stack));
 }
 
 StackTrace StackDepotGet(u32 id) {
@@ -121,37 +121,14 @@ void StackDepotPrintAll() {
 #endif
 }
 
-bool StackDepotReverseMap::IdDescPair::IdComparator(
-    const StackDepotReverseMap::IdDescPair &a,
-    const StackDepotReverseMap::IdDescPair &b) {
-  return a.id < b.id;
-}
-
-void StackDepotReverseMap::Init() const {
-  if (LIKELY(map_.capacity()))
-    return;
-  map_.reserve(StackDepotGetStats().n_uniq_ids + 100);
-  for (int idx = 0; idx < StackDepot::kTabSize; idx++) {
-    atomic_uintptr_t *p = &theDepot.tab[idx];
-    uptr v = atomic_load(p, memory_order_consume);
-    StackDepotNode *s = (StackDepotNode*)(v & ~1);
-    for (; s; s = s->link) {
-      IdDescPair pair = {s->id, s};
-      map_.push_back(pair);
-    }
-  }
-  Sort(map_.data(), map_.size(), &IdDescPair::IdComparator);
+StackDepotHandle StackDepotNode::get_handle(u32 id) {
+  return StackDepotHandle(&theDepot.nodes[id], id);
 }
 
-StackTrace StackDepotReverseMap::Get(u32 id) const {
-  Init();
-  if (!map_.size())
-    return StackTrace();
-  IdDescPair pair = {id, nullptr};
-  uptr idx = InternalLowerBound(map_, pair, IdDescPair::IdComparator);
-  if (idx > map_.size() || map_[idx].id != id)
-    return StackTrace();
-  return map_[idx].desc->load();
+void StackDepotTestOnlyUnmap() {
+  theDepot.TestOnlyUnmap();
+  tracePtrs.TestOnlyUnmap();
+  traceAllocator.TestOnlyUnmap();
 }
 
 } // namespace __sanitizer
diff --git a/libsanitizer/sanitizer_common/sanitizer_stackdepot.h b/libsanitizer/sanitizer_common/sanitizer_stackdepot.h
index 6f79fff..56d655d 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stackdepot.h
+++ b/libsanitizer/sanitizer_common/sanitizer_stackdepot.h
@@ -22,12 +22,12 @@ namespace __sanitizer {
 // StackDepot efficiently stores huge amounts of stack traces.
 struct StackDepotNode;
 struct StackDepotHandle {
-  StackDepotNode *node_;
-  StackDepotHandle() : node_(nullptr) {}
-  explicit StackDepotHandle(StackDepotNode *node) : node_(node) {}
-  bool valid() { return node_; }
-  u32 id();
-  int use_count();
+  StackDepotNode *node_ = nullptr;
+  u32 id_ = 0;
+  StackDepotHandle(StackDepotNode *node, u32 id) : node_(node), id_(id) {}
+  bool valid() const { return node_; }
+  u32 id() const { return id_; }
+  int use_count() const;
   void inc_use_count_unsafe();
 };
 
@@ -43,31 +43,7 @@ void StackDepotLockAll();
 void StackDepotUnlockAll();
 void StackDepotPrintAll();
 
-// Instantiating this class creates a snapshot of StackDepot which can be
-// efficiently queried with StackDepotGet(). You can use it concurrently with
-// StackDepot, but the snapshot is only guaranteed to contain those stack traces
-// which were stored before it was instantiated.
-class StackDepotReverseMap {
- public:
-  StackDepotReverseMap() = default;
-  StackTrace Get(u32 id) const;
-
- private:
-  struct IdDescPair {
-    u32 id;
-    StackDepotNode *desc;
-
-    static bool IdComparator(const IdDescPair &a, const IdDescPair &b);
-  };
-
-  void Init() const;
-
-  mutable InternalMmapVector<IdDescPair> map_;
-
-  // Disallow evil constructors.
-  StackDepotReverseMap(const StackDepotReverseMap&);
-  void operator=(const StackDepotReverseMap&);
-};
+void StackDepotTestOnlyUnmap();
 
 } // namespace __sanitizer
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_stackdepotbase.h b/libsanitizer/sanitizer_common/sanitizer_stackdepotbase.h
index 435f634..96d1ddc 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stackdepotbase.h
+++ b/libsanitizer/sanitizer_common/sanitizer_stackdepotbase.h
@@ -16,72 +16,87 @@
 #include <stdio.h>
 
 #include "sanitizer_atomic.h"
+#include "sanitizer_flat_map.h"
 #include "sanitizer_internal_defs.h"
 #include "sanitizer_mutex.h"
-#include "sanitizer_persistent_allocator.h"
 
 namespace __sanitizer {
 
 template <class Node, int kReservedBits, int kTabSizeLog>
 class StackDepotBase {
+  static constexpr u32 kIdSizeLog =
+      sizeof(u32) * 8 - Max(kReservedBits, 1 /* At least 1 bit for locking. */);
+  static constexpr u32 kNodesSize1Log = kIdSizeLog / 2;
+  static constexpr u32 kNodesSize2Log = kIdSizeLog - kNodesSize1Log;
+  static constexpr int kTabSize = 1 << kTabSizeLog;  // Hash table size.
+  static constexpr u32 kUnlockMask = (1ull << kIdSizeLog) - 1;
+  static constexpr u32 kLockMask = ~kUnlockMask;
+
  public:
   typedef typename Node::args_type args_type;
   typedef typename Node::handle_type handle_type;
   typedef typename Node::hash_type hash_type;
+
+  static constexpr u64 kNodesSize1 = 1ull << kNodesSize1Log;
+  static constexpr u64 kNodesSize2 = 1ull << kNodesSize2Log;
+
   // Maps stack trace to an unique id.
-  handle_type Put(args_type args, bool *inserted = nullptr);
+  u32 Put(args_type args, bool *inserted = nullptr);
   // Retrieves a stored stack trace by the id.
   args_type Get(u32 id);
 
-  StackDepotStats GetStats() const { return stats; }
+  StackDepotStats GetStats() const {
+    return {
+        atomic_load_relaxed(&n_uniq_ids),
+        nodes.MemoryUsage() + Node::allocated(),
+    };
+  }
 
   void LockAll();
   void UnlockAll();
   void PrintAll();
 
- private:
-  static Node *find(Node *s, args_type args, hash_type hash);
-  static Node *lock(atomic_uintptr_t *p);
-  static void unlock(atomic_uintptr_t *p, Node *s);
+  void TestOnlyUnmap() {
+    nodes.TestOnlyUnmap();
+    internal_memset(this, 0, sizeof(*this));
+  }
 
-  static const int kTabSize = 1 << kTabSizeLog;  // Hash table size.
-  static const int kPartBits = 8;
-  static const int kPartShift = sizeof(u32) * 8 - kPartBits - kReservedBits;
-  static const int kPartCount =
-      1 << kPartBits;  // Number of subparts in the table.
-  static const int kPartSize = kTabSize / kPartCount;
-  static const int kMaxId = 1 << kPartShift;
+ private:
+  friend Node;
+  u32 find(u32 s, args_type args, hash_type hash) const;
+  static u32 lock(atomic_uint32_t *p);
+  static void unlock(atomic_uint32_t *p, u32 s);
+  atomic_uint32_t tab[kTabSize];  // Hash table of Node's.
 
-  atomic_uintptr_t tab[kTabSize];   // Hash table of Node's.
-  atomic_uint32_t seq[kPartCount];  // Unique id generators.
+  atomic_uint32_t n_uniq_ids;
 
-  StackDepotStats stats;
+  TwoLevelMap<Node, kNodesSize1, kNodesSize2> nodes;
 
   friend class StackDepotReverseMap;
 };
 
 template <class Node, int kReservedBits, int kTabSizeLog>
-Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::find(Node *s,
-                                                             args_type args,
-                                                             hash_type hash) {
+u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::find(
+    u32 s, args_type args, hash_type hash) const {
   // Searches linked list s for the stack, returns its id.
-  for (; s; s = s->link) {
-    if (s->eq(hash, args)) {
+  for (; s;) {
+    const Node &node = nodes[s];
+    if (node.eq(hash, args))
       return s;
-    }
+    s = node.link;
   }
-  return nullptr;
+  return 0;
 }
 
 template <class Node, int kReservedBits, int kTabSizeLog>
-Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(
-    atomic_uintptr_t *p) {
+u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(atomic_uint32_t *p) {
   // Uses the pointer lsb as mutex.
   for (int i = 0;; i++) {
-    uptr cmp = atomic_load(p, memory_order_relaxed);
-    if ((cmp & 1) == 0 &&
-        atomic_compare_exchange_weak(p, &cmp, cmp | 1, memory_order_acquire))
-      return (Node *)cmp;
+    u32 cmp = atomic_load(p, memory_order_relaxed);
+    if ((cmp & kLockMask) == 0 &&
+        atomic_compare_exchange_weak(p, &cmp, cmp | kLockMask,
+                                     memory_order_acquire))
+      return cmp;
     if (i < 10)
       proc_yield(10);
     else
@@ -91,73 +106,57 @@ Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::lock(
 
 template <class Node, int kReservedBits, int kTabSizeLog>
 void StackDepotBase<Node, kReservedBits, kTabSizeLog>::unlock(
-    atomic_uintptr_t *p, Node *s) {
-  DCHECK_EQ((uptr)s & 1, 0);
-  atomic_store(p, (uptr)s, memory_order_release);
+    atomic_uint32_t *p, u32 s) {
+  DCHECK_EQ(s & kLockMask, 0);
+  atomic_store(p, s, memory_order_release);
 }
 
 template <class Node, int kReservedBits, int kTabSizeLog>
-typename StackDepotBase<Node, kReservedBits, kTabSizeLog>::handle_type
-StackDepotBase<Node, kReservedBits, kTabSizeLog>::Put(args_type args,
-                                                      bool *inserted) {
-  if (inserted) *inserted = false;
-  if (!Node::is_valid(args)) return handle_type();
+u32 StackDepotBase<Node, kReservedBits, kTabSizeLog>::Put(args_type args,
+                                                          bool *inserted) {
+  if (inserted)
+    *inserted = false;
+  if (!LIKELY(Node::is_valid(args)))
+    return 0;
   hash_type h = Node::hash(args);
-  atomic_uintptr_t *p = &tab[h % kTabSize];
-  uptr v = atomic_load(p, memory_order_consume);
-  Node *s = (Node *)(v & ~1);
+  atomic_uint32_t *p = &tab[h % kTabSize];
+  u32 v = atomic_load(p, memory_order_consume);
+  u32 s = v & kUnlockMask;
   // First, try to find the existing stack.
-  Node *node = find(s, args, h);
-  if (node) return node->get_handle();
+  u32 node = find(s, args, h);
+  if (LIKELY(node))
+    return node;
+
   // If failed, lock, retry and insert new.
-  Node *s2 = lock(p);
+  u32 s2 = lock(p);
   if (s2 != s) {
     node = find(s2, args, h);
     if (node) {
       unlock(p, s2);
-      return node->get_handle();
+      return node;
     }
   }
-  uptr part = (h % kTabSize) / kPartSize;
-  u32 id = atomic_fetch_add(&seq[part], 1, memory_order_relaxed) + 1;
-  stats.n_uniq_ids++;
-  CHECK_LT(id, kMaxId);
-  id |= part << kPartShift;
-  CHECK_NE(id, 0);
-  CHECK_EQ(id & (((u32)-1) >> kReservedBits), id);
-  uptr memsz = Node::storage_size(args);
-  s = (Node *)PersistentAlloc(memsz);
-  stats.allocated += memsz;
-  s->id = id;
-  s->store(args, h);
-  s->link = s2;
+  s = atomic_fetch_add(&n_uniq_ids, 1, memory_order_relaxed) + 1;
+  CHECK_EQ(s & kUnlockMask, s);
+  CHECK_EQ(s & (((u32)-1) >> kReservedBits), s);
+  Node &new_node = nodes[s];
+  new_node.store(s, args, h);
+  new_node.link = s2;
   unlock(p, s);
   if (inserted) *inserted = true;
-  return s->get_handle();
+  return s;
 }
 
 template <class Node, int kReservedBits, int kTabSizeLog>
 typename StackDepotBase<Node, kReservedBits, kTabSizeLog>::args_type
 StackDepotBase<Node, kReservedBits, kTabSizeLog>::Get(u32 id) {
-  if (id == 0) {
+  if (id == 0)
     return args_type();
-  }
   CHECK_EQ(id & (((u32)-1) >> kReservedBits), id);
-  // High kPartBits contain part id, so we need to scan at most kPartSize lists.
-  uptr part = id >> kPartShift;
-  for (int i = 0; i != kPartSize; i++) {
-    uptr idx = part * kPartSize + i;
-    CHECK_LT(idx, kTabSize);
-    atomic_uintptr_t *p = &tab[idx];
-    uptr v = atomic_load(p, memory_order_consume);
-    Node *s = (Node *)(v & ~1);
-    for (; s; s = s->link) {
-      if (s->id == id) {
-        return s->load();
-      }
-    }
-  }
-  return args_type();
+  if (!nodes.contains(id))
+    return args_type();
+  const Node &node = nodes[id];
+  return node.load(id);
 }
 
 template <class Node, int kReservedBits, int kTabSizeLog>
@@ -170,24 +169,23 @@ void StackDepotBase<Node, kReservedBits, kTabSizeLog>::LockAll() {
 template <class Node, int kReservedBits, int kTabSizeLog>
 void StackDepotBase<Node, kReservedBits, kTabSizeLog>::UnlockAll() {
   for (int i = 0; i < kTabSize; ++i) {
-    atomic_uintptr_t *p = &tab[i];
+    atomic_uint32_t *p = &tab[i];
     uptr s = atomic_load(p, memory_order_relaxed);
-    unlock(p, (Node *)(s & ~1UL));
+    unlock(p, s & kUnlockMask);
   }
 }
 
 template <class Node, int kReservedBits, int kTabSizeLog>
 void StackDepotBase<Node, kReservedBits, kTabSizeLog>::PrintAll() {
   for (int i = 0; i < kTabSize; ++i) {
-    atomic_uintptr_t *p = &tab[i];
-    lock(p);
-    uptr v = atomic_load(p, memory_order_relaxed);
-    Node *s = (Node *)(v & ~1UL);
-    for (; s; s = s->link) {
-      Printf("Stack for id %u:\n", s->id);
-      s->load().Print();
+    atomic_uint32_t *p = &tab[i];
+    u32 s = atomic_load(p, memory_order_consume) & kUnlockMask;
+    for (; s;) {
+      const Node &node = nodes[s];
+      Printf("Stack for id %u:\n", s);
+      node.load(s).Print();
+      s = node.link;
     }
-    unlock(p, s);
   }
 }
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp
index 15ea495..5a12422 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp
@@ -24,7 +24,7 @@ uptr StackTrace::GetNextInstructionPc(uptr pc) {
   return pc + 8;
 #elif defined(__powerpc__) || defined(__arm__) || defined(__aarch64__) || \
     defined(__hexagon__)
-  return pc + 4;
+  return STRIP_PAC_PC((void *)pc) + 4;
 #elif SANITIZER_RISCV64
   // Current check order is 4 -> 2 -> 6 -> 8
   u8 InsnByte = *(u8 *)(pc);
diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h
index ea330f3..11c6154 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h
+++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h
@@ -209,11 +209,11 @@ static inline bool IsValidFrame(uptr frame, uptr stack_top, uptr stack_bottom) {
 // StackTrace::GetCurrentPc() faster.
 #if defined(__x86_64__)
 #  define GET_CURRENT_PC()                \
-    ({                                    \
+    (__extension__({                      \
       uptr pc;                            \
       asm("lea 0(%%rip), %0" : "=r"(pc)); \
       pc;                                 \
-    })
+    }))
 #else
 #  define GET_CURRENT_PC() StackTrace::GetCurrentPc()
 #endif
diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cpp b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cpp
index ad638a8..c6356da 100644
--- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cpp
@@ -198,7 +198,8 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
       }
       break;
     default:
-      Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p, p);
+      Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
+             (void *)p);
       Die();
     }
   }
@@ -250,7 +251,7 @@ void RenderData(InternalScopedString *buffer, const char *format,
         break;
       default:
         Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p,
-               p);
+               (void *)p);
         Die();
     }
   }
diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp
index f330ed3..869c893 100644
--- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp
@@ -88,11 +88,17 @@ void ReportErrorSummary(const char *error_type, const StackTrace *stack,
 #endif
 }
 
-void ReportMmapWriteExec(int prot) {
+void ReportMmapWriteExec(int prot, int flags) {
 #if SANITIZER_POSIX && (!SANITIZER_GO && !SANITIZER_ANDROID)
-  if ((prot & (PROT_WRITE | PROT_EXEC)) != (PROT_WRITE | PROT_EXEC))
+  int pflags = (PROT_WRITE | PROT_EXEC);
+  if ((prot & pflags) != pflags)
     return;
 
+#  if SANITIZER_MAC && defined(MAP_JIT)
+  if ((flags & MAP_JIT) == MAP_JIT)
+    return;
+#  endif
+
   ScopedErrorReportLock l;
   SanitizerCommonDecorator d;
 
diff --git a/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cpp b/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cpp
index ce5e85d..b13e2dc 100644
--- a/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_tls_get_addr.cpp
@@ -44,7 +44,7 @@ static atomic_uintptr_t number_of_live_dtls;
 static const uptr kDestroyedThread = -1;
 
 static void DTLS_Deallocate(DTLS::DTVBlock *block) {
-  VReport(2, "__tls_get_addr: DTLS_Deallocate %p\n", block);
+  VReport(2, "__tls_get_addr: DTLS_Deallocate %p\n", (void *)block);
   UnmapOrDie(block, sizeof(DTLS::DTVBlock));
   atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed);
 }
@@ -66,12 +66,13 @@ static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) {
   }
   uptr num_live_dtls =
       atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed);
-  VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", &dtls, num_live_dtls);
+  VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", (void *)&dtls,
+          num_live_dtls);
   return new_dtv;
 }
 
 static DTLS::DTV *DTLS_Find(uptr id) {
-  VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", &dtls, id);
+  VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
   static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
   DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
   if (!cur)
@@ -82,7 +83,7 @@ static DTLS::DTV *DTLS_Find(uptr id) {
 
 void DTLS_Destroy() {
   if (!common_flags()->intercept_tls_get_addr) return;
-  VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", &dtls);
+  VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", (void *)&dtls);
   DTLS::DTVBlock *block = (DTLS::DTVBlock *)atomic_exchange(
       &dtls.dtv_block, kDestroyedThread, memory_order_release);
   while (block) {
@@ -120,7 +121,7 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
   VReport(2,
           "__tls_get_addr: %p {0x%zx,0x%zx} => %p; tls_beg: 0x%zx; sp: %p "
           "num_live_dtls %zd\n",
-          arg, arg->dso_id, arg->offset, res, tls_beg, &tls_beg,
+          (void *)arg, arg->dso_id, arg->offset, res, tls_beg, (void *)&tls_beg,
           atomic_load(&number_of_live_dtls, memory_order_relaxed));
   if (dtls.last_memalign_ptr == tls_beg) {
     tls_size = dtls.last_memalign_size;
diff --git a/libsanitizer/sanitizer_common/sanitizer_win.cpp b/libsanitizer/sanitizer_common/sanitizer_win.cpp
index 811aa49..c3607db 100644
--- a/libsanitizer/sanitizer_common/sanitizer_win.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_win.cpp
@@ -1113,7 +1113,7 @@ bool IsProcessRunning(pid_t pid) {
 int WaitForProcess(pid_t pid) { return -1; }
 
 // FIXME implement on this platform.
-void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { }
+void GetMemoryProfile(fill_profile_f cb, uptr *stats) {}
 
 void CheckNoDeepBind(const char *filename, int flag) {
   // Do nothing.
diff --git a/libsanitizer/tsan/Makefile.am b/libsanitizer/tsan/Makefile.am
index de47797..ae588a6 100644
--- a/libsanitizer/tsan/Makefile.am
+++ b/libsanitizer/tsan/Makefile.am
@@ -38,6 +38,7 @@ tsan_files = \
 	tsan_platform_windows.cpp \
 	tsan_report.cpp \
 	tsan_rtl.cpp \
+	tsan_rtl_access.cpp \
 	tsan_rtl_mutex.cpp \
 	tsan_rtl_proc.cpp \
 	tsan_rtl_report.cpp \
diff --git a/libsanitizer/tsan/Makefile.in b/libsanitizer/tsan/Makefile.in
index fa4a477..2781d2f 100644
--- a/libsanitizer/tsan/Makefile.in
+++ b/libsanitizer/tsan/Makefile.in
@@ -153,10 +153,10 @@ am__objects_1 = tsan_clock.lo tsan_debugging.lo tsan_external.lo \
 	tsan_md5.lo tsan_mman.lo tsan_mutexset.lo tsan_new_delete.lo \
 	tsan_platform_linux.lo tsan_platform_mac.lo \
 	tsan_platform_posix.lo tsan_platform_windows.lo tsan_report.lo \
-	tsan_rtl.lo tsan_rtl_mutex.lo tsan_rtl_proc.lo \
-	tsan_rtl_report.lo tsan_rtl_thread.lo tsan_stack_trace.lo \
-	tsan_suppressions.lo tsan_symbolize.lo tsan_sync.lo \
-	tsan_vector_clock.lo
+	tsan_rtl.lo tsan_rtl_access.lo tsan_rtl_mutex.lo \
+	tsan_rtl_proc.lo tsan_rtl_report.lo tsan_rtl_thread.lo \
+	tsan_stack_trace.lo tsan_suppressions.lo tsan_symbolize.lo \
+	tsan_sync.lo tsan_vector_clock.lo
 am_libtsan_la_OBJECTS = $(am__objects_1)
 libtsan_la_OBJECTS = $(am_libtsan_la_OBJECTS)
 AM_V_lt = $(am__v_lt_@AM_V@)
@@ -441,6 +441,7 @@ tsan_files = \
 	tsan_platform_windows.cpp \
 	tsan_report.cpp \
 	tsan_rtl.cpp \
+	tsan_rtl_access.cpp \
 	tsan_rtl_mutex.cpp \
 	tsan_rtl_proc.cpp \
 	tsan_rtl_report.cpp \
@@ -605,6 +606,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_report.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_aarch64.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_access.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_amd64.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_mips64.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_mutex.Plo@am__quote@
diff --git a/libsanitizer/tsan/tsan_interceptors.h b/libsanitizer/tsan/tsan_interceptors.h
index 89b2f99..61dbb81 100644
--- a/libsanitizer/tsan/tsan_interceptors.h
+++ b/libsanitizer/tsan/tsan_interceptors.h
@@ -38,19 +38,31 @@ inline bool in_symbolizer() {
 
 }  // namespace __tsan
 
-#define SCOPED_INTERCEPTOR_RAW(func, ...)      \
-  ThreadState *thr = cur_thread_init();        \
-  const uptr caller_pc = GET_CALLER_PC();      \
-  ScopedInterceptor si(thr, #func, caller_pc); \
-  const uptr pc = GET_CURRENT_PC();            \
-  (void)pc;
+#define SCOPED_INTERCEPTOR_RAW(func, ...)            \
+  ThreadState *thr = cur_thread_init();              \
+  ScopedInterceptor si(thr, #func, GET_CALLER_PC()); \
+  UNUSED const uptr pc = GET_CURRENT_PC();
+
+#ifdef __powerpc64__
+// Debugging of crashes on powerpc after commit:
+// c80604f7a3 ("tsan: remove real func check from interceptors")
+// Somehow replacing if with DCHECK leads to strange failures in:
+// SanitizerCommon-tsan-powerpc64le-Linux :: Linux/ptrace.cpp
+// https://lab.llvm.org/buildbot/#/builders/105
+// https://lab.llvm.org/buildbot/#/builders/121
+// https://lab.llvm.org/buildbot/#/builders/57
+#  define CHECK_REAL_FUNC(func)                                          \
+    if (REAL(func) == 0) {                                               \
+      Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
+      Die();                                                             \
+    }
+#else
+#  define CHECK_REAL_FUNC(func) DCHECK(REAL(func))
+#endif
 
 #define SCOPED_TSAN_INTERCEPTOR(func, ...)                                \
   SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__);                              \
-  if (REAL(func) == 0) {                                                  \
-    Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func);    \
-    Die();                                                                \
-  }                                                                       \
+  CHECK_REAL_FUNC(func);                                                  \
   if (!thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib) \
     return REAL(func)(__VA_ARGS__);
 
diff --git a/libsanitizer/tsan/tsan_interceptors_posix.cpp b/libsanitizer/tsan/tsan_interceptors_posix.cpp
index 617eda6..9a85ee0 100644
--- a/libsanitizer/tsan/tsan_interceptors_posix.cpp
+++ b/libsanitizer/tsan/tsan_interceptors_posix.cpp
@@ -2189,7 +2189,7 @@ void atfork_child() {
     return;
   ThreadState *thr = cur_thread();
   const uptr pc = StackTrace::GetCurrentPc();
-  ForkChildAfter(thr, pc);
+  ForkChildAfter(thr, pc, true);
   FdOnFork(thr, pc);
 }
 
@@ -2210,6 +2210,37 @@ TSAN_INTERCEPTOR(int, vfork, int fake) {
   return WRAP(fork)(fake);
 }
 
+#if SANITIZER_LINUX
+TSAN_INTERCEPTOR(int, clone, int (*fn)(void *), void *stack, int flags,
+                 void *arg, int *parent_tid, void *tls, pid_t *child_tid) {
+  SCOPED_INTERCEPTOR_RAW(clone, fn, stack, flags, arg, parent_tid, tls,
+                         child_tid);
+  struct Arg {
+    int (*fn)(void *);
+    void *arg;
+  };
+  auto wrapper = +[](void *p) -> int {
+    auto *thr = cur_thread();
+    uptr pc = GET_CURRENT_PC();
+    // Start the background thread for fork, but not for clone.
+    // For fork we did this always and it's known to work (or user code has
+    // adopted). But if we do this for the new clone interceptor some code
+    // (sandbox2) fails. So model we used to do for years and don't start the
+    // background thread after clone.
+    ForkChildAfter(thr, pc, false);
+    FdOnFork(thr, pc);
+    auto *arg = static_cast<Arg *>(p);
+    return arg->fn(arg->arg);
+  };
+  ForkBefore(thr, pc);
+  Arg arg_wrapper = {fn, arg};
+  int pid = REAL(clone)(wrapper, stack, flags, &arg_wrapper, parent_tid, tls,
+                        child_tid);
+  ForkParentAfter(thr, pc);
+  return pid;
+}
+#endif
+
 #if !SANITIZER_MAC && !SANITIZER_ANDROID
 typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size,
                                     void *data);
@@ -2544,7 +2575,7 @@ static void syscall_post_fork(uptr pc, int pid) {
   ThreadState *thr = cur_thread();
   if (pid == 0) {
     // child
-    ForkChildAfter(thr, pc);
+    ForkChildAfter(thr, pc, true);
     FdOnFork(thr, pc);
   } else if (pid > 0) {
     // parent
@@ -2841,6 +2872,9 @@ void InitializeInterceptors() {
 
   TSAN_INTERCEPT(fork);
   TSAN_INTERCEPT(vfork);
+#if SANITIZER_LINUX
+  TSAN_INTERCEPT(clone);
+#endif
 #if !SANITIZER_ANDROID
   TSAN_INTERCEPT(dl_iterate_phdr);
 #endif
diff --git a/libsanitizer/tsan/tsan_mutexset.cpp b/libsanitizer/tsan/tsan_mutexset.cpp
index efc0e41..7351796 100644
--- a/libsanitizer/tsan/tsan_mutexset.cpp
+++ b/libsanitizer/tsan/tsan_mutexset.cpp
@@ -10,6 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 #include "tsan_mutexset.h"
+
+#include "sanitizer_common/sanitizer_placement_new.h"
 #include "tsan_rtl.h"
 
 namespace __tsan {
@@ -124,4 +126,7 @@ MutexSet::Desc MutexSet::Get(uptr i) const {
   return descs_[i];
 }
 
+DynamicMutexSet::DynamicMutexSet() : ptr_(New<MutexSet>()) {}
+DynamicMutexSet::~DynamicMutexSet() { DestroyAndFree(ptr_); }
+
 }  // namespace __tsan
diff --git a/libsanitizer/tsan/tsan_mutexset.h b/libsanitizer/tsan/tsan_mutexset.h
index a448cee..93776a6 100644
--- a/libsanitizer/tsan/tsan_mutexset.h
+++ b/libsanitizer/tsan/tsan_mutexset.h
@@ -59,6 +59,24 @@ class MutexSet {
 #endif
 };
 
+// MutexSet is too large to live on stack.
+// DynamicMutexSet can be use used to create local MutexSet's.
+class DynamicMutexSet {
+ public:
+  DynamicMutexSet();
+  ~DynamicMutexSet();
+  MutexSet* operator->() { return ptr_; }
+  operator MutexSet*() { return ptr_; }
+  DynamicMutexSet(const DynamicMutexSet&) = delete;
+  DynamicMutexSet& operator=(const DynamicMutexSet&) = delete;
+
+ private:
+  MutexSet* ptr_;
+#if SANITIZER_GO
+  MutexSet set_;
+#endif
+};
+
 // Go does not have mutexes, so do not spend memory and time.
 // (Go sync.Mutex is actually a semaphore -- can be unlocked
 // in different goroutine).
@@ -71,6 +89,8 @@ void MutexSet::AddAddr(uptr addr, StackID stack_id, bool write) {}
 void MutexSet::DelAddr(uptr addr, bool destroy) {}
 uptr MutexSet::Size() const { return 0; }
 MutexSet::Desc MutexSet::Get(uptr i) const { return Desc(); }
+DynamicMutexSet::DynamicMutexSet() : ptr_(&set_) {}
+DynamicMutexSet::~DynamicMutexSet() {}
 #endif
 
 }  // namespace __tsan
diff --git a/libsanitizer/tsan/tsan_platform.h b/libsanitizer/tsan/tsan_platform.h
index fc27a56..7ff0aca 100644
--- a/libsanitizer/tsan/tsan_platform.h
+++ b/libsanitizer/tsan/tsan_platform.h
@@ -906,7 +906,7 @@ struct RestoreAddrImpl {
     // 3 bits of the compressed addr match that of the app range. If yes, we
     // assume that the compressed address come from that range and restore the
     // missing top bits to match the app range address.
-    static constexpr uptr ranges[] = {
+    const uptr ranges[] = {
         Mapping::kLoAppMemBeg,  Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
         Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
         Mapping::kHeapMemBeg,   Mapping::kHeapMemEnd,
diff --git a/libsanitizer/tsan/tsan_platform_linux.cpp b/libsanitizer/tsan/tsan_platform_linux.cpp
index 2fb753d..73ec148 100644
--- a/libsanitizer/tsan/tsan_platform_linux.cpp
+++ b/libsanitizer/tsan/tsan_platform_linux.cpp
@@ -100,8 +100,7 @@ enum {
   MemCount,
 };
 
-void FillProfileCallback(uptr p, uptr rss, bool file,
-                         uptr *mem, uptr stats_size) {
+void FillProfileCallback(uptr p, uptr rss, bool file, uptr *mem) {
   mem[MemTotal] += rss;
   if (p >= ShadowBeg() && p < ShadowEnd())
     mem[MemShadow] += rss;
@@ -122,7 +121,7 @@ void FillProfileCallback(uptr p, uptr rss, bool file,
 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) {
   uptr mem[MemCount];
   internal_memset(mem, 0, sizeof(mem));
-  GetMemoryProfile(FillProfileCallback, mem, MemCount);
+  GetMemoryProfile(FillProfileCallback, mem);
   auto meta = ctx->metamap.GetMemoryStats();
   StackDepotStats stacks = StackDepotGetStats();
   uptr nthread, nlive;
diff --git a/libsanitizer/tsan/tsan_platform_mac.cpp b/libsanitizer/tsan/tsan_platform_mac.cpp
index 388b383..3faa2d0 100644
--- a/libsanitizer/tsan/tsan_platform_mac.cpp
+++ b/libsanitizer/tsan/tsan_platform_mac.cpp
@@ -240,7 +240,7 @@ void InitializePlatformEarly() {
   uptr max_vm = GetMaxUserVirtualAddress() + 1;
   if (max_vm != HiAppMemEnd()) {
     Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
-           max_vm, HiAppMemEnd());
+           (void *)max_vm, (void *)HiAppMemEnd());
     Die();
   }
 #endif
diff --git a/libsanitizer/tsan/tsan_rtl.cpp b/libsanitizer/tsan/tsan_rtl.cpp
index 1c53f95..46dec04 100644
--- a/libsanitizer/tsan/tsan_rtl.cpp
+++ b/libsanitizer/tsan/tsan_rtl.cpp
@@ -506,7 +506,8 @@ void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
   ctx->thread_registry.Unlock();
 }
 
-void ForkChildAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
+void ForkChildAfter(ThreadState *thr, uptr pc,
+                    bool start_thread) NO_THREAD_SAFETY_ANALYSIS {
   thr->suppress_reports--;  // Enabled in ForkBefore.
   thr->ignore_interceptors--;
   ScopedErrorReportLock::Unlock();
@@ -518,7 +519,8 @@ void ForkChildAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS {
   VPrintf(1, "ThreadSanitizer: forked new process with pid %d,"
       " parent had %d threads\n", (int)internal_getpid(), (int)nthread);
   if (nthread == 1) {
-    StartBackgroundThread();
+    if (start_thread)
+      StartBackgroundThread();
   } else {
     // We've just forked a multi-threaded process. We cannot reasonably function
     // after that (some mutexes may be locked before fork). So just enable
@@ -567,123 +569,6 @@ StackID CurrentStackId(ThreadState *thr, uptr pc) {
 
 namespace v3 {
 
-ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState *thr, uptr pc,
-                                             uptr addr, uptr size,
-                                             AccessType typ) {
-  DCHECK(size == 1 || size == 2 || size == 4 || size == 8);
-  if (!kCollectHistory)
-    return true;
-  EventAccess *ev;
-  if (UNLIKELY(!TraceAcquire(thr, &ev)))
-    return false;
-  u64 size_log = size == 1 ? 0 : size == 2 ? 1 : size == 4 ? 2 : 3;
-  uptr pc_delta = pc - thr->trace_prev_pc + (1 << (EventAccess::kPCBits - 1));
-  thr->trace_prev_pc = pc;
-  if (LIKELY(pc_delta < (1 << EventAccess::kPCBits))) {
-    ev->is_access = 1;
-    ev->is_read = !!(typ & kAccessRead);
-    ev->is_atomic = !!(typ & kAccessAtomic);
-    ev->size_log = size_log;
-    ev->pc_delta = pc_delta;
-    DCHECK_EQ(ev->pc_delta, pc_delta);
-    ev->addr = CompressAddr(addr);
-    TraceRelease(thr, ev);
-    return true;
-  }
-  auto *evex = reinterpret_cast<EventAccessExt *>(ev);
-  evex->is_access = 0;
-  evex->is_func = 0;
-  evex->type = EventType::kAccessExt;
-  evex->is_read = !!(typ & kAccessRead);
-  evex->is_atomic = !!(typ & kAccessAtomic);
-  evex->size_log = size_log;
-  evex->addr = CompressAddr(addr);
-  evex->pc = pc;
-  TraceRelease(thr, evex);
-  return true;
-}
-
-ALWAYS_INLINE USED bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc,
-                                                  uptr addr, uptr size,
-                                                  AccessType typ) {
-  if (!kCollectHistory)
-    return true;
-  EventAccessRange *ev;
-  if (UNLIKELY(!TraceAcquire(thr, &ev)))
-    return false;
-  thr->trace_prev_pc = pc;
-  ev->is_access = 0;
-  ev->is_func = 0;
-  ev->type = EventType::kAccessRange;
-  ev->is_read = !!(typ & kAccessRead);
-  ev->is_free = !!(typ & kAccessFree);
-  ev->size_lo = size;
-  ev->pc = CompressAddr(pc);
-  ev->addr = CompressAddr(addr);
-  ev->size_hi = size >> EventAccessRange::kSizeLoBits;
-  TraceRelease(thr, ev);
-  return true;
-}
-
-void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
-                            AccessType typ) {
-  if (LIKELY(TryTraceMemoryAccessRange(thr, pc, addr, size, typ)))
-    return;
-  TraceSwitchPart(thr);
-  UNUSED bool res = TryTraceMemoryAccessRange(thr, pc, addr, size, typ);
-  DCHECK(res);
-}
-
-void TraceFunc(ThreadState *thr, uptr pc) {
-  if (LIKELY(TryTraceFunc(thr, pc)))
-    return;
-  TraceSwitchPart(thr);
-  UNUSED bool res = TryTraceFunc(thr, pc);
-  DCHECK(res);
-}
-
-void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
-                    StackID stk) {
-  DCHECK(type == EventType::kLock || type == EventType::kRLock);
-  if (!kCollectHistory)
-    return;
-  EventLock ev;
-  ev.is_access = 0;
-  ev.is_func = 0;
-  ev.type = type;
-  ev.pc = CompressAddr(pc);
-  ev.stack_lo = stk;
-  ev.stack_hi = stk >> EventLock::kStackIDLoBits;
-  ev._ = 0;
-  ev.addr = CompressAddr(addr);
-  TraceEvent(thr, ev);
-}
-
-void TraceMutexUnlock(ThreadState *thr, uptr addr) {
-  if (!kCollectHistory)
-    return;
-  EventUnlock ev;
-  ev.is_access = 0;
-  ev.is_func = 0;
-  ev.type = EventType::kUnlock;
-  ev._ = 0;
-  ev.addr = CompressAddr(addr);
-  TraceEvent(thr, ev);
-}
-
-void TraceTime(ThreadState *thr) {
-  if (!kCollectHistory)
-    return;
-  EventTime ev;
-  ev.is_access = 0;
-  ev.is_func = 0;
-  ev.type = EventType::kTime;
-  ev.sid = static_cast<u64>(thr->sid);
-  ev.epoch = static_cast<u64>(thr->epoch);
-  ev._ = 0;
-  TraceEvent(thr, ev);
-}
-
 NOINLINE
 void TraceSwitchPart(ThreadState *thr) {
   Trace *trace = &thr->tctx->trace;
@@ -789,427 +674,6 @@ extern "C" void __tsan_report_race() {
 }
 #endif
 
-ALWAYS_INLINE
-Shadow LoadShadow(u64 *p) {
-  u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed);
-  return Shadow(raw);
-}
-
-ALWAYS_INLINE
-void StoreShadow(u64 *sp, u64 s) {
-  atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed);
-}
-
-ALWAYS_INLINE
-void StoreIfNotYetStored(u64 *sp, u64 *s) {
-  StoreShadow(sp, *s);
-  *s = 0;
-}
-
-ALWAYS_INLINE
-void HandleRace(ThreadState *thr, u64 *shadow_mem,
-                              Shadow cur, Shadow old) {
-  thr->racy_state[0] = cur.raw();
-  thr->racy_state[1] = old.raw();
-  thr->racy_shadow_addr = shadow_mem;
-#if !SANITIZER_GO
-  HACKY_CALL(__tsan_report_race);
-#else
-  ReportRace(thr);
-#endif
-}
-
-static inline bool HappensBefore(Shadow old, ThreadState *thr) {
-  return thr->clock.get(old.TidWithIgnore()) >= old.epoch();
-}
-
-ALWAYS_INLINE
-void MemoryAccessImpl1(ThreadState *thr, uptr addr,
-    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
-    u64 *shadow_mem, Shadow cur) {
-
-  // This potentially can live in an MMX/SSE scratch register.
-  // The required intrinsics are:
-  // __m128i _mm_move_epi64(__m128i*);
-  // _mm_storel_epi64(u64*, __m128i);
-  u64 store_word = cur.raw();
-  bool stored = false;
-
-  // scan all the shadow values and dispatch to 4 categories:
-  // same, replace, candidate and race (see comments below).
-  // we consider only 3 cases regarding access sizes:
-  // equal, intersect and not intersect. initially I considered
-  // larger and smaller as well, it allowed to replace some
-  // 'candidates' with 'same' or 'replace', but I think
-  // it's just not worth it (performance- and complexity-wise).
-
-  Shadow old(0);
-
-  // It release mode we manually unroll the loop,
-  // because empirically gcc generates better code this way.
-  // However, we can't afford unrolling in debug mode, because the function
-  // consumes almost 4K of stack. Gtest gives only 4K of stack to death test
-  // threads, which is not enough for the unrolled loop.
-#if SANITIZER_DEBUG
-  for (int idx = 0; idx < 4; idx++) {
-#  include "tsan_update_shadow_word.inc"
-  }
-#else
-  int idx = 0;
-#  include "tsan_update_shadow_word.inc"
-  idx = 1;
-  if (stored) {
-#  include "tsan_update_shadow_word.inc"
-  } else {
-#  include "tsan_update_shadow_word.inc"
-  }
-  idx = 2;
-  if (stored) {
-#  include "tsan_update_shadow_word.inc"
-  } else {
-#  include "tsan_update_shadow_word.inc"
-  }
-  idx = 3;
-  if (stored) {
-#  include "tsan_update_shadow_word.inc"
-  } else {
-#  include "tsan_update_shadow_word.inc"
-  }
-#endif
-
-  // we did not find any races and had already stored
-  // the current access info, so we are done
-  if (LIKELY(stored))
-    return;
-  // choose a random candidate slot and replace it
-  StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
-  return;
- RACE:
-  HandleRace(thr, shadow_mem, cur, old);
-  return;
-}
-
-void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
-                           AccessType typ) {
-  DCHECK(!(typ & kAccessAtomic));
-  const bool kAccessIsWrite = !(typ & kAccessRead);
-  const bool kIsAtomic = false;
-  while (size) {
-    int size1 = 1;
-    int kAccessSizeLog = kSizeLog1;
-    if (size >= 8 && (addr & ~7) == ((addr + 7) & ~7)) {
-      size1 = 8;
-      kAccessSizeLog = kSizeLog8;
-    } else if (size >= 4 && (addr & ~7) == ((addr + 3) & ~7)) {
-      size1 = 4;
-      kAccessSizeLog = kSizeLog4;
-    } else if (size >= 2 && (addr & ~7) == ((addr + 1) & ~7)) {
-      size1 = 2;
-      kAccessSizeLog = kSizeLog2;
-    }
-    MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic);
-    addr += size1;
-    size -= size1;
-  }
-}
-
-ALWAYS_INLINE
-bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
-  Shadow cur(a);
-  for (uptr i = 0; i < kShadowCnt; i++) {
-    Shadow old(LoadShadow(&s[i]));
-    if (Shadow::Addr0AndSizeAreEqual(cur, old) &&
-        old.TidWithIgnore() == cur.TidWithIgnore() &&
-        old.epoch() > sync_epoch &&
-        old.IsAtomic() == cur.IsAtomic() &&
-        old.IsRead() <= cur.IsRead())
-      return true;
-  }
-  return false;
-}
-
-#if TSAN_VECTORIZE
-#  define SHUF(v0, v1, i0, i1, i2, i3)                    \
-    _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(v0), \
-                                    _mm_castsi128_ps(v1), \
-                                    (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64))
-ALWAYS_INLINE
-bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
-  // This is an optimized version of ContainsSameAccessSlow.
-  // load current access into access[0:63]
-  const m128 access     = _mm_cvtsi64_si128(a);
-  // duplicate high part of access in addr0:
-  // addr0[0:31]        = access[32:63]
-  // addr0[32:63]       = access[32:63]
-  // addr0[64:95]       = access[32:63]
-  // addr0[96:127]      = access[32:63]
-  const m128 addr0      = SHUF(access, access, 1, 1, 1, 1);
-  // load 4 shadow slots
-  const m128 shadow0    = _mm_load_si128((__m128i*)s);
-  const m128 shadow1    = _mm_load_si128((__m128i*)s + 1);
-  // load high parts of 4 shadow slots into addr_vect:
-  // addr_vect[0:31]    = shadow0[32:63]
-  // addr_vect[32:63]   = shadow0[96:127]
-  // addr_vect[64:95]   = shadow1[32:63]
-  // addr_vect[96:127]  = shadow1[96:127]
-  m128 addr_vect        = SHUF(shadow0, shadow1, 1, 3, 1, 3);
-  if (!is_write) {
-    // set IsRead bit in addr_vect
-    const m128 rw_mask1 = _mm_cvtsi64_si128(1<<15);
-    const m128 rw_mask  = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0);
-    addr_vect           = _mm_or_si128(addr_vect, rw_mask);
-  }
-  // addr0 == addr_vect?
-  const m128 addr_res   = _mm_cmpeq_epi32(addr0, addr_vect);
-  // epoch1[0:63]       = sync_epoch
-  const m128 epoch1     = _mm_cvtsi64_si128(sync_epoch);
-  // epoch[0:31]        = sync_epoch[0:31]
-  // epoch[32:63]       = sync_epoch[0:31]
-  // epoch[64:95]       = sync_epoch[0:31]
-  // epoch[96:127]      = sync_epoch[0:31]
-  const m128 epoch      = SHUF(epoch1, epoch1, 0, 0, 0, 0);
-  // load low parts of shadow cell epochs into epoch_vect:
-  // epoch_vect[0:31]   = shadow0[0:31]
-  // epoch_vect[32:63]  = shadow0[64:95]
-  // epoch_vect[64:95]  = shadow1[0:31]
-  // epoch_vect[96:127] = shadow1[64:95]
-  const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2);
-  // epoch_vect >= sync_epoch?
-  const m128 epoch_res  = _mm_cmpgt_epi32(epoch_vect, epoch);
-  // addr_res & epoch_res
-  const m128 res        = _mm_and_si128(addr_res, epoch_res);
-  // mask[0] = res[7]
-  // mask[1] = res[15]
-  // ...
-  // mask[15] = res[127]
-  const int mask        = _mm_movemask_epi8(res);
-  return mask != 0;
-}
-#endif
-
-ALWAYS_INLINE
-bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
-#if TSAN_VECTORIZE
-  bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write);
-  // NOTE: this check can fail if the shadow is concurrently mutated
-  // by other threads. But it still can be useful if you modify
-  // ContainsSameAccessFast and want to ensure that it's not completely broken.
-  // DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write));
-  return res;
-#else
-  return ContainsSameAccessSlow(s, a, sync_epoch, is_write);
-#endif
-}
-
-ALWAYS_INLINE USED
-void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
-    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) {
-  RawShadow *shadow_mem = MemToShadow(addr);
-  DPrintf2("#%d: MemoryAccess: @%p %p size=%d"
-      " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
-      (int)thr->fast_state.tid(), (void*)pc, (void*)addr,
-      (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
-      (uptr)shadow_mem[0], (uptr)shadow_mem[1],
-      (uptr)shadow_mem[2], (uptr)shadow_mem[3]);
-#if SANITIZER_DEBUG
-  if (!IsAppMem(addr)) {
-    Printf("Access to non app mem %zx\n", addr);
-    DCHECK(IsAppMem(addr));
-  }
-  if (!IsShadowMem(shadow_mem)) {
-    Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
-    DCHECK(IsShadowMem(shadow_mem));
-  }
-#endif
-
-  if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
-    // Access to .rodata section, no races here.
-    // Measurements show that it can be 10-20% of all memory accesses.
-    return;
-  }
-
-  FastState fast_state = thr->fast_state;
-  if (UNLIKELY(fast_state.GetIgnoreBit())) {
-    return;
-  }
-
-  Shadow cur(fast_state);
-  cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog);
-  cur.SetWrite(kAccessIsWrite);
-  cur.SetAtomic(kIsAtomic);
-
-  if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
-      thr->fast_synch_epoch, kAccessIsWrite))) {
-    return;
-  }
-
-  if (kCollectHistory) {
-    fast_state.IncrementEpoch();
-    thr->fast_state = fast_state;
-    TraceAddEvent(thr, fast_state, EventTypeMop, pc);
-    cur.IncrementEpoch();
-  }
-
-  MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
-      shadow_mem, cur);
-}
-
-// Called by MemoryAccessRange in tsan_rtl_thread.cpp
-ALWAYS_INLINE USED
-void MemoryAccessImpl(ThreadState *thr, uptr addr,
-    int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic,
-    u64 *shadow_mem, Shadow cur) {
-  if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(),
-      thr->fast_synch_epoch, kAccessIsWrite))) {
-    return;
-  }
-
-  MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
-      shadow_mem, cur);
-}
-
-static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
-                           u64 val) {
-  (void)thr;
-  (void)pc;
-  if (size == 0)
-    return;
-  // FIXME: fix me.
-  uptr offset = addr % kShadowCell;
-  if (offset) {
-    offset = kShadowCell - offset;
-    if (size <= offset)
-      return;
-    addr += offset;
-    size -= offset;
-  }
-  DCHECK_EQ(addr % 8, 0);
-  // If a user passes some insane arguments (memset(0)),
-  // let it just crash as usual.
-  if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
-    return;
-  // Don't want to touch lots of shadow memory.
-  // If a program maps 10MB stack, there is no need reset the whole range.
-  size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
-  // UnmapOrDie/MmapFixedNoReserve does not work on Windows.
-  if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) {
-    RawShadow *p = MemToShadow(addr);
-    CHECK(IsShadowMem(p));
-    CHECK(IsShadowMem(p + size * kShadowCnt / kShadowCell - 1));
-    // FIXME: may overwrite a part outside the region
-    for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) {
-      p[i++] = val;
-      for (uptr j = 1; j < kShadowCnt; j++)
-        p[i++] = 0;
-    }
-  } else {
-    // The region is big, reset only beginning and end.
-    const uptr kPageSize = GetPageSizeCached();
-    RawShadow *begin = MemToShadow(addr);
-    RawShadow *end = begin + size / kShadowCell * kShadowCnt;
-    RawShadow *p = begin;
-    // Set at least first kPageSize/2 to page boundary.
-    while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) {
-      *p++ = val;
-      for (uptr j = 1; j < kShadowCnt; j++)
-        *p++ = 0;
-    }
-    // Reset middle part.
-    RawShadow *p1 = p;
-    p = RoundDown(end, kPageSize);
-    if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1))
-      Die();
-    // Set the ending.
-    while (p < end) {
-      *p++ = val;
-      for (uptr j = 1; j < kShadowCnt; j++)
-        *p++ = 0;
-    }
-  }
-}
-
-void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) {
-  MemoryRangeSet(thr, pc, addr, size, 0);
-}
-
-void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) {
-  // Processing more than 1k (4k of shadow) is expensive,
-  // can cause excessive memory consumption (user does not necessary touch
-  // the whole range) and most likely unnecessary.
-  if (size > 1024)
-    size = 1024;
-  CHECK_EQ(thr->is_freeing, false);
-  thr->is_freeing = true;
-  MemoryAccessRange(thr, pc, addr, size, true);
-  thr->is_freeing = false;
-  if (kCollectHistory) {
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
-  }
-  Shadow s(thr->fast_state);
-  s.ClearIgnoreBit();
-  s.MarkAsFreed();
-  s.SetWrite(true);
-  s.SetAddr0AndSizeLog(0, 3);
-  MemoryRangeSet(thr, pc, addr, size, s.raw());
-}
-
-void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {
-  if (kCollectHistory) {
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
-  }
-  Shadow s(thr->fast_state);
-  s.ClearIgnoreBit();
-  s.SetWrite(true);
-  s.SetAddr0AndSizeLog(0, 3);
-  MemoryRangeSet(thr, pc, addr, size, s.raw());
-}
-
-void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
-                                         uptr size) {
-  if (thr->ignore_reads_and_writes == 0)
-    MemoryRangeImitateWrite(thr, pc, addr, size);
-  else
-    MemoryResetRange(thr, pc, addr, size);
-}
-
-ALWAYS_INLINE USED
-void FuncEntry(ThreadState *thr, uptr pc) {
-  DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc);
-  if (kCollectHistory) {
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
-  }
-
-  // Shadow stack maintenance can be replaced with
-  // stack unwinding during trace switch (which presumably must be faster).
-  DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
-#if !SANITIZER_GO
-  DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
-#else
-  if (thr->shadow_stack_pos == thr->shadow_stack_end)
-    GrowShadowStack(thr);
-#endif
-  thr->shadow_stack_pos[0] = pc;
-  thr->shadow_stack_pos++;
-}
-
-ALWAYS_INLINE USED
-void FuncExit(ThreadState *thr) {
-  DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
-  if (kCollectHistory) {
-    thr->fast_state.IncrementEpoch();
-    TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
-  }
-
-  DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
-#if !SANITIZER_GO
-  DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
-#endif
-  thr->shadow_stack_pos--;
-}
-
 void ThreadIgnoreBegin(ThreadState *thr, uptr pc) {
   DPrintf("#%d: ThreadIgnoreBegin\n", thr->tid);
   thr->ignore_reads_and_writes++;
@@ -1293,8 +757,3 @@ MutexMeta mutex_meta[] = {
 void PrintMutexPC(uptr pc) { StackTrace(&pc, 1).Print(); }
 }  // namespace __sanitizer
 #endif
-
-#if !SANITIZER_GO
-// Must be included in this file to make sure everything is inlined.
-#  include "tsan_interface.inc"
-#endif
diff --git a/libsanitizer/tsan/tsan_rtl.h b/libsanitizer/tsan/tsan_rtl.h
index 669c4ac..eab8370 100644
--- a/libsanitizer/tsan/tsan_rtl.h
+++ b/libsanitizer/tsan/tsan_rtl.h
@@ -440,7 +440,7 @@ void InitializeDynamicAnnotations();
 
 void ForkBefore(ThreadState *thr, uptr pc);
 void ForkParentAfter(ThreadState *thr, uptr pc);
-void ForkChildAfter(ThreadState *thr, uptr pc);
+void ForkChildAfter(ThreadState *thr, uptr pc, bool start_thread);
 
 void ReportRace(ThreadState *thr);
 bool OutputReport(ThreadState *thr, const ScopedReport &srep);
@@ -749,6 +749,44 @@ void TraceTime(ThreadState *thr);
 
 }  // namespace v3
 
+void GrowShadowStack(ThreadState *thr);
+
+ALWAYS_INLINE
+void FuncEntry(ThreadState *thr, uptr pc) {
+  DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void *)pc);
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc);
+  }
+
+  // Shadow stack maintenance can be replaced with
+  // stack unwinding during trace switch (which presumably must be faster).
+  DCHECK_GE(thr->shadow_stack_pos, thr->shadow_stack);
+#if !SANITIZER_GO
+  DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
+#else
+  if (thr->shadow_stack_pos == thr->shadow_stack_end)
+    GrowShadowStack(thr);
+#endif
+  thr->shadow_stack_pos[0] = pc;
+  thr->shadow_stack_pos++;
+}
+
+ALWAYS_INLINE
+void FuncExit(ThreadState *thr) {
+  DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid());
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0);
+  }
+
+  DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack);
+#if !SANITIZER_GO
+  DCHECK_LT(thr->shadow_stack_pos, thr->shadow_stack_end);
+#endif
+  thr->shadow_stack_pos--;
+}
+
 #if !SANITIZER_GO
 extern void (*on_initialize)(void);
 extern int (*on_finalize)(int);
diff --git a/libsanitizer/tsan/tsan_rtl_access.cpp b/libsanitizer/tsan/tsan_rtl_access.cpp
new file mode 100644
index 0000000..7365fda
--- /dev/null
+++ b/libsanitizer/tsan/tsan_rtl_access.cpp
@@ -0,0 +1,604 @@
+//===-- tsan_rtl_access.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Definitions of memory access and function entry/exit entry points.
+//===----------------------------------------------------------------------===//
+
+#include "tsan_rtl.h"
+
+namespace __tsan {
+
+namespace v3 {
+
+ALWAYS_INLINE USED bool TryTraceMemoryAccess(ThreadState *thr, uptr pc,
+                                             uptr addr, uptr size,
+                                             AccessType typ) {
+  DCHECK(size == 1 || size == 2 || size == 4 || size == 8);
+  if (!kCollectHistory)
+    return true;
+  EventAccess *ev;
+  if (UNLIKELY(!TraceAcquire(thr, &ev)))
+    return false;
+  u64 size_log = size == 1 ? 0 : size == 2 ? 1 : size == 4 ? 2 : 3;
+  uptr pc_delta = pc - thr->trace_prev_pc + (1 << (EventAccess::kPCBits - 1));
+  thr->trace_prev_pc = pc;
+  if (LIKELY(pc_delta < (1 << EventAccess::kPCBits))) {
+    ev->is_access = 1;
+    ev->is_read = !!(typ & kAccessRead);
+    ev->is_atomic = !!(typ & kAccessAtomic);
+    ev->size_log = size_log;
+    ev->pc_delta = pc_delta;
+    DCHECK_EQ(ev->pc_delta, pc_delta);
+    ev->addr = CompressAddr(addr);
+    TraceRelease(thr, ev);
+    return true;
+  }
+  auto *evex = reinterpret_cast<EventAccessExt *>(ev);
+  evex->is_access = 0;
+  evex->is_func = 0;
+  evex->type = EventType::kAccessExt;
+  evex->is_read = !!(typ & kAccessRead);
+  evex->is_atomic = !!(typ & kAccessAtomic);
+  evex->size_log = size_log;
+  evex->addr = CompressAddr(addr);
+  evex->pc = pc;
+  TraceRelease(thr, evex);
+  return true;
+}
+
+ALWAYS_INLINE USED bool TryTraceMemoryAccessRange(ThreadState *thr, uptr pc,
+                                                  uptr addr, uptr size,
+                                                  AccessType typ) {
+  if (!kCollectHistory)
+    return true;
+  EventAccessRange *ev;
+  if (UNLIKELY(!TraceAcquire(thr, &ev)))
+    return false;
+  thr->trace_prev_pc = pc;
+  ev->is_access = 0;
+  ev->is_func = 0;
+  ev->type = EventType::kAccessRange;
+  ev->is_read = !!(typ & kAccessRead);
+  ev->is_free = !!(typ & kAccessFree);
+  ev->size_lo = size;
+  ev->pc = CompressAddr(pc);
+  ev->addr = CompressAddr(addr);
+  ev->size_hi = size >> EventAccessRange::kSizeLoBits;
+  TraceRelease(thr, ev);
+  return true;
+}
+
+void TraceMemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                            AccessType typ) {
+  if (LIKELY(TryTraceMemoryAccessRange(thr, pc, addr, size, typ)))
+    return;
+  TraceSwitchPart(thr);
+  UNUSED bool res = TryTraceMemoryAccessRange(thr, pc, addr, size, typ);
+  DCHECK(res);
+}
+
+void TraceFunc(ThreadState *thr, uptr pc) {
+  if (LIKELY(TryTraceFunc(thr, pc)))
+    return;
+  TraceSwitchPart(thr);
+  UNUSED bool res = TryTraceFunc(thr, pc);
+  DCHECK(res);
+}
+
+void TraceMutexLock(ThreadState *thr, EventType type, uptr pc, uptr addr,
+                    StackID stk) {
+  DCHECK(type == EventType::kLock || type == EventType::kRLock);
+  if (!kCollectHistory)
+    return;
+  EventLock ev;
+  ev.is_access = 0;
+  ev.is_func = 0;
+  ev.type = type;
+  ev.pc = CompressAddr(pc);
+  ev.stack_lo = stk;
+  ev.stack_hi = stk >> EventLock::kStackIDLoBits;
+  ev._ = 0;
+  ev.addr = CompressAddr(addr);
+  TraceEvent(thr, ev);
+}
+
+void TraceMutexUnlock(ThreadState *thr, uptr addr) {
+  if (!kCollectHistory)
+    return;
+  EventUnlock ev;
+  ev.is_access = 0;
+  ev.is_func = 0;
+  ev.type = EventType::kUnlock;
+  ev._ = 0;
+  ev.addr = CompressAddr(addr);
+  TraceEvent(thr, ev);
+}
+
+void TraceTime(ThreadState *thr) {
+  if (!kCollectHistory)
+    return;
+  EventTime ev;
+  ev.is_access = 0;
+  ev.is_func = 0;
+  ev.type = EventType::kTime;
+  ev.sid = static_cast<u64>(thr->sid);
+  ev.epoch = static_cast<u64>(thr->epoch);
+  ev._ = 0;
+  TraceEvent(thr, ev);
+}
+
+}  // namespace v3
+
+ALWAYS_INLINE
+Shadow LoadShadow(u64 *p) {
+  u64 raw = atomic_load((atomic_uint64_t *)p, memory_order_relaxed);
+  return Shadow(raw);
+}
+
+ALWAYS_INLINE
+void StoreShadow(u64 *sp, u64 s) {
+  atomic_store((atomic_uint64_t *)sp, s, memory_order_relaxed);
+}
+
+ALWAYS_INLINE
+void StoreIfNotYetStored(u64 *sp, u64 *s) {
+  StoreShadow(sp, *s);
+  *s = 0;
+}
+
+extern "C" void __tsan_report_race();
+
+ALWAYS_INLINE
+void HandleRace(ThreadState *thr, u64 *shadow_mem, Shadow cur, Shadow old) {
+  thr->racy_state[0] = cur.raw();
+  thr->racy_state[1] = old.raw();
+  thr->racy_shadow_addr = shadow_mem;
+#if !SANITIZER_GO
+  HACKY_CALL(__tsan_report_race);
+#else
+  ReportRace(thr);
+#endif
+}
+
+static inline bool HappensBefore(Shadow old, ThreadState *thr) {
+  return thr->clock.get(old.TidWithIgnore()) >= old.epoch();
+}
+
+ALWAYS_INLINE
+void MemoryAccessImpl1(ThreadState *thr, uptr addr, int kAccessSizeLog,
+                       bool kAccessIsWrite, bool kIsAtomic, u64 *shadow_mem,
+                       Shadow cur) {
+  // This potentially can live in an MMX/SSE scratch register.
+  // The required intrinsics are:
+  // __m128i _mm_move_epi64(__m128i*);
+  // _mm_storel_epi64(u64*, __m128i);
+  u64 store_word = cur.raw();
+  bool stored = false;
+
+  // scan all the shadow values and dispatch to 4 categories:
+  // same, replace, candidate and race (see comments below).
+  // we consider only 3 cases regarding access sizes:
+  // equal, intersect and not intersect. initially I considered
+  // larger and smaller as well, it allowed to replace some
+  // 'candidates' with 'same' or 'replace', but I think
+  // it's just not worth it (performance- and complexity-wise).
+
+  Shadow old(0);
+
+  // It release mode we manually unroll the loop,
+  // because empirically gcc generates better code this way.
+  // However, we can't afford unrolling in debug mode, because the function
+  // consumes almost 4K of stack. Gtest gives only 4K of stack to death test
+  // threads, which is not enough for the unrolled loop.
+#if SANITIZER_DEBUG
+  for (int idx = 0; idx < 4; idx++) {
+#  include "tsan_update_shadow_word.inc"
+  }
+#else
+  int idx = 0;
+#  include "tsan_update_shadow_word.inc"
+  idx = 1;
+  if (stored) {
+#  include "tsan_update_shadow_word.inc"
+  } else {
+#  include "tsan_update_shadow_word.inc"
+  }
+  idx = 2;
+  if (stored) {
+#  include "tsan_update_shadow_word.inc"
+  } else {
+#  include "tsan_update_shadow_word.inc"
+  }
+  idx = 3;
+  if (stored) {
+#  include "tsan_update_shadow_word.inc"
+  } else {
+#  include "tsan_update_shadow_word.inc"
+  }
+#endif
+
+  // we did not find any races and had already stored
+  // the current access info, so we are done
+  if (LIKELY(stored))
+    return;
+  // choose a random candidate slot and replace it
+  StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
+  return;
+RACE:
+  HandleRace(thr, shadow_mem, cur, old);
+  return;
+}
+
+void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                           AccessType typ) {
+  DCHECK(!(typ & kAccessAtomic));
+  const bool kAccessIsWrite = !(typ & kAccessRead);
+  const bool kIsAtomic = false;
+  while (size) {
+    int size1 = 1;
+    int kAccessSizeLog = kSizeLog1;
+    if (size >= 8 && (addr & ~7) == ((addr + 7) & ~7)) {
+      size1 = 8;
+      kAccessSizeLog = kSizeLog8;
+    } else if (size >= 4 && (addr & ~7) == ((addr + 3) & ~7)) {
+      size1 = 4;
+      kAccessSizeLog = kSizeLog4;
+    } else if (size >= 2 && (addr & ~7) == ((addr + 1) & ~7)) {
+      size1 = 2;
+      kAccessSizeLog = kSizeLog2;
+    }
+    MemoryAccess(thr, pc, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic);
+    addr += size1;
+    size -= size1;
+  }
+}
+
+ALWAYS_INLINE
+bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
+  Shadow cur(a);
+  for (uptr i = 0; i < kShadowCnt; i++) {
+    Shadow old(LoadShadow(&s[i]));
+    if (Shadow::Addr0AndSizeAreEqual(cur, old) &&
+        old.TidWithIgnore() == cur.TidWithIgnore() &&
+        old.epoch() > sync_epoch && old.IsAtomic() == cur.IsAtomic() &&
+        old.IsRead() <= cur.IsRead())
+      return true;
+  }
+  return false;
+}
+
+#if TSAN_VECTORIZE
+#  define SHUF(v0, v1, i0, i1, i2, i3)                    \
+    _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(v0), \
+                                    _mm_castsi128_ps(v1), \
+                                    (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64))
+ALWAYS_INLINE
+bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
+  // This is an optimized version of ContainsSameAccessSlow.
+  // load current access into access[0:63]
+  const m128 access = _mm_cvtsi64_si128(a);
+  // duplicate high part of access in addr0:
+  // addr0[0:31]        = access[32:63]
+  // addr0[32:63]       = access[32:63]
+  // addr0[64:95]       = access[32:63]
+  // addr0[96:127]      = access[32:63]
+  const m128 addr0 = SHUF(access, access, 1, 1, 1, 1);
+  // load 4 shadow slots
+  const m128 shadow0 = _mm_load_si128((__m128i *)s);
+  const m128 shadow1 = _mm_load_si128((__m128i *)s + 1);
+  // load high parts of 4 shadow slots into addr_vect:
+  // addr_vect[0:31]    = shadow0[32:63]
+  // addr_vect[32:63]   = shadow0[96:127]
+  // addr_vect[64:95]   = shadow1[32:63]
+  // addr_vect[96:127]  = shadow1[96:127]
+  m128 addr_vect = SHUF(shadow0, shadow1, 1, 3, 1, 3);
+  if (!is_write) {
+    // set IsRead bit in addr_vect
+    const m128 rw_mask1 = _mm_cvtsi64_si128(1 << 15);
+    const m128 rw_mask = SHUF(rw_mask1, rw_mask1, 0, 0, 0, 0);
+    addr_vect = _mm_or_si128(addr_vect, rw_mask);
+  }
+  // addr0 == addr_vect?
+  const m128 addr_res = _mm_cmpeq_epi32(addr0, addr_vect);
+  // epoch1[0:63]       = sync_epoch
+  const m128 epoch1 = _mm_cvtsi64_si128(sync_epoch);
+  // epoch[0:31]        = sync_epoch[0:31]
+  // epoch[32:63]       = sync_epoch[0:31]
+  // epoch[64:95]       = sync_epoch[0:31]
+  // epoch[96:127]      = sync_epoch[0:31]
+  const m128 epoch = SHUF(epoch1, epoch1, 0, 0, 0, 0);
+  // load low parts of shadow cell epochs into epoch_vect:
+  // epoch_vect[0:31]   = shadow0[0:31]
+  // epoch_vect[32:63]  = shadow0[64:95]
+  // epoch_vect[64:95]  = shadow1[0:31]
+  // epoch_vect[96:127] = shadow1[64:95]
+  const m128 epoch_vect = SHUF(shadow0, shadow1, 0, 2, 0, 2);
+  // epoch_vect >= sync_epoch?
+  const m128 epoch_res = _mm_cmpgt_epi32(epoch_vect, epoch);
+  // addr_res & epoch_res
+  const m128 res = _mm_and_si128(addr_res, epoch_res);
+  // mask[0] = res[7]
+  // mask[1] = res[15]
+  // ...
+  // mask[15] = res[127]
+  const int mask = _mm_movemask_epi8(res);
+  return mask != 0;
+}
+#endif
+
+ALWAYS_INLINE
+bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
+#if TSAN_VECTORIZE
+  bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write);
+  // NOTE: this check can fail if the shadow is concurrently mutated
+  // by other threads. But it still can be useful if you modify
+  // ContainsSameAccessFast and want to ensure that it's not completely broken.
+  // DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write));
+  return res;
+#else
+  return ContainsSameAccessSlow(s, a, sync_epoch, is_write);
+#endif
+}
+
+ALWAYS_INLINE USED void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
+                                     int kAccessSizeLog, bool kAccessIsWrite,
+                                     bool kIsAtomic) {
+  RawShadow *shadow_mem = MemToShadow(addr);
+  DPrintf2(
+      "#%d: MemoryAccess: @%p %p size=%d"
+      " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
+      (int)thr->fast_state.tid(), (void *)pc, (void *)addr,
+      (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
+      (uptr)shadow_mem[0], (uptr)shadow_mem[1], (uptr)shadow_mem[2],
+      (uptr)shadow_mem[3]);
+#if SANITIZER_DEBUG
+  if (!IsAppMem(addr)) {
+    Printf("Access to non app mem %zx\n", addr);
+    DCHECK(IsAppMem(addr));
+  }
+  if (!IsShadowMem(shadow_mem)) {
+    Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
+    DCHECK(IsShadowMem(shadow_mem));
+  }
+#endif
+
+  if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
+    // Access to .rodata section, no races here.
+    // Measurements show that it can be 10-20% of all memory accesses.
+    return;
+  }
+
+  FastState fast_state = thr->fast_state;
+  if (UNLIKELY(fast_state.GetIgnoreBit())) {
+    return;
+  }
+
+  Shadow cur(fast_state);
+  cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog);
+  cur.SetWrite(kAccessIsWrite);
+  cur.SetAtomic(kIsAtomic);
+
+  if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch,
+                                kAccessIsWrite))) {
+    return;
+  }
+
+  if (kCollectHistory) {
+    fast_state.IncrementEpoch();
+    thr->fast_state = fast_state;
+    TraceAddEvent(thr, fast_state, EventTypeMop, pc);
+    cur.IncrementEpoch();
+  }
+
+  MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
+                    shadow_mem, cur);
+}
+
+// Called by MemoryAccessRange in tsan_rtl_thread.cpp
+ALWAYS_INLINE USED void MemoryAccessImpl(ThreadState *thr, uptr addr,
+                                         int kAccessSizeLog,
+                                         bool kAccessIsWrite, bool kIsAtomic,
+                                         u64 *shadow_mem, Shadow cur) {
+  if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch,
+                                kAccessIsWrite))) {
+    return;
+  }
+
+  MemoryAccessImpl1(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic,
+                    shadow_mem, cur);
+}
+
+static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                           u64 val) {
+  (void)thr;
+  (void)pc;
+  if (size == 0)
+    return;
+  // FIXME: fix me.
+  uptr offset = addr % kShadowCell;
+  if (offset) {
+    offset = kShadowCell - offset;
+    if (size <= offset)
+      return;
+    addr += offset;
+    size -= offset;
+  }
+  DCHECK_EQ(addr % 8, 0);
+  // If a user passes some insane arguments (memset(0)),
+  // let it just crash as usual.
+  if (!IsAppMem(addr) || !IsAppMem(addr + size - 1))
+    return;
+  // Don't want to touch lots of shadow memory.
+  // If a program maps 10MB stack, there is no need reset the whole range.
+  size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1);
+  // UnmapOrDie/MmapFixedNoReserve does not work on Windows.
+  if (SANITIZER_WINDOWS || size < common_flags()->clear_shadow_mmap_threshold) {
+    RawShadow *p = MemToShadow(addr);
+    CHECK(IsShadowMem(p));
+    CHECK(IsShadowMem(p + size * kShadowCnt / kShadowCell - 1));
+    // FIXME: may overwrite a part outside the region
+    for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) {
+      p[i++] = val;
+      for (uptr j = 1; j < kShadowCnt; j++) p[i++] = 0;
+    }
+  } else {
+    // The region is big, reset only beginning and end.
+    const uptr kPageSize = GetPageSizeCached();
+    RawShadow *begin = MemToShadow(addr);
+    RawShadow *end = begin + size / kShadowCell * kShadowCnt;
+    RawShadow *p = begin;
+    // Set at least first kPageSize/2 to page boundary.
+    while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) {
+      *p++ = val;
+      for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0;
+    }
+    // Reset middle part.
+    RawShadow *p1 = p;
+    p = RoundDown(end, kPageSize);
+    if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1))
+      Die();
+    // Set the ending.
+    while (p < end) {
+      *p++ = val;
+      for (uptr j = 1; j < kShadowCnt; j++) *p++ = 0;
+    }
+  }
+}
+
+void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) {
+  MemoryRangeSet(thr, pc, addr, size, 0);
+}
+
+void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) {
+  // Processing more than 1k (4k of shadow) is expensive,
+  // can cause excessive memory consumption (user does not necessary touch
+  // the whole range) and most likely unnecessary.
+  if (size > 1024)
+    size = 1024;
+  CHECK_EQ(thr->is_freeing, false);
+  thr->is_freeing = true;
+  MemoryAccessRange(thr, pc, addr, size, true);
+  thr->is_freeing = false;
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
+  }
+  Shadow s(thr->fast_state);
+  s.ClearIgnoreBit();
+  s.MarkAsFreed();
+  s.SetWrite(true);
+  s.SetAddr0AndSizeLog(0, 3);
+  MemoryRangeSet(thr, pc, addr, size, s.raw());
+}
+
+void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) {
+  if (kCollectHistory) {
+    thr->fast_state.IncrementEpoch();
+    TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc);
+  }
+  Shadow s(thr->fast_state);
+  s.ClearIgnoreBit();
+  s.SetWrite(true);
+  s.SetAddr0AndSizeLog(0, 3);
+  MemoryRangeSet(thr, pc, addr, size, s.raw());
+}
+
+void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr,
+                                         uptr size) {
+  if (thr->ignore_reads_and_writes == 0)
+    MemoryRangeImitateWrite(thr, pc, addr, size);
+  else
+    MemoryResetRange(thr, pc, addr, size);
+}
+
+void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, uptr size,
+                       bool is_write) {
+  if (size == 0)
+    return;
+
+  RawShadow *shadow_mem = MemToShadow(addr);
+  DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n", thr->tid,
+           (void *)pc, (void *)addr, (int)size, is_write);
+
+#if SANITIZER_DEBUG
+  if (!IsAppMem(addr)) {
+    Printf("Access to non app mem %zx\n", addr);
+    DCHECK(IsAppMem(addr));
+  }
+  if (!IsAppMem(addr + size - 1)) {
+    Printf("Access to non app mem %zx\n", addr + size - 1);
+    DCHECK(IsAppMem(addr + size - 1));
+  }
+  if (!IsShadowMem(shadow_mem)) {
+    Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
+    DCHECK(IsShadowMem(shadow_mem));
+  }
+  if (!IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1)) {
+    Printf("Bad shadow addr %p (%zx)\n", shadow_mem + size * kShadowCnt / 8 - 1,
+           addr + size - 1);
+    DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1));
+  }
+#endif
+
+  if (*shadow_mem == kShadowRodata) {
+    DCHECK(!is_write);
+    // Access to .rodata section, no races here.
+    // Measurements show that it can be 10-20% of all memory accesses.
+    return;
+  }
+
+  FastState fast_state = thr->fast_state;
+  if (fast_state.GetIgnoreBit())
+    return;
+
+  fast_state.IncrementEpoch();
+  thr->fast_state = fast_state;
+  TraceAddEvent(thr, fast_state, EventTypeMop, pc);
+
+  bool unaligned = (addr % kShadowCell) != 0;
+
+  // Handle unaligned beginning, if any.
+  for (; addr % kShadowCell && size; addr++, size--) {
+    int const kAccessSizeLog = 0;
+    Shadow cur(fast_state);
+    cur.SetWrite(is_write);
+    cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
+    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem,
+                     cur);
+  }
+  if (unaligned)
+    shadow_mem += kShadowCnt;
+  // Handle middle part, if any.
+  for (; size >= kShadowCell; addr += kShadowCell, size -= kShadowCell) {
+    int const kAccessSizeLog = 3;
+    Shadow cur(fast_state);
+    cur.SetWrite(is_write);
+    cur.SetAddr0AndSizeLog(0, kAccessSizeLog);
+    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem,
+                     cur);
+    shadow_mem += kShadowCnt;
+  }
+  // Handle ending, if any.
+  for (; size; addr++, size--) {
+    int const kAccessSizeLog = 0;
+    Shadow cur(fast_state);
+    cur.SetWrite(is_write);
+    cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
+    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false, shadow_mem,
+                     cur);
+  }
+}
+
+}  // namespace __tsan
+
+#if !SANITIZER_GO
+// Must be included in this file to make sure everything is inlined.
+#  include "tsan_interface.inc"
+#endif
diff --git a/libsanitizer/tsan/tsan_rtl_amd64.S b/libsanitizer/tsan/tsan_rtl_amd64.S
index 10c0122..632b19d 100644
--- a/libsanitizer/tsan/tsan_rtl_amd64.S
+++ b/libsanitizer/tsan/tsan_rtl_amd64.S
@@ -42,6 +42,25 @@ ASM_SYMBOL(__tsan_trace_switch_thunk):
   push %r11
   CFI_ADJUST_CFA_OFFSET(8)
   CFI_REL_OFFSET(%r11, 0)
+  # All XMM registers are caller-saved.
+  sub $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(0x100)
+  vmovdqu %xmm0, 0x0(%rsp)
+  vmovdqu %xmm1, 0x10(%rsp)
+  vmovdqu %xmm2, 0x20(%rsp)
+  vmovdqu %xmm3, 0x30(%rsp)
+  vmovdqu %xmm4, 0x40(%rsp)
+  vmovdqu %xmm5, 0x50(%rsp)
+  vmovdqu %xmm6, 0x60(%rsp)
+  vmovdqu %xmm7, 0x70(%rsp)
+  vmovdqu %xmm8, 0x80(%rsp)
+  vmovdqu %xmm9, 0x90(%rsp)
+  vmovdqu %xmm10, 0xa0(%rsp)
+  vmovdqu %xmm11, 0xb0(%rsp)
+  vmovdqu %xmm12, 0xc0(%rsp)
+  vmovdqu %xmm13, 0xd0(%rsp)
+  vmovdqu %xmm14, 0xe0(%rsp)
+  vmovdqu %xmm15, 0xf0(%rsp)
   # Align stack frame.
   push %rbx  # non-scratch
   CFI_ADJUST_CFA_OFFSET(8)
@@ -59,6 +78,24 @@ ASM_SYMBOL(__tsan_trace_switch_thunk):
   pop %rbx
   CFI_ADJUST_CFA_OFFSET(-8)
   # Restore scratch registers.
+  vmovdqu 0x0(%rsp), %xmm0
+  vmovdqu 0x10(%rsp), %xmm1
+  vmovdqu 0x20(%rsp), %xmm2
+  vmovdqu 0x30(%rsp), %xmm3
+  vmovdqu 0x40(%rsp), %xmm4
+  vmovdqu 0x50(%rsp), %xmm5
+  vmovdqu 0x60(%rsp), %xmm6
+  vmovdqu 0x70(%rsp), %xmm7
+  vmovdqu 0x80(%rsp), %xmm8
+  vmovdqu 0x90(%rsp), %xmm9
+  vmovdqu 0xa0(%rsp), %xmm10
+  vmovdqu 0xb0(%rsp), %xmm11
+  vmovdqu 0xc0(%rsp), %xmm12
+  vmovdqu 0xd0(%rsp), %xmm13
+  vmovdqu 0xe0(%rsp), %xmm14
+  vmovdqu 0xf0(%rsp), %xmm15
+  add $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(-0x100)
   pop %r11
   CFI_ADJUST_CFA_OFFSET(-8)
   pop %r10
@@ -123,6 +160,25 @@ ASM_SYMBOL(__tsan_report_race_thunk):
   push %r11
   CFI_ADJUST_CFA_OFFSET(8)
   CFI_REL_OFFSET(%r11, 0)
+  # All XMM registers are caller-saved.
+  sub $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(0x100)
+  vmovdqu %xmm0, 0x0(%rsp)
+  vmovdqu %xmm1, 0x10(%rsp)
+  vmovdqu %xmm2, 0x20(%rsp)
+  vmovdqu %xmm3, 0x30(%rsp)
+  vmovdqu %xmm4, 0x40(%rsp)
+  vmovdqu %xmm5, 0x50(%rsp)
+  vmovdqu %xmm6, 0x60(%rsp)
+  vmovdqu %xmm7, 0x70(%rsp)
+  vmovdqu %xmm8, 0x80(%rsp)
+  vmovdqu %xmm9, 0x90(%rsp)
+  vmovdqu %xmm10, 0xa0(%rsp)
+  vmovdqu %xmm11, 0xb0(%rsp)
+  vmovdqu %xmm12, 0xc0(%rsp)
+  vmovdqu %xmm13, 0xd0(%rsp)
+  vmovdqu %xmm14, 0xe0(%rsp)
+  vmovdqu %xmm15, 0xf0(%rsp)
   # Align stack frame.
   push %rbx  # non-scratch
   CFI_ADJUST_CFA_OFFSET(8)
@@ -140,6 +196,24 @@ ASM_SYMBOL(__tsan_report_race_thunk):
   pop %rbx
   CFI_ADJUST_CFA_OFFSET(-8)
   # Restore scratch registers.
+  vmovdqu 0x0(%rsp), %xmm0
+  vmovdqu 0x10(%rsp), %xmm1
+  vmovdqu 0x20(%rsp), %xmm2
+  vmovdqu 0x30(%rsp), %xmm3
+  vmovdqu 0x40(%rsp), %xmm4
+  vmovdqu 0x50(%rsp), %xmm5
+  vmovdqu 0x60(%rsp), %xmm6
+  vmovdqu 0x70(%rsp), %xmm7
+  vmovdqu 0x80(%rsp), %xmm8
+  vmovdqu 0x90(%rsp), %xmm9
+  vmovdqu 0xa0(%rsp), %xmm10
+  vmovdqu 0xb0(%rsp), %xmm11
+  vmovdqu 0xc0(%rsp), %xmm12
+  vmovdqu 0xd0(%rsp), %xmm13
+  vmovdqu 0xe0(%rsp), %xmm14
+  vmovdqu 0xf0(%rsp), %xmm15
+  add $0x100, %rsp
+  CFI_ADJUST_CFA_OFFSET(-0x100)
   pop %r11
   CFI_ADJUST_CFA_OFFSET(-8)
   pop %r10
diff --git a/libsanitizer/tsan/tsan_rtl_report.cpp b/libsanitizer/tsan/tsan_rtl_report.cpp
index 1f0bcb3..811695d 100644
--- a/libsanitizer/tsan/tsan_rtl_report.cpp
+++ b/libsanitizer/tsan/tsan_rtl_report.cpp
@@ -560,9 +560,7 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
     if (tctx->thr)
       last_pos = (Event *)atomic_load_relaxed(&tctx->thr->trace_pos);
   }
-  // Too large for stack.
-  alignas(MutexSet) static char mset_storage[sizeof(MutexSet)];
-  MutexSet &mset = *new (mset_storage) MutexSet();
+  DynamicMutexSet mset;
   Vector<uptr> stack;
   uptr prev_pc = 0;
   bool found = false;
@@ -588,7 +586,7 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
           if (match && type == EventType::kAccessExt &&
               IsWithinAccess(addr, size, ev_addr, ev_size) &&
               is_read == ev->is_read && is_atomic == ev->is_atomic && !is_free)
-            RestoreStackMatch(pstk, pmset, &stack, &mset, ev_pc, &found);
+            RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
           return;
         }
         if (evp->is_func) {
@@ -615,7 +613,7 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
                 IsWithinAccess(addr, size, ev_addr, ev_size) &&
                 is_read == ev->is_read && is_atomic == ev->is_atomic &&
                 !is_free)
-              RestoreStackMatch(pstk, pmset, &stack, &mset, ev->pc, &found);
+              RestoreStackMatch(pstk, pmset, &stack, mset, ev->pc, &found);
             break;
           }
           case EventType::kAccessRange: {
@@ -630,7 +628,7 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
             if (match && type == EventType::kAccessExt &&
                 IsWithinAccess(addr, size, ev_addr, ev_size) &&
                 is_read == ev->is_read && !is_atomic && is_free == ev->is_free)
-              RestoreStackMatch(pstk, pmset, &stack, &mset, ev_pc, &found);
+              RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
             break;
           }
           case EventType::kLock:
@@ -644,18 +642,18 @@ bool RestoreStack(Tid tid, EventType type, Sid sid, Epoch epoch, uptr addr,
                 (ev->stack_hi << EventLock::kStackIDLoBits) + ev->stack_lo;
             DPrintf2("  Lock: pc=0x%zx addr=0x%zx stack=%u write=%d\n", ev_pc,
                      ev_addr, stack_id, is_write);
-            mset.AddAddr(ev_addr, stack_id, is_write);
+            mset->AddAddr(ev_addr, stack_id, is_write);
             // Events with ev_pc == 0 are written to the beginning of trace
             // part as initial mutex set (are not real).
             if (match && type == EventType::kLock && addr == ev_addr && ev_pc)
-              RestoreStackMatch(pstk, pmset, &stack, &mset, ev_pc, &found);
+              RestoreStackMatch(pstk, pmset, &stack, mset, ev_pc, &found);
             break;
           }
           case EventType::kUnlock: {
             auto *ev = reinterpret_cast<EventUnlock *>(evp);
             uptr ev_addr = RestoreAddr(ev->addr);
             DPrintf2("  Unlock: addr=0x%zx\n", ev_addr);
-            mset.DelAddr(ev_addr);
+            mset->DelAddr(ev_addr);
             break;
           }
           case EventType::kTime:
@@ -897,11 +895,7 @@ void ReportRace(ThreadState *thr) {
   if (IsFiredSuppression(ctx, typ, traces[0]))
     return;
 
-  // MutexSet is too large to live on stack.
-  Vector<u64> mset_buffer;
-  mset_buffer.Resize(sizeof(MutexSet) / sizeof(u64) + 1);
-  MutexSet *mset2 = new(&mset_buffer[0]) MutexSet();
-
+  DynamicMutexSet mset2;
   Shadow s2(thr->racy_state[1]);
   RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]);
   if (IsFiredSuppression(ctx, typ, traces[1]))
diff --git a/libsanitizer/tsan/tsan_rtl_thread.cpp b/libsanitizer/tsan/tsan_rtl_thread.cpp
index 61133a4..6e652ee 100644
--- a/libsanitizer/tsan/tsan_rtl_thread.cpp
+++ b/libsanitizer/tsan/tsan_rtl_thread.cpp
@@ -323,85 +323,6 @@ void ThreadSetName(ThreadState *thr, const char *name) {
   ctx->thread_registry.SetThreadName(thr->tid, name);
 }
 
-void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
-                       uptr size, bool is_write) {
-  if (size == 0)
-    return;
-
-  RawShadow *shadow_mem = MemToShadow(addr);
-  DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_write=%d\n",
-      thr->tid, (void*)pc, (void*)addr,
-      (int)size, is_write);
-
-#if SANITIZER_DEBUG
-  if (!IsAppMem(addr)) {
-    Printf("Access to non app mem %zx\n", addr);
-    DCHECK(IsAppMem(addr));
-  }
-  if (!IsAppMem(addr + size - 1)) {
-    Printf("Access to non app mem %zx\n", addr + size - 1);
-    DCHECK(IsAppMem(addr + size - 1));
-  }
-  if (!IsShadowMem(shadow_mem)) {
-    Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr);
-    DCHECK(IsShadowMem(shadow_mem));
-  }
-  if (!IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1)) {
-    Printf("Bad shadow addr %p (%zx)\n",
-               shadow_mem + size * kShadowCnt / 8 - 1, addr + size - 1);
-    DCHECK(IsShadowMem(shadow_mem + size * kShadowCnt / 8 - 1));
-  }
-#endif
-
-  if (*shadow_mem == kShadowRodata) {
-    DCHECK(!is_write);
-    // Access to .rodata section, no races here.
-    // Measurements show that it can be 10-20% of all memory accesses.
-    return;
-  }
-
-  FastState fast_state = thr->fast_state;
-  if (fast_state.GetIgnoreBit())
-    return;
-
-  fast_state.IncrementEpoch();
-  thr->fast_state = fast_state;
-  TraceAddEvent(thr, fast_state, EventTypeMop, pc);
-
-  bool unaligned = (addr % kShadowCell) != 0;
-
-  // Handle unaligned beginning, if any.
-  for (; addr % kShadowCell && size; addr++, size--) {
-    int const kAccessSizeLog = 0;
-    Shadow cur(fast_state);
-    cur.SetWrite(is_write);
-    cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
-    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
-        shadow_mem, cur);
-  }
-  if (unaligned)
-    shadow_mem += kShadowCnt;
-  // Handle middle part, if any.
-  for (; size >= kShadowCell; addr += kShadowCell, size -= kShadowCell) {
-    int const kAccessSizeLog = 3;
-    Shadow cur(fast_state);
-    cur.SetWrite(is_write);
-    cur.SetAddr0AndSizeLog(0, kAccessSizeLog);
-    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
-        shadow_mem, cur);
-    shadow_mem += kShadowCnt;
-  }
-  // Handle ending, if any.
-  for (; size; addr++, size--) {
-    int const kAccessSizeLog = 0;
-    Shadow cur(fast_state);
-    cur.SetWrite(is_write);
-    cur.SetAddr0AndSizeLog(addr & (kShadowCell - 1), kAccessSizeLog);
-    MemoryAccessImpl(thr, addr, kAccessSizeLog, is_write, false,
-        shadow_mem, cur);
-  }
-}
-
 #if !SANITIZER_GO
 void FiberSwitchImpl(ThreadState *from, ThreadState *to) {
   Processor *proc = from->proc();
diff --git a/libstdc++-v3/ChangeLog b/libstdc++-v3/ChangeLog
index 3aa8a53..cb6a8a3 100644
--- a/libstdc++-v3/ChangeLog
+++ b/libstdc++-v3/ChangeLog
@@ -1,3 +1,631 @@
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* testsuite/20_util/to_address/1_neg.cc: Adjust error.
+	* testsuite/26_numerics/random/concept.cc: Adjust asserts.
+
+2021-11-15  François Dumont  <fdumont@gcc.gnu.org>
+
+	* include/bits/hashtable_policy.h:
+	(_Hash_code_base<>::_M_hash_code(const _Hash&, const _Hash_node_value<_Value, true>&)): New.
+	(_Hash_code_base<>::_M_hash_code<_H2>(const _H2&, const _Hash_node_value<>&)): New.
+	* include/bits/hashtable.h (_Hashtable<>::_M_merge_unique): Use latter.
+	(_Hashtable<>::_M_merge_multi): Likewise.
+	* testsuite/23_containers/unordered_multiset/modifiers/merge.cc (test05): New test.
+	* testsuite/23_containers/unordered_set/modifiers/merge.cc (test04): New test.
+
+2021-11-15  Jason Merrill  <jason@redhat.com>
+
+	* src/c++17/memory_resource.cc: Add missing constexpr.
+	* include/experimental/internet: Only mark copy constructor
+	as constexpr with __cpp_constexpr_dynamic_alloc.
+
+2021-11-14  François Dumont  <fdumont@gcc.gnu.org>
+
+	* include/debug/safe_container.h (_Safe_container<>::_M_safe): Remove.
+	* include/debug/deque (deque::operator=(initializer_list<>)): Replace
+	_M_base() call with _Base:: call.
+	(deque::operator[](size_type)): Likewise.
+	* include/debug/forward_list (forward_list(forward_list&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(forward_list::operator=(initializer_list<>)): Remove _M_base() calls.
+	(forward_list::splice_after, forward_list::merge): Likewise.
+	* include/debug/list (list(list&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(list::operator=(initializer_list<>)): Remove _M_base() calls.
+	(list::splice, list::merge): Likewise.
+	* include/debug/map.h (map(map&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(map::operator=(initializer_list<>)): Remove _M_base() calls.
+	* include/debug/multimap.h (multimap(multimap&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(multimap::operator=(initializer_list<>)): Remove _M_base() calls.
+	* include/debug/set.h (set(set&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(set::operator=(initializer_list<>)): Remove _M_base() calls.
+	* include/debug/multiset.h (multiset(multiset&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(multiset::operator=(initializer_list<>)): Remove _M_base() calls.
+	* include/debug/string (basic_string(basic_string&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(basic_string::operator=(initializer_list<>)): Remove _M_base() call.
+	(basic_string::operator=(const _CharT*), basic_string::operator=(_CharT)): Likewise.
+	(basic_string::operator[](size_type), basic_string::operator+=(const basic_string&)):
+	Likewise.
+	(basic_string::operator+=(const _Char*), basic_string::operator+=(_CharT)): Likewise.
+	* include/debug/unordered_map (unordered_map(unordered_map&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(unordered_map::operator=(initializer_list<>), unordered_map::merge):
+	Remove _M_base() calls.
+	(unordered_multimap(unordered_multimap&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(unordered_multimap::operator=(initializer_list<>), unordered_multimap::merge):
+	Remove _M_base() calls.
+	* include/debug/unordered_set (unordered_set(unordered_set&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(unordered_set::operator=(initializer_list<>), unordered_set::merge):
+	Remove _M_base() calls.
+	(unordered_multiset(unordered_multiset&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(unordered_multiset::operator=(initializer_list<>), unordered_multiset::merge):
+	Remove _M_base() calls.
+	* include/debug/vector (vector(vector&&, const allocator_type&)):
+	Remove _M_safe() and _M_base() calls.
+	(vector::operator=(initializer_list<>)): Remove _M_base() calls.
+	(vector::operator[](size_type)): Likewise.
+
+2021-11-13  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/Makefile.am: Add spanstream header.
+	* include/Makefile.in: Regenerate.
+	* include/precompiled/stdc++.h: Add spanstream header.
+	* include/std/version (__cpp_lib_spanstream): Define.
+	* include/std/spanstream: New file.
+	* testsuite/27_io/spanstream/1.cc: New test.
+	* testsuite/27_io/spanstream/version.cc: New test.
+
+2021-11-13  Hans-Peter Nilsson  <hp@axis.com>
+
+	PR libstdc++/103166
+	* acinclude.m4 (GLIBCXX_CHECK_GETENTROPY, GLIBCXX_CHECK_ARC4RANDOM):
+	Use GCC_TRY_COMPILE_OR_LINK instead of AC_TRY_COMPILE.
+	* configure: Regenerate.
+
+2021-11-12  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/59675
+	* acinclude.m4 (libtool_VERSION): Bump version.
+	* config/abi/pre/gnu.ver (GLIBCXX_3.4.30): Add version and
+	export new symbol.
+	* configure: Regenerate.
+	* include/bits/c++config (__replacement_assert): Remove, declare
+	__glibcxx_assert_fail instead.
+	* src/c++11/debug.cc (__glibcxx_assert_fail): New function to
+	replace __replacement_assert, writing to stderr instead of
+	stdout.
+	* testsuite/util/testsuite_abi.cc: Update latest version.
+
+2021-11-12  Jonathan Wakely  <jwakely@redhat.com>
+	    Josh Marshall  <joshua.r.marshall.1991@gmail.com>
+
+	* include/bits/alloc_traits.h (_Destroy): Make constexpr for
+	C++20 mode.
+	* include/bits/allocator.h (__shrink_to_fit::_S_do_it):
+	Likewise.
+	* include/bits/stl_algobase.h (__fill_a1): Declare _Bit_iterator
+	overload constexpr for C++20.
+	* include/bits/stl_bvector.h (_Bit_type, _S_word_bit): Move out
+	of inline namespace.
+	(_Bit_reference, _Bit_iterator_base, _Bit_iterator)
+	(_Bit_const_iterator, _Bvector_impl_data, _Bvector_base)
+	(vector<bool, A>>): Add constexpr to every member function.
+	(_Bvector_base::_M_allocate): Initialize storage during constant
+	evaluation.
+	(vector<bool, A>::_M_initialize_value): Use __fill_bvector_n
+	instead of memset.
+	(__fill_bvector_n): New helper function to replace memset during
+	constant evaluation.
+	* include/bits/stl_uninitialized.h (__uninitialized_copy<false>):
+	Move logic to ...
+	(__do_uninit_copy): New function.
+	(__uninitialized_fill<false>): Move logic to ...
+	(__do_uninit_fill): New function.
+	(__uninitialized_fill_n<false>): Move logic to ...
+	(__do_uninit_fill_n): New function.
+	(__uninitialized_copy_a): Add constexpr. Use __do_uninit_copy.
+	(__uninitialized_move_a, __uninitialized_move_if_noexcept_a):
+	Add constexpr.
+	(__uninitialized_fill_a): Add constexpr. Use __do_uninit_fill.
+	(__uninitialized_fill_n_a): Add constexpr. Use
+	__do_uninit_fill_n.
+	(__uninitialized_default_n, __uninitialized_default_n_a)
+	(__relocate_a_1, __relocate_a): Add constexpr.
+	* include/bits/stl_vector.h (_Vector_impl_data, _Vector_impl)
+	(_Vector_base, vector): Add constexpr to every member function.
+	(_Vector_impl::_S_adjust): Disable ASan annotation during
+	constant evaluation.
+	(_Vector_base::_S_use_relocate): Disable bitwise-relocation
+	during constant evaluation.
+	(vector::_Temporary_value): Use a union for storage.
+	* include/bits/vector.tcc (vector, vector<bool>): Add constexpr
+	to every member function.
+	* include/std/vector (erase_if, erase): Add constexpr.
+	* testsuite/23_containers/headers/vector/synopsis.cc: Add
+	constexpr for C++20 mode.
+	* testsuite/23_containers/vector/bool/cmp_c++20.cc: Change to
+	compile-only test using constant expressions.
+	* testsuite/23_containers/vector/bool/capacity/29134.cc: Adjust
+	namespace for _S_word_bit.
+	* testsuite/23_containers/vector/bool/modifiers/insert/31370.cc:
+	Likewise.
+	* testsuite/23_containers/vector/cmp_c++20.cc: Likewise.
+	* testsuite/23_containers/vector/cons/89164.cc: Adjust errors
+	for C++20 and move C++17 test to ...
+	* testsuite/23_containers/vector/cons/89164_c++17.cc: ... here.
+	* testsuite/23_containers/vector/bool/capacity/constexpr.cc: New test.
+	* testsuite/23_containers/vector/bool/cons/constexpr.cc: New test.
+	* testsuite/23_containers/vector/bool/element_access/constexpr.cc: New test.
+	* testsuite/23_containers/vector/bool/modifiers/assign/constexpr.cc: New test.
+	* testsuite/23_containers/vector/bool/modifiers/constexpr.cc: New test.
+	* testsuite/23_containers/vector/bool/modifiers/swap/constexpr.cc: New test.
+	* testsuite/23_containers/vector/capacity/constexpr.cc: New test.
+	* testsuite/23_containers/vector/cons/constexpr.cc: New test.
+	* testsuite/23_containers/vector/data_access/constexpr.cc: New test.
+	* testsuite/23_containers/vector/element_access/constexpr.cc: New test.
+	* testsuite/23_containers/vector/modifiers/assign/constexpr.cc: New test.
+	* testsuite/23_containers/vector/modifiers/constexpr.cc: New test.
+	* testsuite/23_containers/vector/modifiers/swap/constexpr.cc: New test.
+
+2021-11-11  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/debug/deque (deque::operator=(const deque&)): Remove
+	definition.
+	* include/debug/list (list::operator=(const list&)): Likewise.
+	* include/debug/map.h (map::operator=(const map&)): Likewise.
+	* include/debug/multimap.h (multimap::operator=(const multimap&)):
+	Likewise.
+	* include/debug/multiset.h (multiset::operator=(const multiset&)):
+	Likewise.
+	* include/debug/set.h (set::operator=(const set&)): Likewise.
+	* include/debug/string (basic_string::operator=(const basic_string&)):
+	Likewise.
+	* include/debug/vector (vector::operator=(const vector&)):
+	Likewise.
+	(_Safe_vector::operator=(const _Safe_vector&)): Define for
+	C++98 as well.
+
+2021-11-11  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/memory_resource (memory_resource::allocate):
+	Implicitly create objects in the returned storage.
+
+2021-11-11  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/stl_bvector.h (vector<bool>::data()): Give
+	protected access, and delete for C++11 and later.
+
+2021-11-10  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/100117
+	* testsuite/17_intro/headers/c++1998/49745.cc: Explicitly list
+	all C++ headers instead of including <bits/stdc++.h>
+
+2021-11-10  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/100748
+	PR libstdc++/103133
+	* config/os/gnu-linux/os_defines.h (_GLIBCXX_GTHREAD_USE_WEAK):
+	Define for glibc 2.34 and later.
+
+2021-11-09  François Dumont  <fdumont@gcc.gnu.org>
+
+	* include/bits/hashtable_policy.h (__distance_fw): Replace class keyword with
+	typename.
+	* include/bits/hashtable.h (_Hashtable<>::_M_merge_unique): Remove noexcept
+	qualification. Use const_iterator for node extraction/reinsert.
+	(_Hashtable<>::_M_merge_multi): Likewise. Compute new hash code before extract.
+	* include/debug/safe_container.h (_Safe_container<>): Make all methods
+	protected.
+	* include/debug/safe_unordered_container.h
+	(_Safe_unordered_container<>::_UContInvalidatePred<_ExtractKey, _Source>): New.
+	(_Safe_unordered_container<>::_UMContInvalidatePred<_ExtractKey, _Source>): New.
+	(_Safe_unordered_container<>::_UContMergeGuard<_Source, _InvalidatePred>): New.
+	(_Safe_unordered_container<>::_S_uc_guard<_ExtractKey, _Source>): New.
+	(_Safe_unordered_container<>::_S_umc_guard<_ExtractKey, _Source>): New.
+	(_Safe_unordered_container<>::_M_invalide_all): Make public.
+	(_Safe_unordered_container<>::_M_invalide_if): Likewise.
+	(_Safe_unordered_container<>::_M_invalide_local_if): Likewise.
+	* include/debug/unordered_map
+	(unordered_map<>::mapped_type, pointer, const_pointer): New typedef.
+	(unordered_map<>::reference, const_reference, difference_type): New typedef.
+	(unordered_map<>::get_allocator, empty, size, max_size): Add usings.
+	(unordered_map<>::bucket_count, max_bucket_count, bucket): Add usings.
+	(unordered_map<>::hash_function, key_equal, count, contains): Add usings.
+	(unordered_map<>::operator[], at, rehash, reserve): Add usings.
+	(unordered_map<>::merge): New.
+	(unordered_multimap<>::mapped_type, pointer, const_pointer): New typedef.
+	(unordered_multimap<>::reference, const_reference, difference_type): New typedef.
+	(unordered_multimap<>::get_allocator, empty, size, max_size): Add usings.
+	(unordered_multimap<>::bucket_count, max_bucket_count, bucket): Add usings.
+	(unordered_multimap<>::hash_function, key_equal, count, contains): Add usings.
+	(unordered_multimap<>::rehash, reserve): Add usings.
+	(unordered_multimap<>::merge): New.
+	* include/debug/unordered_set
+	(unordered_set<>::mapped_type, pointer, const_pointer): New typedef.
+	(unordered_set<>::reference, const_reference, difference_type): New typedef.
+	(unordered_set<>::get_allocator, empty, size, max_size): Add usings.
+	(unordered_set<>::bucket_count, max_bucket_count, bucket): Add usings.
+	(unordered_set<>::hash_function, key_equal, count, contains): Add usings.
+	(unordered_set<>::rehash, reserve): Add usings.
+	(unordered_set<>::merge): New.
+	(unordered_multiset<>::mapped_type, pointer, const_pointer): New typedef.
+	(unordered_multiset<>::reference, const_reference, difference_type): New typedef.
+	(unordered_multiset<>::get_allocator, empty, size, max_size): Add usings.
+	(unordered_multiset<>::bucket_count, max_bucket_count, bucket): Add usings.
+	(unordered_multiset<>::hash_function, key_equal, count, contains): Add usings.
+	(unordered_multiset<>::rehash, reserve): Add usings.
+	(unordered_multiset<>::merge): New.
+	* testsuite/23_containers/unordered_map/debug/merge1_neg.cc: New test.
+	* testsuite/23_containers/unordered_map/debug/merge2_neg.cc: New test.
+	* testsuite/23_containers/unordered_map/debug/merge3_neg.cc: New test.
+	* testsuite/23_containers/unordered_map/debug/merge4_neg.cc: New test.
+	* testsuite/23_containers/unordered_multimap/debug/merge1_neg.cc: New test.
+	* testsuite/23_containers/unordered_multimap/debug/merge2_neg.cc: New test.
+	* testsuite/23_containers/unordered_multimap/debug/merge3_neg.cc: New test.
+	* testsuite/23_containers/unordered_multimap/debug/merge4_neg.cc: New test.
+	* testsuite/23_containers/unordered_multiset/debug/merge1_neg.cc: New test.
+	* testsuite/23_containers/unordered_multiset/debug/merge2_neg.cc: New test.
+	* testsuite/23_containers/unordered_multiset/debug/merge3_neg.cc: New test.
+	* testsuite/23_containers/unordered_multiset/debug/merge4_neg.cc: New test.
+	* testsuite/23_containers/unordered_set/debug/merge1_neg.cc: New test.
+	* testsuite/23_containers/unordered_set/debug/merge2_neg.cc: New test.
+	* testsuite/23_containers/unordered_set/debug/merge3_neg.cc: New test.
+	* testsuite/23_containers/unordered_set/debug/merge4_neg.cc: New test.
+	* testsuite/util/testsuite_abi.h: [_GLIBCXX_DEBUG] Use normal unordered
+	container implementation.
+
+2021-11-09  Jonathan Wakely  <jwakely@redhat.com>
+
+	* testsuite/26_numerics/random/random_device/cons/token.cc:
+	Print results of random_device_available checks.
+
+2021-11-09  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/103146
+	* src/c++11/random.cc: Check __powerpc64__ not __powerpc__.
+
+2021-11-09  Jonathan Wakely  <jwakely@redhat.com>
+
+	* acinclude.m4 (GLIBCXX_CHECK_GETENTROPY, GLIBCXX_CHECK_ARC4RANDOM):
+	Define.
+	* configure.ac (GLIBCXX_CHECK_GETENTROPY, GLIBCXX_CHECK_ARC4RANDOM):
+	Use them.
+	* config.h.in: Regenerate.
+	* configure: Regenerate.
+	* src/c++11/random.cc (random_device): Add getentropy and
+	arc4random as sources.
+	* testsuite/26_numerics/random/random_device/cons/token.cc:
+	Check new tokens.
+	* testsuite/26_numerics/random/random_device/entropy.cc:
+	Likewise.
+
+2021-11-09  Jonathan Wakely  <jwakely@redhat.com>
+
+	* testsuite/26_numerics/random/random_device/cons/token.cc:
+	Retry if random devices produce the same value.
+
+2021-11-09  Rasmus Villemoes  <rasmus.villemoes@prevas.dk>
+
+	* config/os/vxworks/os_defines.h (_GLIBCXX_HAVE_TLS): Only
+	define for VxWorks >= 6.6.
+
+2021-11-05  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/range_access.h (begin(valarray), end(valarray)):
+	Add noexcept.
+
+2021-11-05  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/tuple (tuple_size_v): Fix pack expansion.
+
+2021-11-05  Jonathan Wakely  <jwakely@redhat.com>
+
+	* src/c++11/random.cc (__x86_rdrand, __x86_rdseed): Add
+	[[unlikely]] attribute.
+
+2021-11-05  Jonathan Wakely  <jwakely@redhat.com>
+
+	* src/c++11/random.cc [__powerpc__] (USE_DARN): Define.
+	(__ppc_darn): New function to use POWER9 DARN instruction.
+	(Which): Add 'darn' enumerator.
+	(which_source): Check for __ppc_darn.
+	(random_device::_M_init): Support "darn" and "hw" tokens.
+	(random_device::_M_getentropy): Add darn to switch.
+	* testsuite/26_numerics/random/random_device/cons/token.cc:
+	Check "darn" token.
+	* testsuite/26_numerics/random/random_device/entropy.cc:
+	Likewise.
+
+2021-11-05  Jonathan Wakely  <jwakely@redhat.com>
+
+	* testsuite/lib/gdb-test.exp: Add target selector support to the
+	dg-final directives.
+	* testsuite/libstdc++-prettyprinters/80276.cc: Add xfail for
+	C++20.
+	* testsuite/libstdc++-prettyprinters/libfundts.cc: Likewise.
+	* testsuite/libstdc++-prettyprinters/prettyprinters.exp: Tweak
+	comment.
+
+2021-11-04  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/103086
+	* python/libstdcxx/v6/printers.py (_tuple_impl_get): New helper
+	for accessing the tuple element stored in a _Tuple_impl node.
+	(tuple_get): New function for accessing a tuple element.
+	(unique_ptr_get): New function for accessing a unique_ptr.
+	(UniquePointerPrinter, StdPathPrinter): Use unique_ptr_get.
+	* python/libstdcxx/v6/xmethods.py (UniquePtrGetWorker): Cast
+	tuple to its base class before accessing _M_head_impl.
+
+2021-11-04  Jonathan Wakely  <jwakely@redhat.com>
+
+	* doc/xml/manual/evolution.xml: Document deprecations.
+	* doc/html/*: Regenerate.
+	* libsupc++/exception (unexpected_handler, unexpected)
+	(get_unexpected, set_unexpected): Add deprecated attribute.
+	Do not define without _GLIBCXX_USE_DEPRECATED for C++17 and up.
+	* libsupc++/eh_personality.cc (PERSONALITY_FUNCTION): Disable
+	deprecated warnings.
+	* libsupc++/eh_ptr.cc (std::rethrow_exception): Likewise.
+	* libsupc++/eh_terminate.cc: Likewise.
+	* libsupc++/eh_throw.cc (__cxa_init_primary_exception):
+	Likewise.
+	* libsupc++/unwind-cxx.h (struct __cxa_exception): Use
+	terminate_handler instead of unexpected_handler.
+	(struct __cxa_dependent_exception): Likewise.
+	(__unexpected): Likewise.
+	* testsuite/18_support/headers/exception/synopsis.cc: Add
+	dg-warning for deprecated warning.
+	* testsuite/18_support/exception_ptr/60612-unexpected.cc:
+	Disable deprecated warnings.
+	* testsuite/18_support/set_unexpected.cc: Likewise.
+	* testsuite/18_support/unexpected_handler.cc: Likewise.
+
+2021-11-04  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/utility.h (__find_uniq_type_in_pack): Move
+	definition to here, ...
+	* include/std/tuple (__find_uniq_type_in_pack): ... from here.
+	* include/std/variant (__detail__variant::__index_of): Remove.
+	(__detail::__variant::__exactly_once): Define using
+	__find_uniq_type_in_pack instead of __index_of.
+	(get<T>, get_if<T>, variant::__index_of): Likewise.
+
+2021-11-04  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/stl_pair.h (tuple_size_v): Define partial
+	specializations for std::pair.
+	* include/bits/utility.h (_Nth_type): Move definition here
+	and define primary template.
+	(tuple_size_v): Move definition here.
+	* include/std/array (tuple_size_v): Define partial
+	specializations for std::array.
+	* include/std/tuple (tuple_size_v): Move primary template to
+	<bits/utility.h>.  Define partial specializations for
+	std::tuple.
+	(tuple_element): Change definition to use _Nth_type.
+	* include/std/variant (_Nth_type): Move to <bits/utility.h>.
+	(variant_alternative, variant): Adjust qualification of
+	_Nth_type.
+	* testsuite/20_util/tuple/element_access/get_neg.cc: Prune
+	additional errors from _Nth_type.
+
+2021-11-04  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/variant (__detail::__variant::__emplace): New
+	function template.
+	(_Copy_assign_base::operator=): Reorder conditions to match
+	bulleted list of effects in the standard. Use __emplace instead
+	of _M_reset followed by _Construct.
+	(_Move_assign_base::operator=): Likewise.
+	(__construct_by_index): Remove.
+	(variant::emplace): Use __emplace instead of _M_reset followed
+	by __construct_by_index.
+	(variant::swap): Hoist valueless cases out of visitor. Use
+	__emplace to replace _M_reset followed by _Construct.
+
+2021-11-04  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/variant (_Nth_type): Define partial
+	specializations to reduce number of instantiations.
+	(variant_size_v): Define partial specializations to avoid
+	instantiations.
+	(variant_alternative): Use _Nth_type. Add static assert.
+	(__tuple_count, __tuple_count_v): Replace with ...
+	(__count): New variable template.
+	(_Variant_union): Add deleted constructor.
+	(variant::__to_type): Use _Nth_type.
+	(variant::emplace): Use _Nth_type. Add deleted overloads for
+	invalid types and indices.
+
+2021-11-04  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/102912
+	* include/std/variant (_Variant_storage::__index_of): Remove.
+	(__variant_construct_single): Remove.
+	(__variant_construct): Remove.
+	(_Copy_ctor_base::_Copy_ctor_base(const _Copy_ctor_base&)): Do
+	construction directly instead of using __variant_construct.
+	(_Move_ctor_base::_Move_ctor_base(_Move_ctor_base&&)): Likewise.
+	(_Move_ctor_base::_M_destructive_move()): Remove.
+	(_Move_ctor_base::_M_destructive_copy()): Remove.
+	(_Copy_assign_base::operator=(const _Copy_assign_base&)): Do
+	construction directly instead of using _M_destructive_copy.
+	(variant::swap): Do construction directly instead of using
+	_M_destructive_move.
+	* testsuite/20_util/variant/102912.cc: New test.
+
+2021-11-03  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/66742
+	* include/bits/list.tcc (list::sort): Use mutable iterators for
+	comparisons.
+	* include/bits/stl_list.h (_Scratch_list::_Ptr_cmp): Likewise.
+	* testsuite/23_containers/list/operations/66742.cc: Check
+	non-const comparisons.
+
+2021-11-03  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/valarray (valarray::valarray()): Add noexcept.
+	(valarray::operator[]): Likewise.
+
+2021-11-01  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/debug/stl_iterator.h (__valid_range): Add constexpr
+	for C++20. Qualify call to avoid ADL.
+	(__get_distance, __can_advance, __unsafe, __base): Likewise.
+	* testsuite/25_algorithms/move/constexpr.cc: Also check with
+	std::reverse_iterator arguments.
+
+2021-11-01  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/span (span(Range&&)): Reorder constraints.
+
+2021-11-01  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/103022
+	* include/std/valarray (begin, end): Do not dereference an empty
+	valarray. Add noexcept and [[nodiscard]].
+	* testsuite/26_numerics/valarray/range_access.cc: Check empty
+	valarray. Check iterator properties. Run as well as compiling.
+	* testsuite/26_numerics/valarray/range_access2.cc: Likewise.
+	* testsuite/26_numerics/valarray/103022.cc: New test.
+
+2021-10-29  Jonathan Wakely  <jwakely@redhat.com>
+
+	* testsuite/23_containers/stack/deduction.cc: Fix typo.
+
+2021-10-26  Martin Sebor  <msebor@redhat.com>
+
+	* testsuite/21_strings/basic_string/capacity/1.cc: Also suppress
+	-Wstringop-overread.
+	* testsuite/27_io/filesystem/path/factory/u8path-char8_t.cc: Same.
+
+2021-10-26  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* testsuite/28_regex/basic_regex/84110.cc (test01)
+	[__cpp_exceptions]: Disambiguate extended.
+
+2021-10-26  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* testsuite/17_intro/names.cc [__sun__] (r, x): Undef.
+
+2021-10-22  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/102894
+	* include/std/any (make_any): Add SFINAE constraint.
+	* testsuite/20_util/any/102894.cc: New test.
+
+2021-10-21  Jonathan Wakely  <jwakely@redhat.com>
+
+	* doc/doxygen/stdheader.cc: Refactor. Use C++23. Add new
+	headers.
+	* scripts/run_doxygen: Fix post-processing of #include
+	directives in man pages. Use new xg++ to compile helper program.
+
+2021-10-21  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/mofunc_impl.h: Add doxygen comments.
+	* include/std/functional: Likewise.
+
+2021-10-21  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/alloc_traits.h: Suppress doxygen documentation.
+	* include/bits/allocated_ptr.h: Likewise.
+	* include/bits/enable_special_members.h: Likewise.
+	* include/bits/hashtable.h: Likewise.
+	* include/bits/hashtable_policy.h: Likewise.
+	* include/bits/uses_allocator.h: Likewise.
+	* include/bits/node_handle.h: Document node handles and suppress
+	documentation for protected members.
+	* include/std/any: Suppress documentation for implementation
+	details.
+
+2021-10-21  Patrick Palka  <ppalka@redhat.com>
+
+	PR libstdc++/102358
+	* include/bits/stl_iterator.h (__niter_base): Make constexpr
+	for C++20.
+	(__miter_base): Likewise.
+	* testsuite/25_algorithms/move/constexpr.cc: New test.
+
+2021-10-21  Patrick Palka  <ppalka@redhat.com>
+
+	* include/std/ranges (istream_view): Replace this function
+	template with an alias template as per P2432R1.
+	(wistream_view): Define as per P2432R1.
+	(views::_Istream, views::istream): Likewise.
+	* testsuite/std/ranges/istream_view.cc (test07): New test.
+
+2021-10-21  Patrick Palka  <ppalka@redhat.com>
+
+	* include/bits/ranges_util.h (views::_Drop): Forward declare.
+	(subrange): Befriend views::_Drop.
+	(subrange::_S_store_size): Declare constexpr instead of just
+	const, remove obsolete comment.
+	* include/std/ranges (views::__detail::__is_empty_view): Define.
+	(views::__detail::__is_basic_string_view): Likewise.
+	(views::__detail::__is_subrange): Likewise.
+	(views::__detail::__is_iota_view): Likewise.
+	(views::__detail::__can_take_view): Rename template parm _Tp to _Dp.
+	(views::_Take): Rename template parm _Tp to _Dp, make it non-deducible
+	and fix it to range_difference_t<_Range>.  Implement P1739R4 and
+	LWG 3407 changes.
+	(views::__detail::__can_drop_view): Rename template parm _Tp to _Dp.
+	(views::_Drop): As with views::_Take.
+	(views::_Counted): Implement P1739R4 changes.
+	* include/std/span (__detail::__is_std_span): Rename to ...
+	(__detail::__is_span): ... this and turn it into a variable
+	template.
+	(__detail::__is_std_array): Turn it into a variable template.
+	(span::span): Adjust uses of __is_std_span and __is_std_array
+	accordingly.
+	* testsuite/std/ranges/adaptors/p1739.cc: New test.
+
+2021-10-21  Patrick Palka  <ppalka@redhat.com>
+
+	* include/bits/stl_iterator.h (common_iterator::__arrow_proxy):
+	Make fully constexpr as per LWG 3595.
+	(common_iterator::__postfix_proxy): Likewise.
+
+2021-10-21  Patrick Palka  <ppalka@redhat.com>
+
+	* include/std/ranges (lazy_split_view::base): Add forward_range
+	constraint as per LWG 3591.
+	(lazy_split_view::begin, lazy_split_view::end): Also check
+	simpleness of _Pattern as per LWG 3592.
+	(split_view::base): Relax copyable constraint as per LWG 3590.
+
+2021-10-21  Patrick Palka  <ppalka@redhat.com>
+
+	* include/std/ranges (join_view::__iter_cat::_S_iter_cat): Adjust
+	criteria for returning bidirectional_iterator_tag as per LWG 3535.
+	(join_view::_Iterator::_S_iter_concept): Likewise.
+
+2021-10-21  Patrick Palka  <ppalka@redhat.com>
+
+	* include/bits/ranges_base.h (viewable_range): Adjust as per
+	LWG 3481.
+	* testsuite/std/ranges/adaptors/all.cc (test07): New test.
+
+2021-10-21  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/102863
+	* include/std/optional (optional::and_then, optional::transform):
+	Remove requires-clause.
+	* testsuite/20_util/optional/monadic/and_then.cc: Check
+	overload resolution doesn't cause errors.
+	* testsuite/20_util/optional/monadic/transform.cc: Likewise.
+
 2021-10-20  Jonathan Wakely  <jwakely@redhat.com>
 
 	* testsuite/20_util/optional/monadic/transform.cc: Check that
diff --git a/libstdc++-v3/acinclude.m4 b/libstdc++-v3/acinclude.m4
index 90ecc4a..30bd92d 100644
--- a/libstdc++-v3/acinclude.m4
+++ b/libstdc++-v3/acinclude.m4
@@ -3798,7 +3798,7 @@ changequote([,])dnl
 fi
 
 # For libtool versioning info, format is CURRENT:REVISION:AGE
-libtool_VERSION=6:29:0
+libtool_VERSION=6:30:0
 
 # Everything parsed; figure out what files and settings to use.
 case $enable_symvers in
@@ -4830,6 +4830,52 @@ AC_DEFUN([GLIBCXX_CHECK_EXCEPTION_PTR_SYMVER], [
   fi
 ])
 
+dnl
+dnl Check whether getentropy is present in <unistd.h>.
+dnl
+AC_DEFUN([GLIBCXX_CHECK_GETENTROPY], [
+
+  AC_LANG_SAVE
+  AC_LANG_CPLUSPLUS
+  AC_MSG_CHECKING([for getentropy])
+  AC_CACHE_VAL(glibcxx_cv_getentropy, [
+      GCC_TRY_COMPILE_OR_LINK(
+	[#include <unistd.h>],
+	[unsigned i;
+	 ::getentropy(&i, sizeof(i));],
+	[glibcxx_cv_getentropy=yes], [glibcxx_cv_getentropy=no])
+    ])
+
+  if test $glibcxx_cv_getentropy = yes; then
+    AC_DEFINE(HAVE_GETENTROPY, 1, [Define if getentropy is available in <unistd.h>.])
+  fi
+  AC_MSG_RESULT($glibcxx_cv_getentropy)
+  AC_LANG_RESTORE
+])
+
+dnl
+dnl Check whether arc4random is present in <stdlib.h>.
+dnl
+AC_DEFUN([GLIBCXX_CHECK_ARC4RANDOM], [
+
+  AC_LANG_SAVE
+  AC_LANG_CPLUSPLUS
+  AC_MSG_CHECKING([for arc4random])
+  AC_CACHE_VAL(glibcxx_cv_arc4random, [
+      GCC_TRY_COMPILE_OR_LINK(
+	[#include <stdlib.h>],
+	[unsigned i = ::arc4random();],
+	[glibcxx_cv_arc4random=yes], [glibcxx_cv_arc4random=no])
+    ])
+
+  if test $glibcxx_cv_arc4random = yes; then
+    AC_DEFINE(HAVE_ARC4RANDOM, 1, [Define if arc4random is available in <stdlib.h>.])
+  fi
+  AC_MSG_RESULT($glibcxx_cv_arc4random)
+  AC_LANG_RESTORE
+])
+
+
 # Macros from the top-level gcc directory.
 m4_include([../config/gc++filt.m4])
 m4_include([../config/tls.m4])
diff --git a/libstdc++-v3/config.h.in b/libstdc++-v3/config.h.in
index 228a758..420021f 100644
--- a/libstdc++-v3/config.h.in
+++ b/libstdc++-v3/config.h.in
@@ -9,6 +9,9 @@
 /* Define to 1 if you have the `aligned_alloc' function. */
 #undef HAVE_ALIGNED_ALLOC
 
+/* Define if arc4random is available in <stdlib.h>. */
+#undef HAVE_ARC4RANDOM
+
 /* Define to 1 if you have the <arpa/inet.h> header file. */
 #undef HAVE_ARPA_INET_H
 
@@ -132,6 +135,9 @@
 /* Define to 1 if you have the `frexpl' function. */
 #undef HAVE_FREXPL
 
+/* Define if getentropy is available in <unistd.h>. */
+#undef HAVE_GETENTROPY
+
 /* Define if _Unwind_GetIPInfo is available. */
 #undef HAVE_GETIPINFO
 
diff --git a/libstdc++-v3/config/abi/pre/gnu.ver b/libstdc++-v3/config/abi/pre/gnu.ver
index 5323c7f..8f3c7b3 100644
--- a/libstdc++-v3/config/abi/pre/gnu.ver
+++ b/libstdc++-v3/config/abi/pre/gnu.ver
@@ -2397,6 +2397,12 @@ GLIBCXX_3.4.29 {
 
 } GLIBCXX_3.4.28;
 
+GLIBCXX_3.4.30 {
+
+    _ZSt21__glibcxx_assert_fail*;
+
+} GLIBCXX_3.4.29;
+
 # Symbols in the support library (libsupc++) have their own tag.
 CXXABI_1.3 {
 
diff --git a/libstdc++-v3/config/os/gnu-linux/os_defines.h b/libstdc++-v3/config/os/gnu-linux/os_defines.h
index d5bb2a1..3a05345 100644
--- a/libstdc++-v3/config/os/gnu-linux/os_defines.h
+++ b/libstdc++-v3/config/os/gnu-linux/os_defines.h
@@ -61,4 +61,10 @@
   (__gthread_active_p() ? __gthread_self() : (__gthread_t)1)
 #endif
 
+#if __GLIBC_PREREQ(2, 34)
+// Since glibc 2.34 all pthreads functions are usable without linking to
+// libpthread.
+# define _GLIBCXX_GTHREAD_USE_WEAK 0
+#endif
+
 #endif
diff --git a/libstdc++-v3/config/os/vxworks/os_defines.h b/libstdc++-v3/config/os/vxworks/os_defines.h
index c881b2b..75a68bc 100644
--- a/libstdc++-v3/config/os/vxworks/os_defines.h
+++ b/libstdc++-v3/config/os/vxworks/os_defines.h
@@ -45,8 +45,10 @@
 #define _GLIBCXX_USE_WEAK_REF 0
 #endif
 
-// We support TLS on VxWorks (either directly or with emutls)
+// We support TLS on VxWorks >= 6.6 (either directly or with emutls)
+#if !_VXWORKS_PRE(6, 6)
 #define _GLIBCXX_HAVE_TLS 1
+#endif
 
 // VxWorks7 comes with a DinkumWare library and the system headers which we
 // are going to include for libstdc++ have a few related intrinsic
diff --git a/libstdc++-v3/configure b/libstdc++-v3/configure
index c1aea82..3eb391f 100755
--- a/libstdc++-v3/configure
+++ b/libstdc++-v3/configure
@@ -74892,7 +74892,7 @@ $as_echo "$as_me: WARNING: === Symbol versioning will be disabled." >&2;}
 fi
 
 # For libtool versioning info, format is CURRENT:REVISION:AGE
-libtool_VERSION=6:29:0
+libtool_VERSION=6:30:0
 
 # Everything parsed; figure out what files and settings to use.
 case $enable_symvers in
@@ -75429,6 +75429,158 @@ $as_echo "#define _GLIBCXX_X86_RDSEED 1" >>confdefs.h
 $as_echo "$ac_cv_x86_rdseed" >&6; }
 
 
+# Check for other random number APIs
+
+
+
+  ac_ext=cpp
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for getentropy" >&5
+$as_echo_n "checking for getentropy... " >&6; }
+  if ${glibcxx_cv_getentropy+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+
+      if test x$gcc_no_link = xyes; then
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <unistd.h>
+int
+main ()
+{
+unsigned i;
+	 ::getentropy(&i, sizeof(i));
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_cxx_try_compile "$LINENO"; then :
+  glibcxx_cv_getentropy=yes
+else
+  glibcxx_cv_getentropy=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  if test x$gcc_no_link = xyes; then
+  as_fn_error $? "Link tests are not allowed after GCC_NO_EXECUTABLES." "$LINENO" 5
+fi
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <unistd.h>
+int
+main ()
+{
+unsigned i;
+	 ::getentropy(&i, sizeof(i));
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_cxx_try_link "$LINENO"; then :
+  glibcxx_cv_getentropy=yes
+else
+  glibcxx_cv_getentropy=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+fi
+
+fi
+
+
+  if test $glibcxx_cv_getentropy = yes; then
+
+$as_echo "#define HAVE_GETENTROPY 1" >>confdefs.h
+
+  fi
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $glibcxx_cv_getentropy" >&5
+$as_echo "$glibcxx_cv_getentropy" >&6; }
+  ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+
+
+  ac_ext=cpp
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for arc4random" >&5
+$as_echo_n "checking for arc4random... " >&6; }
+  if ${glibcxx_cv_arc4random+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+
+      if test x$gcc_no_link = xyes; then
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <stdlib.h>
+int
+main ()
+{
+unsigned i = ::arc4random();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_cxx_try_compile "$LINENO"; then :
+  glibcxx_cv_arc4random=yes
+else
+  glibcxx_cv_arc4random=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  if test x$gcc_no_link = xyes; then
+  as_fn_error $? "Link tests are not allowed after GCC_NO_EXECUTABLES." "$LINENO" 5
+fi
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <stdlib.h>
+int
+main ()
+{
+unsigned i = ::arc4random();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_cxx_try_link "$LINENO"; then :
+  glibcxx_cv_arc4random=yes
+else
+  glibcxx_cv_arc4random=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+fi
+
+fi
+
+
+  if test $glibcxx_cv_arc4random = yes; then
+
+$as_echo "#define HAVE_ARC4RANDOM 1" >>confdefs.h
+
+  fi
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $glibcxx_cv_arc4random" >&5
+$as_echo "$glibcxx_cv_arc4random" >&6; }
+  ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
 # This depends on GLIBCXX_ENABLE_SYMVERS and GLIBCXX_IS_NATIVE.
 
   # Do checks for resource limit functions.
diff --git a/libstdc++-v3/configure.ac b/libstdc++-v3/configure.ac
index 2d68b36..5b3c92f 100644
--- a/libstdc++-v3/configure.ac
+++ b/libstdc++-v3/configure.ac
@@ -468,6 +468,10 @@ GLIBCXX_CHECK_X86_RDRAND
 # Check if assembler supports rdseed opcode.
 GLIBCXX_CHECK_X86_RDSEED
 
+# Check for other random number APIs
+GLIBCXX_CHECK_GETENTROPY
+GLIBCXX_CHECK_ARC4RANDOM
+
 # This depends on GLIBCXX_ENABLE_SYMVERS and GLIBCXX_IS_NATIVE.
 GLIBCXX_CONFIGURE_TESTSUITE
 
diff --git a/libstdc++-v3/doc/doxygen/stdheader.cc b/libstdc++-v3/doc/doxygen/stdheader.cc
index 8bcb1a0..67f54ee 100644
--- a/libstdc++-v3/doc/doxygen/stdheader.cc
+++ b/libstdc++-v3/doc/doxygen/stdheader.cc
@@ -1,171 +1,256 @@
-// This is a slow larval-stage kludge to help massage the generated man
-// pages.  It's used like this:
-const char* const usage = 
-"\nTakes on stdin, whitespace-separated words of the form\n"
-"\n"
-"    [bits/]stl_foo.h\n"
-"    [bits/]std_foo.h\n"
-"\n"
-"and writes on stdout the nearest matching standard header name.\n"
-"\n"
-"Takes no command-line arguments.\n"
-"\n";
-
-#include <cstdlib>
 #include <string>
+#include <string_view>
 #include <map>
+#include <set>
+#include <algorithm>
+#include <iterator>
 #include <iostream>
 
-typedef std::map<std::string, std::string>   Map;
+// This is a slow larval-stage kludge to help massage the generated man
+// pages.  It's used like this:
+const std::string_view usage = R"(
+Takes on stdin, whitespace-separated words of the form
+
+    [bits/]stl_foo.h
+    [bits/]std_foo.h
+
+and writes on stdout the nearest matching standard header name.
 
-Map  headers;
+Takes no command-line arguments.
+)";
+
+// List of standard headers
+std::set<std::string_view> std_headers;
+// Map of partial header filenames to standard headers.
+std::map<std::string_view, std::string_view>  headers;
 
 void init_map()
 {
     // Enter the glamourous world of data entry!!  Maintain these!
+    // Because the map_header function removes common prefixes and suffixes,
+    // a header "bits/st[dl]_foo.h" will automatically map to "foo" if that
+    // is a standard header, so we don't need to list those cases here.
+    headers["atomic_base.h"]            = "atomic";
+    headers["atomic_lockfree_defines.h"] = "atomic";
+    headers["atomic_timed_wait.h"]      = "atomic";
+    headers["atomic_wait.h"]            = "atomic";
+    headers["algorithmfwd.h"]           = "algorithm";
     headers["algo.h"]                   = "algorithm";
     headers["algobase.h"]               = "algorithm";
-    headers["algorithm.h"]              = "algorithm";
+    headers["ranges_algo.h"]            = "algorithm";
+    headers["ranges_algobase.h"]        = "algorithm";
     headers["heap.h"]                   = "algorithm";
-    headers["bitset.h"]                 = "bitset";
-    headers["complex.h"]                = "complex";
-    //headers["construct.h"]              stl_construct.h entirely internal
-    headers["deque.h"]                  = "deque";
-    headers["deque.tcc"]                = "deque";
-    headers["fstream.h"]                = "fstream";
-    headers["fstream.tcc"]              = "fstream";
+    headers["exception_ptr.h"]          = "exception";
+    headers["nested_exception.h"]       = "exception";
+    headers["fs_dir.h"]                 = "filesystem";
+    headers["fs_fwd.h"]                 = "filesystem";
+    headers["fs_ops.h"]                 = "filesystem";
+    headers["fs_path.h"]                = "filesystem";
+    headers["binders.h"]                = "functional";
     headers["function.h"]               = "functional";
-    headers["functional.h"]             = "functional";
-    headers["iomanip.h"]                = "iomanip";
+    headers["functional_hash.h"]        = "functional";
+    headers["mofunc_impl.h"]            = "functional";
+    headers["move_only_function.h"]     = "functional";
+    headers["invoke.h"]                 = "functional";
+    headers["refwrap.h"]                = "functional";
+    headers["quoted_string.h"]          = "iomanip";
+    headers["ios_base.h"]               = "ios";
     headers["basic_ios.h"]              = "ios";
     headers["basic_ios.tcc"]            = "ios";
-    headers["ios.h"]                    = "ios";
     headers["iosfwd.h"]                 = "iosfwd";
     headers["iostream.h"]               = "iostream";
-    headers["istream.h"]                = "istream";
-    headers["istream.tcc"]              = "istream";
-    headers["iterator.h"]               = "iterator";
     headers["iterator_base_funcs.h"]    = "iterator";
     headers["iterator_base_types.h"]    = "iterator";
     headers["stream_iterator.h"]        = "iterator";
     headers["streambuf_iterator.h"]     = "iterator";
-    headers["limits.h"]                 = "limits";
-    headers["list.h"]                   = "list";
-    headers["list.tcc"]                 = "list";
+    headers["iterator_concepts.h"]      = "iterator";
+    headers["range_access.h"]           = "iterator";
     headers["codecvt.h"]                = "locale";
-    headers["locale.h"]                 = "locale";
+    headers["c++locale.h"]              = "locale";
     headers["localefwd.h"]              = "locale";
+    headers["ctype_base.h"]             = "locale";
     headers["locale_classes.h"]         = "locale";
+    headers["locale_classes.tcc"]       = "locale";
     headers["locale_facets.h"]          = "locale";
     headers["locale_facets.tcc"]        = "locale";
-    headers["map.h"]                    = "map";
+    headers["locale_facets_nonio.h"]    = "locale";
+    headers["locale_facets_nonio.tcc"]  = "locale";
+    headers["locale_conv.h"]            = "locale";
     headers["multimap.h"]               = "map";
-    headers["memory.h"]                 = "memory";
+    headers["memoryfwd.h"]              = "memory";
+    headers["align.h"]                  = "memory";
+    headers["alloc_traits.h"]           = "memory";
+    headers["auto_ptr.h"]		= "memory";
+    headers["construct.h"]              = "memory";
     headers["allocator.h"]              = "memory";
     headers["raw_storage_iter.h"]       = "memory";
     headers["tempbuf.h"]                = "memory";
     headers["uninitialized.h"]          = "memory";
-    headers["numeric.h"]                = "numeric";
-    headers["ostream.h"]                = "ostream";
-    headers["ostream.tcc"]              = "ostream";
-    headers["queue.h"]                  = "queue";
-    headers["set.h"]                    = "set";
+    headers["shared_ptr.h"]             = "memory";
+    headers["shared_ptr_base.h"]        = "memory";
+    headers["shared_ptr_atomic.h"]      = "memory";
+    headers["unique_ptr.h"]             = "memory";
+    headers["ranges_uninitialized.h"]   = "memory";
+    headers["ptr_traits.h"]             = "memory";
+    headers["uses_allocator.h"]         = "memory";
+    headers["uses_allocator_args.h"]    = "memory";
+    headers["unique_lock.h"]            = "mutex";
+    headers["uniform_int_dist.h"]       = "random";
+    headers["ranges_base.h"]            = "ranges";
+    headers["ranges_util.h"]            = "ranges";
+    headers["ranges_cmp.h"]             = "functional";
+    headers["regex_automaton.h"]        = "regex";
+    headers["regex_automaton.tcc"]      = "regex";
+    headers["regex_compiler.h"]         = "regex";
+    headers["regex_compiler.tcc"]       = "regex";
+    headers["regex_constants.h"]        = "regex";
+    headers["regex_error.h"]            = "regex";
+    headers["regex_executor.h"]         = "regex";
+    headers["regex_executor.tcc"]       = "regex";
+    headers["regex_scanner.h"]          = "regex";
+    headers["regex_scanner.tcc"]        = "regex";
+    headers["semaphore_base.h"]         = "semaphore";
     headers["multiset.h"]               = "set";
-    headers["sstream.h"]                = "sstream";
-    headers["sstream.tcc"]              = "sstream";
-    headers["stack.h"]                  = "stack";
+    headers["node_handle.h"]            = "set";
     headers["functexcept.h"]            = "stdexcept";
-    headers["stdexcept.h"]              = "stdexcept";
-    headers["streambuf.h"]              = "streambuf";
-    headers["streambuf.tcc"]            = "streambuf";
-    headers["string.h"]                 = "string";
     headers["char_traits.h"]            = "string";
+    headers["stringfwd.h"]              = "string";
     headers["postypes.h"]               = "string";
     headers["basic_string.h"]           = "string";
     headers["basic_string.tcc"]         = "string";
-    headers["tree.h"]                   = "backward/tree.h";
+    headers["cow_string.h"]             = "string";
+    headers["string_view.tcc"]          = "string_view";
+    headers["this_thread_sleep.h"]      = "thread";
+    headers["tree.h"]                   = "map";
     headers["pair.h"]                   = "utility";
-    headers["utility.h"]                = "utility";
     headers["relops.h"]                 = "utility";
     headers["gslice.h"]                 = "valarray";
     headers["gslice_array.h"]           = "valarray";
     headers["indirect_array.h"]         = "valarray";
     headers["mask_array.h"]             = "valarray";
     headers["slice_array.h"]            = "valarray";
-    headers["valarray.h"]               = "valarray";
     headers["valarray_after.h"]         = "valarray";
     headers["valarray_before.h"]        = "valarray";
     headers["valarray_array.h"]         = "valarray";
     headers["valarray_array.tcc"]       = "valarray";
     headers["valarray_meta.h"]          = "valarray";
     headers["bvector.h"]                = "vector";
-    headers["vector.h"]                 = "vector";
-    headers["vector.tcc"]               = "vector";
 
     //headers["concurrence.h"]             who knows
     //headers["atomicity.h"]               who knows
 
-    // C wrappers -- probably was an easier way to do these, but oh well
-    headers["cassert.h"]                = "cassert";
-    headers["cctype.h"]                 = "cctype";
-    headers["cerrno.h"]                 = "cerrno";
-    headers["cfloat.h"]                 = "cfloat";
-    headers["climits.h"]                = "climits";
-    headers["clocale.h"]                = "clocale";
-    headers["cmath.h"]                  = "cmath";
-    headers["csetjmp.h"]                = "csetjmp";
-    headers["csignal.h"]                = "csignal";
-    headers["cstdarg.h"]                = "cstdarg";
-    headers["cstddef.h"]                = "cstddef";
-    headers["cstdio.h"]                 = "cstdio";
-    headers["cstdlib.h"]                = "cstdlib";
-    headers["cstring.h"]                = "cstring";
-    headers["ctime.h"]                  = "ctime";
-    headers["cwchar.h"]                 = "cwchar";
-    headers["cwctype.h"]                = "cwctype";
+    headers["abs.h"]                    = "cstdlib";
+    headers["specfun.h"]                = "cmath";
+
+    // This list is complete as of the October 2021 working draft.
+    std_headers = {
+	"algorithm", "any", "array", "atomic",
+	"barrier", "bit", "bitset",
+	"charconv", "chrono", "codecvt", "compare", "complex",
+	"concepts", "condition_variable", "coroutine",
+	"deque",
+	"exception", "execution",
+	"filesystem", "format", "forward_list", "fstream",
+	"functional", "future",
+	"initializer_list", "iomanip", "ios", "iosfwd",
+	"iostream", "istream", "iterator",
+	"latch", "limits", "list", "locale",
+	"map", "memory", "memory_resource", "mutex",
+	"new", "numbers", "numeric",
+	"optional", "ostream",
+	"queue",
+	"random", "ranges", "ratio", "regex",
+	"scoped_allocator", "semaphore", "set", "shared_mutex",
+	"source_location", "span", "spanstream", "sstream",
+	"stack", "stacktrace", "stdexcept", "stop_token",
+	"streambuf", "string", "string_view", "strstream",
+	"syncstream", "system_error",
+	"thread", "tuple", "typeindex", "typeinfo", "type_traits",
+	"unordered_map", "unordered_set", "utility",
+	"valarray", "variant", "vector", "version",
+
+	"cassert", "cctype", "cerrno", "cfenv", "cfloat",
+	"cinttypes", "climits", "clocale", "cmath", "csetjmp",
+	"csignal", "cstdarg", "cstddef", "cstdint", "cstdio",
+	"cstdlib", "cstring", "ctime", "cuchar", "cwchar",
+	"cwctype",
+
+	"assert.h", "ctype.h", "errno.h", "fenv.h", "float.h",
+	"inttypes.h", "limits.h", "locale.h", "math.h", "setjmp.h",
+	"signal.h", "stdarg.h", "stddef.h", "stdint.h", "stdio.h",
+	"stdlib.h", "string.h", "time.h", "uchar.h", "wchar.h",
+	"wctype.h",
+    };
+
+    // In case we missed any:
+    for (const auto& h : headers)
+	std_headers.insert(h.second);
 }
 
 
-void do_word (std::string const& longheader)
+std::string_view map_header (std::string_view header)
 {
-    std::string::size_type start = 0;
-
     // if it doesn't contain a "." then it's already a std header
-    if (longheader.find(".") == std::string::npos)
+    if (!header.contains('.'))
     {
-        std::cout << longheader << '\n';
-        return;
+	// make sure it's in the set:
+	std_headers.insert(header);
+	return header;
     }
 
-    if (longheader.substr(start,5) == "bits/")  start += 5;
-    if ((longheader.substr(start,4) == "stl_") ||
-        (longheader.substr(start,4) == "std_"))
-    {
-        start += 4;
-    }
+    for (std::string_view prefix : {"bits/", "stl_", "std_"})
+	if (header.starts_with(prefix))
+	    header.remove_prefix(prefix.size());
+
+    if (auto it = headers.find(header); it != headers.end())
+	return it->second;
+
+    for (std::string_view ext : {".h", ".tcc"})
+	if (header.ends_with(ext))
+	{
+	    header.remove_suffix(ext.size());
+	    break;
+	}
+
+    if (auto it = std_headers.find(header); it != std_headers.end())
+	return *it;
+
+    return {};
+}
+
+std::string map_header_or_complain (std::string header)
+{
+    // For <experimental/xxx.h> and <tr1/xxx.h> try to map <xxx.h>
+    // then add the directory back to it.
+    if (header.contains('.'))
+	for (std::string_view dir : {"experimental/", "tr1/"})
+	    if (header.starts_with(dir))
+	    {
+		auto h = map_header(header.substr(dir.size()));
+		if (!h.empty())
+		    return std::string(dir) + std::string(h);
+		return std::string(header);
+	    }
 
-    // come on, gdb, find `p' already...
-    const char* p = longheader.substr(start).c_str();
-    Map::iterator word = headers.find(p);
-    if (word != headers.end())
-        std::cout << word->second << '\n';
-    else std::cout << "MAYBE_AN_ERROR_MESSAGE_HERE\n";
+    if (auto mapped = map_header(header); !mapped.empty())
+	return std::string(mapped);
+
+    std::cerr << "Could not map <" << header << "> to a standard header\n";
+    return std::string(header);
 }
 
 
-int main (int argc, char**)
+int main (int argc, char** argv)
 {
     if (argc > 1)
     {
-        std::cerr << usage;
-	std::exit(0);
+        std::cerr << "Usage: " << argv[0] << '\n' << usage;
+	return 1;
     }
 
     init_map();
 
-    std::string w;
-    while (std::cin >> w)
-        do_word (w);
+    std::transform(std::istream_iterator<std::string>(std::cin), {},
+		   std::ostream_iterator<std::string>(std::cout),
+		   map_header_or_complain);
 }
-
-
diff --git a/libstdc++-v3/doc/html/manual/api.html b/libstdc++-v3/doc/html/manual/api.html
index 376a403..3281078 100644
--- a/libstdc++-v3/doc/html/manual/api.html
+++ b/libstdc++-v3/doc/html/manual/api.html
@@ -440,4 +440,8 @@ now defaults to zero.
 The <code class="function">std::random_shuffle</code> algorithms are deprecated
 for C++14 and later. The C++11 <code class="function">std::shuffle</code> algorithm
 can be used instead.
+</p><p>
+The <code class="function">std::unexpected</code> function and related typedef and
+accessors for the unexpected handler are deprecated for C++11 and later.
+Dynamic exception specifications should be replaced with <code class="code">noexcept</code>.
 </p></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="abi.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="appendix_porting.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="backwards.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">ABI Policy and Guidelines </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Backwards Compatibility</td></tr></table></div></body></html>
\ No newline at end of file
diff --git a/libstdc++-v3/doc/xml/manual/evolution.xml b/libstdc++-v3/doc/xml/manual/evolution.xml
index 59b71b0..9aef84a 100644
--- a/libstdc++-v3/doc/xml/manual/evolution.xml
+++ b/libstdc++-v3/doc/xml/manual/evolution.xml
@@ -1027,6 +1027,12 @@ for C++14 and later. The C++11 <function>std::shuffle</function> algorithm
 can be used instead.
 </para>
 
+<para>
+The <function>std::unexpected</function> function and related typedef and
+accessors for the unexpected handler are deprecated for C++11 and later.
+Dynamic exception specifications should be replaced with <code>noexcept</code>.
+</para>
+
 </section>
 
 </section>
diff --git a/libstdc++-v3/include/Makefile.am b/libstdc++-v3/include/Makefile.am
index 0e43f14..25a8d9c8 100644
--- a/libstdc++-v3/include/Makefile.am
+++ b/libstdc++-v3/include/Makefile.am
@@ -76,6 +76,7 @@ std_headers = \
 	${std_srcdir}/shared_mutex \
 	${std_srcdir}/source_location \
 	${std_srcdir}/span \
+	${std_srcdir}/spanstream \
 	${std_srcdir}/sstream \
 	${std_srcdir}/syncstream \
 	${std_srcdir}/stack \
diff --git a/libstdc++-v3/include/Makefile.in b/libstdc++-v3/include/Makefile.in
index 0571a63..47a5d98 100644
--- a/libstdc++-v3/include/Makefile.in
+++ b/libstdc++-v3/include/Makefile.in
@@ -426,6 +426,7 @@ std_headers = \
 	${std_srcdir}/shared_mutex \
 	${std_srcdir}/source_location \
 	${std_srcdir}/span \
+	${std_srcdir}/spanstream \
 	${std_srcdir}/sstream \
 	${std_srcdir}/syncstream \
 	${std_srcdir}/stack \
diff --git a/libstdc++-v3/include/bits/alloc_traits.h b/libstdc++-v3/include/bits/alloc_traits.h
index 05b584f..a0a95f3 100644
--- a/libstdc++-v3/include/bits/alloc_traits.h
+++ b/libstdc++-v3/include/bits/alloc_traits.h
@@ -45,6 +45,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #if __cplusplus >= 201103L
 #define __cpp_lib_allocator_traits_is_always_equal 201411
 
+  /// @cond undocumented
   struct __allocator_traits_base
   {
     template<typename _Tp, typename _Up, typename = void>
@@ -77,10 +78,13 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   template<typename _Alloc, typename _Up>
     using __alloc_rebind
       = typename __allocator_traits_base::template __rebind<_Alloc, _Up>::type;
+  /// @endcond
 
   /**
    * @brief  Uniform interface to all allocator types.
+   * @headerfile memory
    * @ingroup allocators
+   * @since C++11
   */
   template<typename _Alloc>
     struct allocator_traits : __allocator_traits_base
@@ -829,6 +833,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
    */
 
   template<typename _ForwardIterator, typename _Allocator>
+    _GLIBCXX20_CONSTEXPR
     void
     _Destroy(_ForwardIterator __first, _ForwardIterator __last,
 	     _Allocator& __alloc)
@@ -843,6 +848,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     }
 
   template<typename _ForwardIterator, typename _Tp>
+    _GLIBCXX20_CONSTEXPR
     inline void
     _Destroy(_ForwardIterator __first, _ForwardIterator __last,
 	     allocator<_Tp>&)
diff --git a/libstdc++-v3/include/bits/allocated_ptr.h b/libstdc++-v3/include/bits/allocated_ptr.h
index 340964e..695695f 100644
--- a/libstdc++-v3/include/bits/allocated_ptr.h
+++ b/libstdc++-v3/include/bits/allocated_ptr.h
@@ -40,6 +40,7 @@
 namespace std _GLIBCXX_VISIBILITY(default)
 {
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
+/// @cond undocumented
 
   /// Non-standard RAII type for managing pointers obtained from allocators.
   template<typename _Alloc>
@@ -97,6 +98,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       return { __a, std::allocator_traits<_Alloc>::allocate(__a, 1) };
     }
 
+/// @endcond
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace std
 
diff --git a/libstdc++-v3/include/bits/allocator.h b/libstdc++-v3/include/bits/allocator.h
index a6883c6..f83e6b8 100644
--- a/libstdc++-v3/include/bits/allocator.h
+++ b/libstdc++-v3/include/bits/allocator.h
@@ -309,6 +309,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   template<typename _Tp>
     struct __shrink_to_fit_aux<_Tp, true>
     {
+      _GLIBCXX20_CONSTEXPR
       static bool
       _S_do_it(_Tp& __c) noexcept
       {
diff --git a/libstdc++-v3/include/bits/c++config b/libstdc++-v3/include/bits/c++config
index a649580..4b7fa65 100644
--- a/libstdc++-v3/include/bits/c++config
+++ b/libstdc++-v3/include/bits/c++config
@@ -526,22 +526,17 @@ namespace std
   // Avoid the use of assert, because we're trying to keep the <cassert>
   // include out of the mix.
   extern "C++" _GLIBCXX_NORETURN
-  inline void
-  __replacement_assert(const char* __file, int __line,
-		       const char* __function, const char* __condition)
-  _GLIBCXX_NOEXCEPT
-  {
-    __builtin_printf("%s:%d: %s: Assertion '%s' failed.\n", __file, __line,
-		     __function, __condition);
-    __builtin_abort();
-  }
+  void
+  __glibcxx_assert_fail(const char* __file, int __line,
+			const char* __function, const char* __condition)
+  _GLIBCXX_NOEXCEPT;
 }
-#define __glibcxx_assert_impl(_Condition)			       \
-  if (__builtin_expect(!bool(_Condition), false))		       \
-  {								       \
-    __glibcxx_constexpr_assert(false);				       \
-    std::__replacement_assert(__FILE__, __LINE__, __PRETTY_FUNCTION__, \
-			      #_Condition);			       \
+#define __glibcxx_assert_impl(_Condition)				\
+  if (__builtin_expect(!bool(_Condition), false))			\
+  {									\
+    __glibcxx_constexpr_assert(false);					\
+    std::__glibcxx_assert_fail(__FILE__, __LINE__, __PRETTY_FUNCTION__,	\
+			       #_Condition);				\
   }
 # else // ! VERBOSE_ASSERT
 # define __glibcxx_assert_impl(_Condition)		\
@@ -550,7 +545,7 @@ namespace std
     __glibcxx_constexpr_assert(false);			\
     __builtin_abort();					\
   }
-#endif
+# endif
 #endif
 
 #if defined(_GLIBCXX_ASSERTIONS)
diff --git a/libstdc++-v3/include/bits/enable_special_members.h b/libstdc++-v3/include/bits/enable_special_members.h
index 8361a06..ac59f72 100644
--- a/libstdc++-v3/include/bits/enable_special_members.h
+++ b/libstdc++-v3/include/bits/enable_special_members.h
@@ -37,6 +37,7 @@
 namespace std _GLIBCXX_VISIBILITY(default)
 {
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
+/// @cond undocumented
 
   struct _Enable_default_constructor_tag
   {
@@ -308,6 +309,7 @@ template<typename _Tag>
     operator=(_Enable_copy_move&&) noexcept                         = delete;
   };
 
+/// @endcond
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace std
 
diff --git a/libstdc++-v3/include/bits/hashtable.h b/libstdc++-v3/include/bits/hashtable.h
index ff8af22..6e2d4c1 100644
--- a/libstdc++-v3/include/bits/hashtable.h
+++ b/libstdc++-v3/include/bits/hashtable.h
@@ -41,6 +41,7 @@
 namespace std _GLIBCXX_VISIBILITY(default)
 {
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
+/// @cond undocumented
 
   template<typename _Tp, typename _Hash>
     using __cache_default
@@ -1064,18 +1065,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       /// Merge from a compatible container into one with unique keys.
       template<typename _Compatible_Hashtable>
 	void
-	_M_merge_unique(_Compatible_Hashtable& __src) noexcept
+	_M_merge_unique(_Compatible_Hashtable& __src)
 	{
 	  static_assert(is_same_v<typename _Compatible_Hashtable::node_type,
 	      node_type>, "Node types are compatible");
 	  __glibcxx_assert(get_allocator() == __src.get_allocator());
 
 	  auto __n_elt = __src.size();
-	  for (auto __i = __src.begin(), __end = __src.end(); __i != __end;)
+	  for (auto __i = __src.cbegin(), __end = __src.cend(); __i != __end;)
 	    {
 	      auto __pos = __i++;
 	      const key_type& __k = _ExtractKey{}(*__pos);
-	      __hash_code __code = this->_M_hash_code(__k);
+	      __hash_code __code
+		= this->_M_hash_code(__src.hash_function(), *__pos._M_cur);
 	      size_type __bkt = _M_bucket_index(__code);
 	      if (_M_find_node(__bkt, __k, __code) == nullptr)
 		{
@@ -1092,15 +1094,23 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       /// Merge from a compatible container into one with equivalent keys.
       template<typename _Compatible_Hashtable>
 	void
-	_M_merge_multi(_Compatible_Hashtable& __src) noexcept
+	_M_merge_multi(_Compatible_Hashtable& __src)
 	{
 	  static_assert(is_same_v<typename _Compatible_Hashtable::node_type,
 	      node_type>, "Node types are compatible");
 	  __glibcxx_assert(get_allocator() == __src.get_allocator());
 
+	  __node_ptr __hint = nullptr;
 	  this->reserve(size() + __src.size());
-	  for (auto __i = __src.begin(), __end = __src.end(); __i != __end;)
-	    _M_reinsert_node_multi(cend(), __src.extract(__i++));
+	  for (auto __i = __src.cbegin(), __end = __src.cend(); __i != __end;)
+	    {
+	      auto __pos = __i++;
+	      __hash_code __code
+		= this->_M_hash_code(__src.hash_function(), *__pos._M_cur);
+	      auto __nh = __src.extract(__pos);
+	      __hint = _M_insert_multi_node(__hint, __code, __nh._M_ptr)._M_cur;
+	      __nh._M_ptr = nullptr;
+	    }
 	}
 #endif // C++17
 
@@ -2546,6 +2556,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       = __enable_if_t<!__or_<is_integral<_Hash>, __is_allocator<_Hash>>::value>;
 #endif
 
+/// @endcond
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace std
 
diff --git a/libstdc++-v3/include/bits/hashtable_policy.h b/libstdc++-v3/include/bits/hashtable_policy.h
index 994c7b6..0b5443f 100644
--- a/libstdc++-v3/include/bits/hashtable_policy.h
+++ b/libstdc++-v3/include/bits/hashtable_policy.h
@@ -38,6 +38,7 @@
 namespace std _GLIBCXX_VISIBILITY(default)
 {
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
+/// @cond undocumented
 
   template<typename _Key, typename _Value, typename _Alloc,
 	   typename _ExtractKey, typename _Equal,
@@ -59,19 +60,19 @@ namespace __detail
 
   // Helper function: return distance(first, last) for forward
   // iterators, or 0/1 for input iterators.
-  template<class _Iterator>
+  template<typename _Iterator>
     inline typename std::iterator_traits<_Iterator>::difference_type
     __distance_fw(_Iterator __first, _Iterator __last,
 		  std::input_iterator_tag)
     { return __first != __last ? 1 : 0; }
 
-  template<class _Iterator>
+  template<typename _Iterator>
     inline typename std::iterator_traits<_Iterator>::difference_type
     __distance_fw(_Iterator __first, _Iterator __last,
 		  std::forward_iterator_tag)
     { return std::distance(__first, __last); }
 
-  template<class _Iterator>
+  template<typename _Iterator>
     inline typename std::iterator_traits<_Iterator>::difference_type
     __distance_fw(_Iterator __first, _Iterator __last)
     { return __distance_fw(__first, __last,
@@ -1249,6 +1250,19 @@ namespace __detail
 	  return _M_hash()(__k);
 	}
 
+      __hash_code
+      _M_hash_code(const _Hash&,
+		   const _Hash_node_value<_Value, true>& __n) const
+      { return __n._M_hash_code; }
+
+      // Compute hash code using _Hash as __n _M_hash_code, if present, was
+      // computed using _H2.
+      template<typename _H2>
+	__hash_code
+	_M_hash_code(const _H2&,
+		const _Hash_node_value<_Value, __cache_hash_code>& __n) const
+	{ return _M_hash_code(_ExtractKey{}(__n._M_v())); }
+
       std::size_t
       _M_bucket_index(__hash_code __c, std::size_t __bkt_count) const
       { return _RangeHash{}(__c, __bkt_count); }
@@ -1970,6 +1984,7 @@ namespace __detail
 
  ///@} hashtable-detail
 } // namespace __detail
+/// @endcond
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace std
 
diff --git a/libstdc++-v3/include/bits/list.tcc b/libstdc++-v3/include/bits/list.tcc
index 7f4e156..cc750c9 100644
--- a/libstdc++-v3/include/bits/list.tcc
+++ b/libstdc++-v3/include/bits/list.tcc
@@ -499,7 +499,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	_Scratch_list* __fill = __tmp;
 	_Scratch_list* __counter;
 
-	_Scratch_list::_Ptr_cmp<const_iterator, void> __ptr_comp;
+	_Scratch_list::_Ptr_cmp<iterator, void> __ptr_comp;
 
 	__try
 	  {
@@ -623,7 +623,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	  _Scratch_list* __fill = __tmp;
 	  _Scratch_list* __counter;
 
-	_Scratch_list::_Ptr_cmp<const_iterator, _StrictWeakOrdering> __ptr_comp
+	_Scratch_list::_Ptr_cmp<iterator, _StrictWeakOrdering> __ptr_comp
 	  = { __comp };
 
 	  __try
diff --git a/libstdc++-v3/include/bits/mofunc_impl.h b/libstdc++-v3/include/bits/mofunc_impl.h
index 968d235..6cc9711 100644
--- a/libstdc++-v3/include/bits/mofunc_impl.h
+++ b/libstdc++-v3/include/bits/mofunc_impl.h
@@ -44,6 +44,22 @@ namespace std _GLIBCXX_VISIBILITY(default)
 {
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
+  /**
+   *  @brief Polymorphic function wrapper.
+   *  @ingroup functors
+   *  @since C++23
+   *  @headername functional
+   *
+   *  The `std::move_only_function` class template is a call wrapper similar
+   *  to *  `std::function`, but does not require the stored target function
+   *  to be copyable.
+   *
+   *  It also supports const-qualification, ref-qualification, and
+   *  no-throw guarantees. The qualifications and exception-specification
+   *  of the `move_only_function::operator()` member function are respected
+   *  when invoking the target function.
+   *
+   */
   template<typename _Res, typename... _ArgTypes, bool _Noex>
     class move_only_function<_Res(_ArgTypes...) _GLIBCXX_MOF_CV
 			       _GLIBCXX_MOF_REF noexcept(_Noex)>
@@ -64,15 +80,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     public:
       using result_type = _Res;
 
+      /// Creates an empty object.
       move_only_function() noexcept { }
 
+      /// Creates an empty object.
       move_only_function(nullptr_t) noexcept { }
 
+      /// Moves the target object, leaving the source empty.
       move_only_function(move_only_function&& __x) noexcept
       : _Mofunc_base(static_cast<_Mofunc_base&&>(__x)),
 	_M_invoke(std::__exchange(__x._M_invoke, nullptr))
       { }
 
+      /// Stores a target object initialized from the argument.
       template<typename _Fn, typename _Vt = decay_t<_Fn>>
 	requires (!is_same_v<_Vt, move_only_function>)
 	  && (!__is_in_place_type_v<_Vt>) && __is_callable_from<_Vt>
@@ -89,6 +109,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  _M_invoke = &_S_invoke<_Vt>;
 	}
 
+      /// Stores a target object initialized from the arguments.
       template<typename _Tp, typename... _Args>
 	requires is_constructible_v<_Tp, _Args...>
 	  && __is_callable_from<_Tp>
@@ -101,6 +122,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  _M_init<_Tp>(std::forward<_Args>(__args)...);
 	}
 
+      /// Stores a target object initialized from the arguments.
       template<typename _Tp, typename _Up, typename... _Args>
 	requires is_constructible_v<_Tp, initializer_list<_Up>&, _Args...>
 	  && __is_callable_from<_Tp>
@@ -114,6 +136,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  _M_init<_Tp>(__il, std::forward<_Args>(__args)...);
 	}
 
+      /// Stores a new target object, leaving `x` empty.
       move_only_function&
       operator=(move_only_function&& __x) noexcept
       {
@@ -122,6 +145,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return *this;
       }
 
+      /// Destroys the target object (if any).
       move_only_function&
       operator=(nullptr_t) noexcept
       {
@@ -130,6 +154,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return *this;
       }
 
+      /// Stores a new target object, initialized from the argument.
       template<typename _Fn>
 	requires is_constructible_v<move_only_function, _Fn>
 	move_only_function&
@@ -142,8 +167,17 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
       ~move_only_function() = default;
 
+      /// True if a target object is present, false otherwise.
       explicit operator bool() const noexcept { return _M_invoke != nullptr; }
 
+      /** Invoke the target object.
+       *
+       * The target object will be invoked using the supplied arguments,
+       * and as an lvalue or rvalue, and as const or non-const, as dictated
+       * by the template arguments of the `move_only_function` specialization.
+       *
+       * @pre Must not be empty.
+       */
       _Res
       operator()(_ArgTypes... __args) _GLIBCXX_MOF_CV_REF noexcept(_Noex)
       {
@@ -151,6 +185,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	return _M_invoke(this, std::forward<_ArgTypes>(__args)...);
       }
 
+      /// Exchange the target objects (if any).
       void
       swap(move_only_function& __x) noexcept
       {
@@ -158,10 +193,12 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	std::swap(_M_invoke, __x._M_invoke);
       }
 
+      /// Exchange the target objects (if any).
       friend void
       swap(move_only_function& __x, move_only_function& __y) noexcept
       { __x.swap(__y); }
 
+      /// Check for emptiness by comparing with `nullptr`.
       friend bool
       operator==(const move_only_function& __x, nullptr_t) noexcept
       { return __x._M_invoke == nullptr; }
diff --git a/libstdc++-v3/include/bits/node_handle.h b/libstdc++-v3/include/bits/node_handle.h
index 9a43caf..7d8d0dc 100644
--- a/libstdc++-v3/include/bits/node_handle.h
+++ b/libstdc++-v3/include/bits/node_handle.h
@@ -44,6 +44,21 @@ namespace std _GLIBCXX_VISIBILITY(default)
 {
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
+  /**
+   * @defgroup node_handles Node handles
+   * @ingroup associative_containers
+   * @since C++17
+   *
+   * The associative containers (`map`, `set`, `multimap` and `multiset`)
+   * support extracting and re-inserting nodes from the container. Those
+   * operations use the container's `node_handle` type, which is an alias
+   * for a `_Node_handle<...>` type. You should always use the container's
+   * `node_handle` type (e.g. `std::set<int>::node_handle`) to refer to
+   * these types, not the non-standard internal `_Node_handle` names.
+   *
+   * @{
+   */
+
   /// Base class for node handle types of maps and sets.
   template<typename _Val, typename _NodeAlloc>
     class _Node_handle_common
@@ -64,6 +79,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
       [[nodiscard]] bool empty() const noexcept { return _M_ptr == nullptr; }
 
+    /// @cond undocumented
     protected:
       constexpr _Node_handle_common() noexcept : _M_ptr() { }
 
@@ -214,6 +230,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       template<typename _Key2, typename _Value2, typename _KeyOfValue,
 	       typename _Compare, typename _ValueAlloc>
 	friend class _Rb_tree;
+
+      /// @endcond
     };
 
   /// Node handle type for maps.
@@ -367,6 +385,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       _NodeHandle	node;
     };
 
+  /// @}
+
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace std
 
diff --git a/libstdc++-v3/include/bits/range_access.h b/libstdc++-v3/include/bits/range_access.h
index 3dec687..5e4c472 100644
--- a/libstdc++-v3/include/bits/range_access.h
+++ b/libstdc++-v3/include/bits/range_access.h
@@ -110,10 +110,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   template<typename _Tp> class valarray;
   // These overloads must be declared for cbegin and cend to use them.
-  template<typename _Tp> _Tp* begin(valarray<_Tp>&);
-  template<typename _Tp> const _Tp* begin(const valarray<_Tp>&);
-  template<typename _Tp> _Tp* end(valarray<_Tp>&);
-  template<typename _Tp> const _Tp* end(const valarray<_Tp>&);
+  template<typename _Tp> _Tp* begin(valarray<_Tp>&) noexcept;
+  template<typename _Tp> const _Tp* begin(const valarray<_Tp>&) noexcept;
+  template<typename _Tp> _Tp* end(valarray<_Tp>&) noexcept;
+  template<typename _Tp> const _Tp* end(const valarray<_Tp>&) noexcept;
 
   /**
    *  @brief  Return an iterator pointing to the first element of
diff --git a/libstdc++-v3/include/bits/ranges_util.h b/libstdc++-v3/include/bits/ranges_util.h
index a52b8fc..060e665 100644
--- a/libstdc++-v3/include/bits/ranges_util.h
+++ b/libstdc++-v3/include/bits/ranges_util.h
@@ -216,6 +216,8 @@ namespace ranges
 
   } // namespace __detail
 
+  namespace views { struct _Drop; } // defined in <ranges>
+
   enum class subrange_kind : bool { unsized, sized };
 
   /// The ranges::subrange class template
@@ -226,10 +228,11 @@ namespace ranges
     class subrange : public view_interface<subrange<_It, _Sent, _Kind>>
     {
     private:
-      // XXX: gcc complains when using constexpr here
-      static const bool _S_store_size
+      static constexpr bool _S_store_size
 	= _Kind == subrange_kind::sized && !sized_sentinel_for<_Sent, _It>;
 
+      friend struct views::_Drop; // Needs to inspect _S_store_size.
+
       _It _M_begin = _It();
       [[no_unique_address]] _Sent _M_end = _Sent();
 
diff --git a/libstdc++-v3/include/bits/stl_algobase.h b/libstdc++-v3/include/bits/stl_algobase.h
index 0e05868..f441165 100644
--- a/libstdc++-v3/include/bits/stl_algobase.h
+++ b/libstdc++-v3/include/bits/stl_algobase.h
@@ -956,6 +956,7 @@ _GLIBCXX_END_NAMESPACE_CONTAINER
 	      const _GLIBCXX_STD_C::_Deque_iterator<_Tp, _Tp&, _Tp*>&,
 	      const _VTp&);
 
+  _GLIBCXX20_CONSTEXPR
   void
   __fill_a1(_GLIBCXX_STD_C::_Bit_iterator, _GLIBCXX_STD_C::_Bit_iterator,
 	    const bool&);
diff --git a/libstdc++-v3/include/bits/stl_bvector.h b/libstdc++-v3/include/bits/stl_bvector.h
index 3778d5a..381c47b 100644
--- a/libstdc++-v3/include/bits/stl_bvector.h
+++ b/libstdc++-v3/include/bits/stl_bvector.h
@@ -64,29 +64,38 @@
 namespace std _GLIBCXX_VISIBILITY(default)
 {
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
-_GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   typedef unsigned long _Bit_type;
   enum { _S_word_bit = int(__CHAR_BIT__ * sizeof(_Bit_type)) };
 
+  __attribute__((__nonnull__))
+  _GLIBCXX20_CONSTEXPR
+  void
+  __fill_bvector_n(_Bit_type*, size_t, bool) _GLIBCXX_NOEXCEPT;
+
+_GLIBCXX_BEGIN_NAMESPACE_CONTAINER
+
   struct _Bit_reference
   {
     _Bit_type * _M_p;
     _Bit_type _M_mask;
 
+    _GLIBCXX20_CONSTEXPR
     _Bit_reference(_Bit_type * __x, _Bit_type __y)
     : _M_p(__x), _M_mask(__y) { }
 
+    _GLIBCXX20_CONSTEXPR
     _Bit_reference() _GLIBCXX_NOEXCEPT : _M_p(0), _M_mask(0) { }
 
 #if __cplusplus >= 201103L
     _Bit_reference(const _Bit_reference&) = default;
 #endif
 
-    _GLIBCXX_NODISCARD
+    _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
     operator bool() const _GLIBCXX_NOEXCEPT
     { return !!(*_M_p & _M_mask); }
 
+    _GLIBCXX20_CONSTEXPR
     _Bit_reference&
     operator=(bool __x) _GLIBCXX_NOEXCEPT
     {
@@ -97,26 +106,29 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return *this;
     }
 
+    _GLIBCXX20_CONSTEXPR
     _Bit_reference&
     operator=(const _Bit_reference& __x) _GLIBCXX_NOEXCEPT
     { return *this = bool(__x); }
 
-    _GLIBCXX_NODISCARD
+    _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
     bool
     operator==(const _Bit_reference& __x) const
     { return bool(*this) == bool(__x); }
 
-    _GLIBCXX_NODISCARD
+    _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
     bool
     operator<(const _Bit_reference& __x) const
     { return !bool(*this) && bool(__x); }
 
+    _GLIBCXX20_CONSTEXPR
     void
     flip() _GLIBCXX_NOEXCEPT
     { *_M_p ^= _M_mask; }
   };
 
 #if __cplusplus >= 201103L
+  _GLIBCXX20_CONSTEXPR
   inline void
   swap(_Bit_reference __x, _Bit_reference __y) noexcept
   {
@@ -125,6 +137,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     __y = __tmp;
   }
 
+  _GLIBCXX20_CONSTEXPR
   inline void
   swap(_Bit_reference __x, bool& __y) noexcept
   {
@@ -133,6 +146,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     __y = __tmp;
   }
 
+  _GLIBCXX20_CONSTEXPR
   inline void
   swap(bool& __x, _Bit_reference __y) noexcept
   {
@@ -148,9 +162,11 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     _Bit_type * _M_p;
     unsigned int _M_offset;
 
+    _GLIBCXX20_CONSTEXPR
     _Bit_iterator_base(_Bit_type * __x, unsigned int __y)
     : _M_p(__x), _M_offset(__y) { }
 
+    _GLIBCXX20_CONSTEXPR
     void
     _M_bump_up()
     {
@@ -161,6 +177,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	}
     }
 
+    _GLIBCXX20_CONSTEXPR
     void
     _M_bump_down()
     {
@@ -171,6 +188,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	}
     }
 
+    _GLIBCXX20_CONSTEXPR
     void
     _M_incr(ptrdiff_t __i)
     {
@@ -230,7 +248,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     { return !(__x < __y); }
 #endif // three-way comparison
 
-    friend ptrdiff_t
+    friend _GLIBCXX20_CONSTEXPR ptrdiff_t
     operator-(const _Bit_iterator_base& __x, const _Bit_iterator_base& __y)
     {
       return (int(_S_word_bit) * (__x._M_p - __y._M_p)
@@ -248,20 +266,24 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #endif
     typedef _Bit_iterator   iterator;
 
+    _GLIBCXX20_CONSTEXPR
     _Bit_iterator() : _Bit_iterator_base(0, 0) { }
 
+    _GLIBCXX20_CONSTEXPR
     _Bit_iterator(_Bit_type * __x, unsigned int __y)
     : _Bit_iterator_base(__x, __y) { }
 
+    _GLIBCXX20_CONSTEXPR
     iterator
     _M_const_cast() const
     { return *this; }
 
-    _GLIBCXX_NODISCARD
+    _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
     reference
     operator*() const
     { return reference(_M_p, 1UL << _M_offset); }
 
+    _GLIBCXX20_CONSTEXPR
     iterator&
     operator++()
     {
@@ -269,6 +291,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return *this;
     }
 
+    _GLIBCXX20_CONSTEXPR
     iterator
     operator++(int)
     {
@@ -277,6 +300,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return __tmp;
     }
 
+    _GLIBCXX20_CONSTEXPR
     iterator&
     operator--()
     {
@@ -284,6 +308,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return *this;
     }
 
+    _GLIBCXX20_CONSTEXPR
     iterator
     operator--(int)
     {
@@ -292,6 +317,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return __tmp;
     }
 
+    _GLIBCXX20_CONSTEXPR
     iterator&
     operator+=(difference_type __i)
     {
@@ -299,6 +325,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return *this;
     }
 
+    _GLIBCXX20_CONSTEXPR
     iterator&
     operator-=(difference_type __i)
     {
@@ -306,13 +333,13 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return *this;
     }
 
-    _GLIBCXX_NODISCARD
+    _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
     reference
     operator[](difference_type __i) const
     { return *(*this + __i); }
 
     _GLIBCXX_NODISCARD
-    friend iterator
+    friend _GLIBCXX20_CONSTEXPR iterator
     operator+(const iterator& __x, difference_type __n)
     {
       iterator __tmp = __x;
@@ -321,12 +348,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
     _GLIBCXX_NODISCARD
-    friend iterator
+    friend _GLIBCXX20_CONSTEXPR iterator
     operator+(difference_type __n, const iterator& __x)
     { return __x + __n; }
 
     _GLIBCXX_NODISCARD
-    friend iterator
+    friend _GLIBCXX20_CONSTEXPR iterator
     operator-(const iterator& __x, difference_type __n)
     {
       iterator __tmp = __x;
@@ -346,23 +373,28 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #endif
     typedef _Bit_const_iterator  const_iterator;
 
+    _GLIBCXX20_CONSTEXPR
     _Bit_const_iterator() : _Bit_iterator_base(0, 0) { }
 
+    _GLIBCXX20_CONSTEXPR
     _Bit_const_iterator(_Bit_type * __x, unsigned int __y)
     : _Bit_iterator_base(__x, __y) { }
 
+    _GLIBCXX20_CONSTEXPR
     _Bit_const_iterator(const _Bit_iterator& __x)
     : _Bit_iterator_base(__x._M_p, __x._M_offset) { }
 
+    _GLIBCXX20_CONSTEXPR
     _Bit_iterator
     _M_const_cast() const
     { return _Bit_iterator(_M_p, _M_offset); }
 
-    _GLIBCXX_NODISCARD
+    _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
     const_reference
     operator*() const
     { return _Bit_reference(_M_p, 1UL << _M_offset); }
 
+    _GLIBCXX20_CONSTEXPR
     const_iterator&
     operator++()
     {
@@ -370,6 +402,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return *this;
     }
 
+    _GLIBCXX20_CONSTEXPR
     const_iterator
     operator++(int)
     {
@@ -378,6 +411,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return __tmp;
     }
 
+    _GLIBCXX20_CONSTEXPR
     const_iterator&
     operator--()
     {
@@ -385,6 +419,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return *this;
     }
 
+    _GLIBCXX20_CONSTEXPR
     const_iterator
     operator--(int)
     {
@@ -393,6 +428,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return __tmp;
     }
 
+    _GLIBCXX20_CONSTEXPR
     const_iterator&
     operator+=(difference_type __i)
     {
@@ -400,6 +436,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return *this;
     }
 
+    _GLIBCXX20_CONSTEXPR
     const_iterator&
     operator-=(difference_type __i)
     {
@@ -407,13 +444,13 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       return *this;
     }
 
-    _GLIBCXX_NODISCARD
+    _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
     const_reference
     operator[](difference_type __i) const
     { return *(*this + __i); }
 
     _GLIBCXX_NODISCARD
-    friend const_iterator
+    friend _GLIBCXX20_CONSTEXPR const_iterator
     operator+(const const_iterator& __x, difference_type __n)
     {
       const_iterator __tmp = __x;
@@ -422,7 +459,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
     _GLIBCXX_NODISCARD
-    friend const_iterator
+    friend _GLIBCXX20_CONSTEXPR const_iterator
     operator-(const const_iterator& __x, difference_type __n)
     {
       const_iterator __tmp = __x;
@@ -431,7 +468,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
     _GLIBCXX_NODISCARD
-    friend const_iterator
+    friend _GLIBCXX20_CONSTEXPR const_iterator
     operator+(difference_type __n, const const_iterator& __x)
     { return __x + __n; }
   };
@@ -454,25 +491,30 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	struct {
 	  _Bit_type* _M_p;
 	  // Allow assignment from iterators (assume offset is zero):
+	  _GLIBCXX20_CONSTEXPR
 	  void operator=(_Bit_iterator __it) { _M_p = __it._M_p; }
 	} _M_start;
 #endif
 	_Bit_iterator	_M_finish;
 	_Bit_pointer	_M_end_of_storage;
 
+	_GLIBCXX20_CONSTEXPR
 	_Bvector_impl_data() _GLIBCXX_NOEXCEPT
 	: _M_start(), _M_finish(), _M_end_of_storage()
 	{ }
 
 #if __cplusplus >= 201103L
 	_Bvector_impl_data(const _Bvector_impl_data&) = default;
+
 	_Bvector_impl_data&
 	operator=(const _Bvector_impl_data&) = default;
 
+	_GLIBCXX20_CONSTEXPR
 	_Bvector_impl_data(_Bvector_impl_data&& __x) noexcept
 	: _Bvector_impl_data(__x)
 	{ __x._M_reset(); }
 
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_move_data(_Bvector_impl_data&& __x) noexcept
 	{
@@ -481,10 +523,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	}
 #endif
 
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_reset() _GLIBCXX_NOEXCEPT
 	{ *this = _Bvector_impl_data(); }
 
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_swap_data(_Bvector_impl_data& __x) _GLIBCXX_NOEXCEPT
 	{
@@ -497,11 +541,13 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       struct _Bvector_impl
 	: public _Bit_alloc_type, public _Bvector_impl_data
       {
+	_GLIBCXX20_CONSTEXPR
 	_Bvector_impl() _GLIBCXX_NOEXCEPT_IF(
 	  is_nothrow_default_constructible<_Bit_alloc_type>::value)
 	: _Bit_alloc_type()
 	{ }
 
+	_GLIBCXX20_CONSTEXPR
 	_Bvector_impl(const _Bit_alloc_type& __a) _GLIBCXX_NOEXCEPT
 	: _Bit_alloc_type(__a)
 	{ }
@@ -509,15 +555,18 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #if __cplusplus >= 201103L
 	// Not defaulted, to enforce noexcept(true) even when
 	// !is_nothrow_move_constructible<_Bit_alloc_type>.
+	_GLIBCXX20_CONSTEXPR
 	_Bvector_impl(_Bvector_impl&& __x) noexcept
 	: _Bit_alloc_type(std::move(__x)), _Bvector_impl_data(std::move(__x))
 	{ }
 
+	_GLIBCXX20_CONSTEXPR
 	_Bvector_impl(_Bit_alloc_type&& __a, _Bvector_impl&& __x) noexcept
 	: _Bit_alloc_type(std::move(__a)), _Bvector_impl_data(std::move(__x))
 	{ }
 #endif
 
+	_GLIBCXX20_CONSTEXPR
 	_Bit_type*
 	_M_end_addr() const _GLIBCXX_NOEXCEPT
 	{
@@ -530,14 +579,17 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     public:
       typedef _Alloc allocator_type;
 
+      _GLIBCXX20_CONSTEXPR
       _Bit_alloc_type&
       _M_get_Bit_allocator() _GLIBCXX_NOEXCEPT
       { return this->_M_impl; }
 
+      _GLIBCXX20_CONSTEXPR
       const _Bit_alloc_type&
       _M_get_Bit_allocator() const _GLIBCXX_NOEXCEPT
       { return this->_M_impl; }
 
+      _GLIBCXX20_CONSTEXPR
       allocator_type
       get_allocator() const _GLIBCXX_NOEXCEPT
       { return allocator_type(_M_get_Bit_allocator()); }
@@ -548,27 +600,43 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       _Bvector_base() { }
 #endif
 
+      _GLIBCXX20_CONSTEXPR
       _Bvector_base(const allocator_type& __a)
       : _M_impl(__a) { }
 
 #if __cplusplus >= 201103L
       _Bvector_base(_Bvector_base&&) = default;
 
+      _GLIBCXX20_CONSTEXPR
       _Bvector_base(_Bvector_base&& __x, const allocator_type& __a) noexcept
       : _M_impl(_Bit_alloc_type(__a), std::move(__x._M_impl))
       { }
 #endif
 
+      _GLIBCXX20_CONSTEXPR
       ~_Bvector_base()
       { this->_M_deallocate(); }
 
     protected:
       _Bvector_impl _M_impl;
 
+      _GLIBCXX20_CONSTEXPR
       _Bit_pointer
       _M_allocate(size_t __n)
-      { return _Bit_alloc_traits::allocate(_M_impl, _S_nword(__n)); }
+      {
+	_Bit_pointer __p = _Bit_alloc_traits::allocate(_M_impl, _S_nword(__n));
+#if __has_builtin(__builtin_is_constant_evaluated)
+	if (__builtin_is_constant_evaluated())
+	{
+	  __n = _S_nword(__n);
+	  for (size_t __i = 0; __i < __n; ++__i)
+	    __p[__i] = 0ul;
+	}
+#endif
+	return __p;
+      }
 
+      _GLIBCXX20_CONSTEXPR
       void
       _M_deallocate()
       {
@@ -583,11 +651,13 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
 #if __cplusplus >= 201103L
+      _GLIBCXX20_CONSTEXPR
       void
       _M_move_data(_Bvector_base&& __x) noexcept
       { _M_impl._M_move_data(std::move(__x._M_impl)); }
 #endif
 
+      _GLIBCXX_CONSTEXPR
       static size_t
       _S_nword(size_t __n)
       { return (__n + int(_S_word_bit) - 1) / int(_S_word_bit); }
@@ -637,6 +707,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       typedef std::reverse_iterator<iterator>		reverse_iterator;
       typedef _Alloc					allocator_type;
 
+      _GLIBCXX20_CONSTEXPR
       allocator_type
       get_allocator() const
       { return _Base::get_allocator(); }
@@ -654,16 +725,19 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       vector() { }
 #endif
 
+      _GLIBCXX20_CONSTEXPR
       explicit
       vector(const allocator_type& __a)
       : _Base(__a) { }
 
 #if __cplusplus >= 201103L
+      _GLIBCXX20_CONSTEXPR
       explicit
       vector(size_type __n, const allocator_type& __a = allocator_type())
       : vector(__n, false, __a)
       { }
 
+      _GLIBCXX20_CONSTEXPR
       vector(size_type __n, const bool& __value,
 	     const allocator_type& __a = allocator_type())
 #else
@@ -677,6 +751,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	_M_initialize_value(__value);
       }
 
+      _GLIBCXX20_CONSTEXPR
       vector(const vector& __x)
       : _Base(_Bit_alloc_traits::_S_select_on_copy(__x._M_get_Bit_allocator()))
       {
@@ -688,10 +763,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       vector(vector&&) = default;
 
     private:
+      _GLIBCXX20_CONSTEXPR
       vector(vector&& __x, const allocator_type& __a, true_type) noexcept
       : _Base(std::move(__x), __a)
       { }
 
+      _GLIBCXX20_CONSTEXPR
       vector(vector&& __x, const allocator_type& __a, false_type)
       : _Base(__a)
       {
@@ -706,12 +783,14 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
     public:
+      _GLIBCXX20_CONSTEXPR
       vector(vector&& __x, const __type_identity_t<allocator_type>& __a)
       noexcept(_Bit_alloc_traits::_S_always_equal())
       : vector(std::move(__x), __a,
 	       typename _Bit_alloc_traits::is_always_equal{})
       { }
 
+      _GLIBCXX20_CONSTEXPR
       vector(const vector& __x, const __type_identity_t<allocator_type>& __a)
       : _Base(__a)
       {
@@ -719,6 +798,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	_M_copy_aligned(__x.begin(), __x.end(), begin());
       }
 
+      _GLIBCXX20_CONSTEXPR
       vector(initializer_list<bool> __l,
 	     const allocator_type& __a = allocator_type())
       : _Base(__a)
@@ -731,6 +811,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #if __cplusplus >= 201103L
       template<typename _InputIterator,
 	       typename = std::_RequireInputIter<_InputIterator>>
+	_GLIBCXX20_CONSTEXPR
 	vector(_InputIterator __first, _InputIterator __last,
 	       const allocator_type& __a = allocator_type())
 	: _Base(__a)
@@ -750,8 +831,10 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	}
 #endif
 
+      _GLIBCXX20_CONSTEXPR
       ~vector() _GLIBCXX_NOEXCEPT { }
 
+      _GLIBCXX20_CONSTEXPR
       vector&
       operator=(const vector& __x)
       {
@@ -783,6 +866,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
 #if __cplusplus >= 201103L
+      _GLIBCXX20_CONSTEXPR
       vector&
       operator=(vector&& __x) noexcept(_Bit_alloc_traits::_S_nothrow_move())
       {
@@ -808,6 +892,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	return *this;
       }
 
+      _GLIBCXX20_CONSTEXPR
       vector&
       operator=(initializer_list<bool> __l)
       {
@@ -820,6 +905,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       // versions: one that takes a count, and one that takes a range.
       // The range version is a member template, so we dispatch on whether
       // or not the type is an integer.
+      _GLIBCXX20_CONSTEXPR
       void
       assign(size_type __n, const bool& __x)
       { _M_fill_assign(__n, __x); }
@@ -827,6 +913,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #if __cplusplus >= 201103L
       template<typename _InputIterator,
 	       typename = std::_RequireInputIter<_InputIterator>>
+	_GLIBCXX20_CONSTEXPR
 	void
 	assign(_InputIterator __first, _InputIterator __last)
 	{ _M_assign_aux(__first, __last, std::__iterator_category(__first)); }
@@ -842,79 +929,80 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #endif
 
 #if __cplusplus >= 201103L
+      _GLIBCXX20_CONSTEXPR
       void
       assign(initializer_list<bool> __l)
       { _M_assign_aux(__l.begin(), __l.end(), random_access_iterator_tag()); }
 #endif
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       iterator
       begin() _GLIBCXX_NOEXCEPT
       { return iterator(this->_M_impl._M_start._M_p, 0); }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_iterator
       begin() const _GLIBCXX_NOEXCEPT
       { return const_iterator(this->_M_impl._M_start._M_p, 0); }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       iterator
       end() _GLIBCXX_NOEXCEPT
       { return this->_M_impl._M_finish; }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_iterator
       end() const _GLIBCXX_NOEXCEPT
       { return this->_M_impl._M_finish; }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       reverse_iterator
       rbegin() _GLIBCXX_NOEXCEPT
       { return reverse_iterator(end()); }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_reverse_iterator
       rbegin() const _GLIBCXX_NOEXCEPT
       { return const_reverse_iterator(end()); }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       reverse_iterator
       rend() _GLIBCXX_NOEXCEPT
       { return reverse_iterator(begin()); }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_reverse_iterator
       rend() const _GLIBCXX_NOEXCEPT
       { return const_reverse_iterator(begin()); }
 
 #if __cplusplus >= 201103L
-      [[__nodiscard__]]
+      [[__nodiscard__]] _GLIBCXX20_CONSTEXPR
       const_iterator
       cbegin() const noexcept
       { return const_iterator(this->_M_impl._M_start._M_p, 0); }
 
-      [[__nodiscard__]]
+      [[__nodiscard__]] _GLIBCXX20_CONSTEXPR
       const_iterator
       cend() const noexcept
       { return this->_M_impl._M_finish; }
 
-      [[__nodiscard__]]
+      [[__nodiscard__]] _GLIBCXX20_CONSTEXPR
       const_reverse_iterator
       crbegin() const noexcept
       { return const_reverse_iterator(end()); }
 
-      [[__nodiscard__]]
+      [[__nodiscard__]] _GLIBCXX20_CONSTEXPR
       const_reverse_iterator
       crend() const noexcept
       { return const_reverse_iterator(begin()); }
 #endif
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       size_type
       size() const _GLIBCXX_NOEXCEPT
       { return size_type(end() - begin()); }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       size_type
       max_size() const _GLIBCXX_NOEXCEPT
       {
@@ -927,27 +1015,29 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 		? __asize * int(_S_word_bit) : __isize);
       }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       size_type
       capacity() const _GLIBCXX_NOEXCEPT
       { return size_type(const_iterator(this->_M_impl._M_end_addr(), 0)
 			 - begin()); }
 
-      _GLIBCXX_NODISCARD bool
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
+      bool
       empty() const _GLIBCXX_NOEXCEPT
       { return begin() == end(); }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       reference
       operator[](size_type __n)
       { return begin()[__n]; }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_reference
       operator[](size_type __n) const
       { return begin()[__n]; }
 
     protected:
+      _GLIBCXX20_CONSTEXPR
       void
       _M_range_check(size_type __n) const
       {
@@ -959,14 +1049,23 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
     public:
+      _GLIBCXX20_CONSTEXPR
       reference
       at(size_type __n)
-      { _M_range_check(__n); return (*this)[__n]; }
+      {
+	_M_range_check(__n);
+	return (*this)[__n];
+      }
 
+      _GLIBCXX20_CONSTEXPR
       const_reference
       at(size_type __n) const
-      { _M_range_check(__n); return (*this)[__n]; }
+      {
+	_M_range_check(__n);
+	return (*this)[__n];
+      }
 
+      _GLIBCXX20_CONSTEXPR
       void
       reserve(size_type __n)
       {
@@ -976,34 +1075,27 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	  _M_reallocate(__n);
       }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       reference
       front()
       { return *begin(); }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_reference
       front() const
       { return *begin(); }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       reference
       back()
       { return *(end() - 1); }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_reference
       back() const
       { return *(end() - 1); }
 
-      // _GLIBCXX_RESOLVE_LIB_DEFECTS
-      // DR 464. Suggestion for new member functions in standard containers.
-      // N.B. DR 464 says nothing about vector<bool> but we need something
-      // here due to the way we are implementing DR 464 in the debug-mode
-      // vector class.
-      void
-      data() _GLIBCXX_NOEXCEPT { }
-
+      _GLIBCXX20_CONSTEXPR
       void
       push_back(bool __x)
       {
@@ -1013,6 +1105,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	  _M_insert_aux(end(), __x);
       }
 
+      _GLIBCXX20_CONSTEXPR
       void
       swap(vector& __x) _GLIBCXX_NOEXCEPT
       {
@@ -1026,6 +1119,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
       // [23.2.5]/1, third-to-last entry in synopsis listing
+      _GLIBCXX20_CONSTEXPR
       static void
       swap(reference __x, reference __y) _GLIBCXX_NOEXCEPT
       {
@@ -1034,6 +1128,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	__y = __tmp;
       }
 
+      _GLIBCXX20_CONSTEXPR
       iterator
 #if __cplusplus >= 201103L
       insert(const_iterator __position, const bool& __x = bool())
@@ -1053,6 +1148,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #if __cplusplus >= 201103L
       template<typename _InputIterator,
 	       typename = std::_RequireInputIter<_InputIterator>>
+	_GLIBCXX20_CONSTEXPR
 	iterator
 	insert(const_iterator __position,
 	       _InputIterator __first, _InputIterator __last)
@@ -1076,6 +1172,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #endif
 
 #if __cplusplus >= 201103L
+      _GLIBCXX20_CONSTEXPR
       iterator
       insert(const_iterator __position, size_type __n, const bool& __x)
       {
@@ -1090,15 +1187,18 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #endif
 
 #if __cplusplus >= 201103L
+      _GLIBCXX20_CONSTEXPR
       iterator
       insert(const_iterator __p, initializer_list<bool> __l)
       { return this->insert(__p, __l.begin(), __l.end()); }
 #endif
 
+      _GLIBCXX20_CONSTEXPR
       void
       pop_back()
       { --this->_M_impl._M_finish; }
 
+      _GLIBCXX20_CONSTEXPR
       iterator
 #if __cplusplus >= 201103L
       erase(const_iterator __position)
@@ -1107,6 +1207,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #endif
       { return _M_erase(__position._M_const_cast()); }
 
+      _GLIBCXX20_CONSTEXPR
       iterator
 #if __cplusplus >= 201103L
       erase(const_iterator __first, const_iterator __last)
@@ -1115,6 +1216,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #endif
       { return _M_erase(__first._M_const_cast(), __last._M_const_cast()); }
 
+      _GLIBCXX20_CONSTEXPR
       void
       resize(size_type __new_size, bool __x = bool())
       {
@@ -1125,11 +1227,13 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
 #if __cplusplus >= 201103L
+      _GLIBCXX20_CONSTEXPR
       void
       shrink_to_fit()
       { _M_shrink_to_fit(); }
 #endif
 
+      _GLIBCXX20_CONSTEXPR
       void
       flip() _GLIBCXX_NOEXCEPT
       {
@@ -1138,6 +1242,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	  *__p = ~*__p;
       }
 
+      _GLIBCXX20_CONSTEXPR
       void
       clear() _GLIBCXX_NOEXCEPT
       { _M_erase_at_end(begin()); }
@@ -1145,6 +1250,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #if __cplusplus >= 201103L
       template<typename... _Args>
 #if __cplusplus > 201402L
+	_GLIBCXX20_CONSTEXPR
 	reference
 #else
 	void
@@ -1158,6 +1264,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	}
 
       template<typename... _Args>
+	_GLIBCXX20_CONSTEXPR
 	iterator
 	emplace(const_iterator __pos, _Args&&... __args)
 	{ return insert(__pos, bool(__args...)); }
@@ -1165,6 +1272,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
     protected:
       // Precondition: __first._M_offset == 0 && __result._M_offset == 0.
+      _GLIBCXX20_CONSTEXPR
       iterator
       _M_copy_aligned(const_iterator __first, const_iterator __last,
 		      iterator __result)
@@ -1174,6 +1282,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 			 iterator(__q, 0));
       }
 
+      _GLIBCXX20_CONSTEXPR
       void
       _M_initialize(size_type __n)
       {
@@ -1187,19 +1296,20 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	  }
       }
 
+      _GLIBCXX20_CONSTEXPR
       void
-      _M_initialize_value(bool __x)
+      _M_initialize_value(bool __x) _GLIBCXX_NOEXCEPT
       {
 	if (_Bit_type* __p = this->_M_impl._M_start._M_p)
-	  __builtin_memset(__p, __x ? ~0 : 0,
-			   (this->_M_impl._M_end_addr() - __p)
-			   * sizeof(_Bit_type));
+	  __fill_bvector_n(__p, this->_M_impl._M_end_addr() - __p, __x);
       }
 
+      _GLIBCXX20_CONSTEXPR
       void
       _M_reallocate(size_type __n);
 
 #if __cplusplus >= 201103L
+      _GLIBCXX20_CONSTEXPR
       bool
       _M_shrink_to_fit();
 #endif
@@ -1224,6 +1334,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #endif
 
       template<typename _InputIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_initialize_range(_InputIterator __first, _InputIterator __last,
 			    std::input_iterator_tag)
@@ -1233,6 +1344,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	}
 
       template<typename _ForwardIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_initialize_range(_ForwardIterator __first, _ForwardIterator __last,
 			    std::forward_iterator_tag)
@@ -1257,6 +1369,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	{ _M_assign_aux(__first, __last, std::__iterator_category(__first)); }
 #endif
 
+      _GLIBCXX20_CONSTEXPR
       void
       _M_fill_assign(size_t __n, bool __x)
       {
@@ -1273,6 +1386,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
       template<typename _InputIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_assign_aux(_InputIterator __first, _InputIterator __last,
 		      std::input_iterator_tag)
@@ -1287,6 +1401,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	}
 
       template<typename _ForwardIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_assign_aux(_ForwardIterator __first, _ForwardIterator __last,
 		      std::forward_iterator_tag)
@@ -1321,10 +1436,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 			  std::__iterator_category(__first)); }
 #endif
 
+      _GLIBCXX20_CONSTEXPR
       void
       _M_fill_insert(iterator __position, size_type __n, bool __x);
 
       template<typename _InputIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_insert_range(iterator __pos, _InputIterator __first,
 			_InputIterator __last, std::input_iterator_tag)
@@ -1337,13 +1454,16 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	}
 
       template<typename _ForwardIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_insert_range(iterator __position, _ForwardIterator __first,
 			_ForwardIterator __last, std::forward_iterator_tag);
 
+      _GLIBCXX20_CONSTEXPR
       void
       _M_insert_aux(iterator __position, bool __x);
 
+      _GLIBCXX20_CONSTEXPR
       size_type
       _M_check_len(size_type __n, const char* __s) const
       {
@@ -1354,25 +1474,40 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	return (__len < size() || __len > max_size()) ? max_size() : __len;
       }
 
+      _GLIBCXX20_CONSTEXPR
       void
       _M_erase_at_end(iterator __pos)
       { this->_M_impl._M_finish = __pos; }
 
+      _GLIBCXX20_CONSTEXPR
       iterator
       _M_erase(iterator __pos);
 
+      _GLIBCXX20_CONSTEXPR
       iterator
       _M_erase(iterator __first, iterator __last);
-  };
+
+    protected:
+      // _GLIBCXX_RESOLVE_LIB_DEFECTS
+      // DR 464. Suggestion for new member functions in standard containers.
+      // N.B. DR 464 says nothing about vector<bool> but we need something
+      // here due to the using-declaration in __gnu_debug::vector.
+      // vector class.
+#if __cplusplus >= 201103L
+      void data() = delete;
+#else
+      void data() { }
+#endif
+    };
 
 _GLIBCXX_END_NAMESPACE_CONTAINER
 
+  // Fill a partial word.
+  _GLIBCXX20_CONSTEXPR
   inline void
-  __fill_bvector(_GLIBCXX_STD_C::_Bit_type * __v,
-		 unsigned int __first, unsigned int __last, bool __x)
+  __fill_bvector(_Bit_type* __v, unsigned int __first, unsigned int __last,
+		 bool __x) _GLIBCXX_NOEXCEPT
   {
-    using _GLIBCXX_STD_C::_Bit_type;
-    using _GLIBCXX_STD_C::_S_word_bit;
     const _Bit_type __fmask = ~0ul << __first;
     const _Bit_type __lmask = ~0ul >> (_S_word_bit - __last);
     const _Bit_type __mask = __fmask & __lmask;
@@ -1383,20 +1518,36 @@ _GLIBCXX_END_NAMESPACE_CONTAINER
       *__v &= ~__mask;
   }
 
+  // Fill N full words, as if using memset, but usable in constant expressions.
+  __attribute__((__nonnull__))
+  _GLIBCXX20_CONSTEXPR
+  inline void
+  __fill_bvector_n(_Bit_type* __p, size_t __n, bool __x) _GLIBCXX_NOEXCEPT
+  {
+#if __has_builtin(__builtin_is_constant_evaluated)
+    if (__builtin_is_constant_evaluated())
+    {
+      for (size_t __i = 0; __i < __n; ++__i)
+	__p[__i] = __x ? ~0ul : 0ul;
+      return;
+    }
+#endif
+    __builtin_memset(__p, __x ? ~0 : 0, __n * sizeof(_Bit_type));
+  }
+
+
+  _GLIBCXX20_CONSTEXPR
   inline void
   __fill_a1(_GLIBCXX_STD_C::_Bit_iterator __first,
 	    _GLIBCXX_STD_C::_Bit_iterator __last, const bool& __x)
   {
-    using _GLIBCXX_STD_C::_Bit_type;
-    using _GLIBCXX_STD_C::_S_word_bit;
     if (__first._M_p != __last._M_p)
       {
 	_Bit_type* __first_p = __first._M_p;
 	if (__first._M_offset != 0)
 	  __fill_bvector(__first_p++, __first._M_offset, _S_word_bit, __x);
 
-	__builtin_memset(__first_p, __x ? ~0 : 0,
-			 (__last._M_p - __first_p) * sizeof(_Bit_type));
+	__fill_bvector_n(__first_p, __last._M_p - __first_p, __x);
 
 	if (__last._M_offset != 0)
 	  __fill_bvector(__last._M_p, 0, __last._M_offset, __x);
diff --git a/libstdc++-v3/include/bits/stl_iterator.h b/libstdc++-v3/include/bits/stl_iterator.h
index 0090b2d..f6504ec 100644
--- a/libstdc++-v3/include/bits/stl_iterator.h
+++ b/libstdc++-v3/include/bits/stl_iterator.h
@@ -2472,6 +2472,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   /// @} group iterators
 
   template<typename _Iterator>
+    _GLIBCXX20_CONSTEXPR
     auto
     __niter_base(move_iterator<_Iterator> __it)
     -> decltype(make_move_iterator(__niter_base(__it.base())))
@@ -2485,6 +2486,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     };
 
   template<typename _Iterator>
+    _GLIBCXX20_CONSTEXPR
     auto
     __miter_base(move_iterator<_Iterator> __it)
     -> decltype(__miter_base(__it.base()))
diff --git a/libstdc++-v3/include/bits/stl_list.h b/libstdc++-v3/include/bits/stl_list.h
index 96d2a2f..ffaaa6d 100644
--- a/libstdc++-v3/include/bits/stl_list.h
+++ b/libstdc++-v3/include/bits/stl_list.h
@@ -174,8 +174,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  _Cmp _M_cmp;
 
 	  bool
-	  operator()(const __detail::_List_node_base* __lhs,
-		     const __detail::_List_node_base* __rhs) /* not const */
+	  operator()(__detail::_List_node_base* __lhs,
+		     __detail::_List_node_base* __rhs) /* not const */
 	  { return _M_cmp(*_Iter(__lhs), *_Iter(__rhs)); }
 	};
 
@@ -183,8 +183,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	struct _Ptr_cmp<_Iter, void>
 	{
 	  bool
-	  operator()(const __detail::_List_node_base* __lhs,
-		     const __detail::_List_node_base* __rhs) const
+	  operator()(__detail::_List_node_base* __lhs,
+		     __detail::_List_node_base* __rhs) const
 	  { return *_Iter(__lhs) < *_Iter(__rhs); }
 	};
 
diff --git a/libstdc++-v3/include/bits/stl_pair.h b/libstdc++-v3/include/bits/stl_pair.h
index 5b400da..6081e0c 100644
--- a/libstdc++-v3/include/bits/stl_pair.h
+++ b/libstdc++-v3/include/bits/stl_pair.h
@@ -771,6 +771,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     struct tuple_element<1, pair<_Tp1, _Tp2>>
     { typedef _Tp2 type; };
 
+#if __cplusplus >= 201703L
+  template<typename _Tp1, typename _Tp2>
+    inline constexpr size_t tuple_size_v<pair<_Tp1, _Tp2>> = 2;
+
+  template<typename _Tp1, typename _Tp2>
+    inline constexpr size_t tuple_size_v<const pair<_Tp1, _Tp2>> = 2;
+#endif
+
   /// @cond undocumented
   template<size_t _Int>
     struct __pair_get;
diff --git a/libstdc++-v3/include/bits/stl_uninitialized.h b/libstdc++-v3/include/bits/stl_uninitialized.h
index 2fee1e1..673f6a0 100644
--- a/libstdc++-v3/include/bits/stl_uninitialized.h
+++ b/libstdc++-v3/include/bits/stl_uninitialized.h
@@ -107,6 +107,26 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     __is_trivial(T) && __is_assignable(T&, U)
 #endif
 
+  template<typename _InputIterator, typename _ForwardIterator>
+    _GLIBCXX20_CONSTEXPR
+    _ForwardIterator
+    __do_uninit_copy(_InputIterator __first, _InputIterator __last,
+		     _ForwardIterator __result)
+    {
+      _ForwardIterator __cur = __result;
+      __try
+	{
+	  for (; __first != __last; ++__first, (void)++__cur)
+	    std::_Construct(std::__addressof(*__cur), *__first);
+	  return __cur;
+	}
+      __catch(...)
+	{
+	  std::_Destroy(__result, __cur);
+	  __throw_exception_again;
+	}
+    }
+
   template<bool _TrivialValueTypes>
     struct __uninitialized_copy
     {
@@ -114,20 +134,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
         static _ForwardIterator
         __uninit_copy(_InputIterator __first, _InputIterator __last,
 		      _ForwardIterator __result)
-        {
-	  _ForwardIterator __cur = __result;
-	  __try
-	    {
-	      for (; __first != __last; ++__first, (void)++__cur)
-		std::_Construct(std::__addressof(*__cur), *__first);
-	      return __cur;
-	    }
-	  __catch(...)
-	    {
-	      std::_Destroy(__result, __cur);
-	      __throw_exception_again;
-	    }
-	}
+	{ return std::__do_uninit_copy(__first, __last, __result); }
     };
 
   template<>
@@ -180,6 +187,24 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   /// @cond undocumented
 
+  template<typename _ForwardIterator, typename _Tp>
+    _GLIBCXX20_CONSTEXPR void
+    __do_uninit_fill(_ForwardIterator __first, _ForwardIterator __last,
+		     const _Tp& __x)
+    {
+      _ForwardIterator __cur = __first;
+      __try
+	{
+	  for (; __cur != __last; ++__cur)
+	    std::_Construct(std::__addressof(*__cur), __x);
+	}
+      __catch(...)
+	{
+	  std::_Destroy(__first, __cur);
+	  __throw_exception_again;
+	}
+    }
+
   template<bool _TrivialValueType>
     struct __uninitialized_fill
     {
@@ -187,19 +212,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
         static void
         __uninit_fill(_ForwardIterator __first, _ForwardIterator __last,
 		      const _Tp& __x)
-        {
-	  _ForwardIterator __cur = __first;
-	  __try
-	    {
-	      for (; __cur != __last; ++__cur)
-		std::_Construct(std::__addressof(*__cur), __x);
-	    }
-	  __catch(...)
-	    {
-	      std::_Destroy(__first, __cur);
-	      __throw_exception_again;
-	    }
-	}
+	{ std::__do_uninit_fill(__first, __last, __x); }
     };
 
   template<>
@@ -242,34 +255,40 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   /// @cond undocumented
 
+  template<typename _ForwardIterator, typename _Size, typename _Tp>
+    _GLIBCXX20_CONSTEXPR
+    _ForwardIterator
+    __do_uninit_fill_n(_ForwardIterator __first, _Size __n, const _Tp& __x)
+    {
+      _ForwardIterator __cur = __first;
+      __try
+	{
+	  for (; __n > 0; --__n, (void) ++__cur)
+	    std::_Construct(std::__addressof(*__cur), __x);
+	  return __cur;
+	}
+      __catch(...)
+	{
+	  std::_Destroy(__first, __cur);
+	  __throw_exception_again;
+	}
+    }
+
   template<bool _TrivialValueType>
     struct __uninitialized_fill_n
     {
       template<typename _ForwardIterator, typename _Size, typename _Tp>
-        static _ForwardIterator
+	static _ForwardIterator
         __uninit_fill_n(_ForwardIterator __first, _Size __n,
 			const _Tp& __x)
-        {
-	  _ForwardIterator __cur = __first;
-	  __try
-	    {
-	      for (; __n > 0; --__n, (void) ++__cur)
-		std::_Construct(std::__addressof(*__cur), __x);
-	      return __cur;
-	    }
-	  __catch(...)
-	    {
-	      std::_Destroy(__first, __cur);
-	      __throw_exception_again;
-	    }
-	}
+	{ return std::__do_uninit_fill_n(__first, __n, __x); }
     };
 
   template<>
     struct __uninitialized_fill_n<true>
     {
       template<typename _ForwardIterator, typename _Size, typename _Tp>
-        static _ForwardIterator
+	static _ForwardIterator
         __uninit_fill_n(_ForwardIterator __first, _Size __n,
 			const _Tp& __x)
         { return std::fill_n(__first, __n, __x); }
@@ -315,11 +334,12 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   // Extensions: versions of uninitialized_copy, uninitialized_fill,
   //  and uninitialized_fill_n that take an allocator parameter.
   //  We dispatch back to the standard versions when we're given the
-  //  default allocator.  For nondefault allocators we do not use 
+  //  default allocator.  For nondefault allocators we do not use
   //  any of the POD optimizations.
 
   template<typename _InputIterator, typename _ForwardIterator,
 	   typename _Allocator>
+    _GLIBCXX20_CONSTEXPR
     _ForwardIterator
     __uninitialized_copy_a(_InputIterator __first, _InputIterator __last,
 			   _ForwardIterator __result, _Allocator& __alloc)
@@ -340,13 +360,21 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     }
 
   template<typename _InputIterator, typename _ForwardIterator, typename _Tp>
+    _GLIBCXX20_CONSTEXPR
     inline _ForwardIterator
     __uninitialized_copy_a(_InputIterator __first, _InputIterator __last,
 			   _ForwardIterator __result, allocator<_Tp>&)
-    { return std::uninitialized_copy(__first, __last, __result); }
+    {
+#ifdef __cpp_lib_is_constant_evaluated
+      if (std::is_constant_evaluated())
+	return std::__do_uninit_copy(__first, __last, __result);
+#endif
+      return std::uninitialized_copy(__first, __last, __result);
+    }
 
   template<typename _InputIterator, typename _ForwardIterator,
 	   typename _Allocator>
+    _GLIBCXX20_CONSTEXPR
     inline _ForwardIterator
     __uninitialized_move_a(_InputIterator __first, _InputIterator __last,
 			   _ForwardIterator __result, _Allocator& __alloc)
@@ -358,6 +386,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   template<typename _InputIterator, typename _ForwardIterator,
 	   typename _Allocator>
+    _GLIBCXX20_CONSTEXPR
     inline _ForwardIterator
     __uninitialized_move_if_noexcept_a(_InputIterator __first,
 				       _InputIterator __last,
@@ -370,6 +399,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     }
 
   template<typename _ForwardIterator, typename _Tp, typename _Allocator>
+    _GLIBCXX20_CONSTEXPR
     void
     __uninitialized_fill_a(_ForwardIterator __first, _ForwardIterator __last,
 			   const _Tp& __x, _Allocator& __alloc)
@@ -389,15 +419,23 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     }
 
   template<typename _ForwardIterator, typename _Tp, typename _Tp2>
+    _GLIBCXX20_CONSTEXPR
     inline void
     __uninitialized_fill_a(_ForwardIterator __first, _ForwardIterator __last,
 			   const _Tp& __x, allocator<_Tp2>&)
-    { std::uninitialized_fill(__first, __last, __x); }
+    {
+#ifdef __cpp_lib_is_constant_evaluated
+      if (std::is_constant_evaluated())
+	return std::__do_uninit_fill(__first, __last, __x);
+#endif
+      std::uninitialized_fill(__first, __last, __x);
+    }
 
   template<typename _ForwardIterator, typename _Size, typename _Tp,
 	   typename _Allocator>
+     _GLIBCXX20_CONSTEXPR
     _ForwardIterator
-    __uninitialized_fill_n_a(_ForwardIterator __first, _Size __n, 
+    __uninitialized_fill_n_a(_ForwardIterator __first, _Size __n,
 			     const _Tp& __x, _Allocator& __alloc)
     {
       _ForwardIterator __cur = __first;
@@ -417,10 +455,17 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   template<typename _ForwardIterator, typename _Size, typename _Tp,
 	   typename _Tp2>
+    _GLIBCXX20_CONSTEXPR
     inline _ForwardIterator
-    __uninitialized_fill_n_a(_ForwardIterator __first, _Size __n, 
+    __uninitialized_fill_n_a(_ForwardIterator __first, _Size __n,
 			     const _Tp& __x, allocator<_Tp2>&)
-    { return std::uninitialized_fill_n(__first, __n, __x); }
+    {
+#ifdef __cpp_lib_is_constant_evaluated
+      if (std::is_constant_evaluated())
+	return std::__do_uninit_fill_n(__first, __n, __x);
+#endif
+      return std::uninitialized_fill_n(__first, __n, __x);
+    }
 
 
   // Extensions: __uninitialized_copy_move, __uninitialized_move_copy,
@@ -483,7 +528,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  __throw_exception_again;
 	}
     }
-  
+
   // __uninitialized_fill_move
   // Fills [result, mid) with x, and moves [first, last) into
   //  [mid, mid + (last - first)).
@@ -582,6 +627,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     struct __uninitialized_default_n_1
     {
       template<typename _ForwardIterator, typename _Size>
+	_GLIBCXX20_CONSTEXPR
         static _ForwardIterator
         __uninit_default_n(_ForwardIterator __first, _Size __n)
         {
@@ -604,6 +650,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     struct __uninitialized_default_n_1<true>
     {
       template<typename _ForwardIterator, typename _Size>
+	_GLIBCXX20_CONSTEXPR
         static _ForwardIterator
         __uninit_default_n(_ForwardIterator __first, _Size __n)
         {
@@ -639,6 +686,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   // __uninitialized_default_n
   // Fills [first, first + n) with value-initialized value_types.
   template<typename _ForwardIterator, typename _Size>
+    _GLIBCXX20_CONSTEXPR
     inline _ForwardIterator
     __uninitialized_default_n(_ForwardIterator __first, _Size __n)
     {
@@ -689,8 +737,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   // Fills [first, first + n) with value_types constructed by the allocator
   // alloc, with no arguments passed to the construct call.
   template<typename _ForwardIterator, typename _Size, typename _Allocator>
-    _ForwardIterator
-    __uninitialized_default_n_a(_ForwardIterator __first, _Size __n, 
+    _GLIBCXX20_CONSTEXPR _ForwardIterator
+    __uninitialized_default_n_a(_ForwardIterator __first, _Size __n,
 				_Allocator& __alloc)
     {
       _ForwardIterator __cur = __first;
@@ -711,8 +759,9 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   // __uninitialized_default_n_a specialization for std::allocator,
   // which ignores the allocator and value-initializes the elements.
   template<typename _ForwardIterator, typename _Size, typename _Tp>
+    _GLIBCXX20_CONSTEXPR
     inline _ForwardIterator
-    __uninitialized_default_n_a(_ForwardIterator __first, _Size __n, 
+    __uninitialized_default_n_a(_ForwardIterator __first, _Size __n,
 				allocator<_Tp>&)
     { return std::__uninitialized_default_n(__first, __n); }
 
@@ -1022,6 +1071,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     : is_trivial<_Tp> { };
 
   template <typename _Tp, typename _Up>
+    _GLIBCXX20_CONSTEXPR
     inline __enable_if_t<std::__is_bitwise_relocatable<_Tp>::value, _Tp*>
     __relocate_a_1(_Tp* __first, _Tp* __last,
 		   _Tp* __result, allocator<_Up>&) noexcept
@@ -1034,6 +1084,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   template <typename _InputIterator, typename _ForwardIterator,
 	    typename _Allocator>
+    _GLIBCXX20_CONSTEXPR
     inline _ForwardIterator
     __relocate_a_1(_InputIterator __first, _InputIterator __last,
 		   _ForwardIterator __result, _Allocator& __alloc)
@@ -1056,6 +1107,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   template <typename _InputIterator, typename _ForwardIterator,
 	    typename _Allocator>
+    _GLIBCXX20_CONSTEXPR
     inline _ForwardIterator
     __relocate_a(_InputIterator __first, _InputIterator __last,
 		 _ForwardIterator __result, _Allocator& __alloc)
diff --git a/libstdc++-v3/include/bits/stl_vector.h b/libstdc++-v3/include/bits/stl_vector.h
index 2023581..bd15b6b 100644
--- a/libstdc++-v3/include/bits/stl_vector.h
+++ b/libstdc++-v3/include/bits/stl_vector.h
@@ -94,17 +94,20 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	pointer _M_finish;
 	pointer _M_end_of_storage;
 
+	_GLIBCXX20_CONSTEXPR
 	_Vector_impl_data() _GLIBCXX_NOEXCEPT
 	: _M_start(), _M_finish(), _M_end_of_storage()
 	{ }
 
 #if __cplusplus >= 201103L
+	_GLIBCXX20_CONSTEXPR
 	_Vector_impl_data(_Vector_impl_data&& __x) noexcept
 	: _M_start(__x._M_start), _M_finish(__x._M_finish),
 	  _M_end_of_storage(__x._M_end_of_storage)
 	{ __x._M_start = __x._M_finish = __x._M_end_of_storage = pointer(); }
 #endif
 
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_copy_data(_Vector_impl_data const& __x) _GLIBCXX_NOEXCEPT
 	{
@@ -113,6 +116,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	  _M_end_of_storage = __x._M_end_of_storage;
 	}
 
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_swap_data(_Vector_impl_data& __x) _GLIBCXX_NOEXCEPT
 	{
@@ -128,11 +132,13 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       struct _Vector_impl
 	: public _Tp_alloc_type, public _Vector_impl_data
       {
+	_GLIBCXX20_CONSTEXPR
 	_Vector_impl() _GLIBCXX_NOEXCEPT_IF(
 	    is_nothrow_default_constructible<_Tp_alloc_type>::value)
 	: _Tp_alloc_type()
 	{ }
 
+	_GLIBCXX20_CONSTEXPR
 	_Vector_impl(_Tp_alloc_type const& __a) _GLIBCXX_NOEXCEPT
 	: _Tp_alloc_type(__a)
 	{ }
@@ -140,14 +146,17 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #if __cplusplus >= 201103L
 	// Not defaulted, to enforce noexcept(true) even when
 	// !is_nothrow_move_constructible<_Tp_alloc_type>.
+	_GLIBCXX20_CONSTEXPR
 	_Vector_impl(_Vector_impl&& __x) noexcept
 	: _Tp_alloc_type(std::move(__x)), _Vector_impl_data(std::move(__x))
 	{ }
 
+	_GLIBCXX20_CONSTEXPR
 	_Vector_impl(_Tp_alloc_type&& __a) noexcept
 	: _Tp_alloc_type(std::move(__a))
 	{ }
 
+	_GLIBCXX20_CONSTEXPR
 	_Vector_impl(_Tp_alloc_type&& __a, _Vector_impl&& __rv) noexcept
 	: _Tp_alloc_type(std::move(__a)), _Vector_impl_data(std::move(__rv))
 	{ }
@@ -160,15 +169,17 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	    typedef typename __gnu_cxx::__alloc_traits<_Tp_alloc_type>
 	      ::size_type size_type;
 
-	    static void _S_shrink(_Vector_impl&, size_type) { }
-	    static void _S_on_dealloc(_Vector_impl&) { }
+	    static _GLIBCXX20_CONSTEXPR void
+	    _S_shrink(_Vector_impl&, size_type) { }
+	    static _GLIBCXX20_CONSTEXPR void
+	    _S_on_dealloc(_Vector_impl&) { }
 
 	    typedef _Vector_impl& _Reinit;
 
 	    struct _Grow
 	    {
-	      _Grow(_Vector_impl&, size_type) { }
-	      void _M_grew(size_type) { }
+	      _GLIBCXX20_CONSTEXPR _Grow(_Vector_impl&, size_type) { }
+	      _GLIBCXX20_CONSTEXPR void _M_grew(size_type) { }
 	    };
 	  };
 
@@ -181,22 +192,25 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
 	    // Adjust ASan annotation for [_M_start, _M_end_of_storage) to
 	    // mark end of valid region as __curr instead of __prev.
-	    static void
+	    static _GLIBCXX20_CONSTEXPR void
 	    _S_adjust(_Vector_impl& __impl, pointer __prev, pointer __curr)
 	    {
+#if __has_builtin(__builtin_is_constant_evaluated)
+	      if (!__builtin_is_constant_evaluated())
+#endif
 	      __sanitizer_annotate_contiguous_container(__impl._M_start,
 		  __impl._M_end_of_storage, __prev, __curr);
 	    }
 
-	    static void
+	    static _GLIBCXX20_CONSTEXPR void
 	    _S_grow(_Vector_impl& __impl, size_type __n)
 	    { _S_adjust(__impl, __impl._M_finish, __impl._M_finish + __n); }
 
-	    static void
+	    static _GLIBCXX20_CONSTEXPR void
 	    _S_shrink(_Vector_impl& __impl, size_type __n)
 	    { _S_adjust(__impl, __impl._M_finish + __n, __impl._M_finish); }
 
-	    static void
+	    static _GLIBCXX20_CONSTEXPR void
 	    _S_on_dealloc(_Vector_impl& __impl)
 	    {
 	      if (__impl._M_start)
@@ -206,12 +220,14 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	    // Used on reallocation to tell ASan unused capacity is invalid.
 	    struct _Reinit
 	    {
-	      explicit _Reinit(_Vector_impl& __impl) : _M_impl(__impl)
+	      explicit _GLIBCXX20_CONSTEXPR
+	      _Reinit(_Vector_impl& __impl) : _M_impl(__impl)
 	      {
 		// Mark unused capacity as valid again before deallocating it.
 		_S_on_dealloc(_M_impl);
 	      }
 
+	      _GLIBCXX20_CONSTEXPR
 	      ~_Reinit()
 	      {
 		// Mark unused capacity as invalid after reallocation.
@@ -231,12 +247,15 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	    // Tell ASan when unused capacity is initialized to be valid.
 	    struct _Grow
 	    {
+	      _GLIBCXX20_CONSTEXPR
 	      _Grow(_Vector_impl& __impl, size_type __n)
 	      : _M_impl(__impl), _M_n(__n)
 	      { _S_grow(_M_impl, __n); }
 
+	      _GLIBCXX20_CONSTEXPR
 	      ~_Grow() { if (_M_n) _S_shrink(_M_impl, _M_n); }
 
+	      _GLIBCXX20_CONSTEXPR
 	      void _M_grew(size_type __n) { _M_n -= __n; }
 
 #if __cplusplus >= 201103L
@@ -272,14 +291,17 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     public:
       typedef _Alloc allocator_type;
 
+      _GLIBCXX20_CONSTEXPR
       _Tp_alloc_type&
       _M_get_Tp_allocator() _GLIBCXX_NOEXCEPT
       { return this->_M_impl; }
 
+      _GLIBCXX20_CONSTEXPR
       const _Tp_alloc_type&
       _M_get_Tp_allocator() const _GLIBCXX_NOEXCEPT
       { return this->_M_impl; }
 
+      _GLIBCXX20_CONSTEXPR
       allocator_type
       get_allocator() const _GLIBCXX_NOEXCEPT
       { return allocator_type(_M_get_Tp_allocator()); }
@@ -290,16 +312,19 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       _Vector_base() { }
 #endif
 
+      _GLIBCXX20_CONSTEXPR
       _Vector_base(const allocator_type& __a) _GLIBCXX_NOEXCEPT
       : _M_impl(__a) { }
 
       // Kept for ABI compatibility.
 #if !_GLIBCXX_INLINE_VERSION
+      _GLIBCXX20_CONSTEXPR
       _Vector_base(size_t __n)
       : _M_impl()
       { _M_create_storage(__n); }
 #endif
 
+      _GLIBCXX20_CONSTEXPR
       _Vector_base(size_t __n, const allocator_type& __a)
       : _M_impl(__a)
       { _M_create_storage(__n); }
@@ -309,9 +334,11 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
       // Kept for ABI compatibility.
 # if !_GLIBCXX_INLINE_VERSION
+      _GLIBCXX20_CONSTEXPR
       _Vector_base(_Tp_alloc_type&& __a) noexcept
       : _M_impl(std::move(__a)) { }
 
+      _GLIBCXX20_CONSTEXPR
       _Vector_base(_Vector_base&& __x, const allocator_type& __a)
       : _M_impl(__a)
       {
@@ -325,11 +352,13 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 # endif
 
+      _GLIBCXX20_CONSTEXPR
       _Vector_base(const allocator_type& __a, _Vector_base&& __x)
       : _M_impl(_Tp_alloc_type(__a), std::move(__x._M_impl))
       { }
 #endif
 
+      _GLIBCXX20_CONSTEXPR
       ~_Vector_base() _GLIBCXX_NOEXCEPT
       {
 	_M_deallocate(_M_impl._M_start,
@@ -339,6 +368,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     public:
       _Vector_impl _M_impl;
 
+      _GLIBCXX20_CONSTEXPR
       pointer
       _M_allocate(size_t __n)
       {
@@ -346,6 +376,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	return __n != 0 ? _Tr::allocate(_M_impl, __n) : pointer();
       }
 
+      _GLIBCXX20_CONSTEXPR
       void
       _M_deallocate(pointer __p, size_t __n)
       {
@@ -355,6 +386,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
     protected:
+      _GLIBCXX20_CONSTEXPR
       void
       _M_create_storage(size_t __n)
       {
@@ -443,25 +475,29 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       static constexpr bool
       _S_use_relocate()
       {
+#if __cplusplus >= 202002L && __has_builtin(__builtin_is_constant_evaluated)
+	if (__builtin_is_constant_evaluated())
+	  return false; // Cannot use memcpy in constant evaluation contexts.
+#endif
 	// Instantiating std::__relocate_a might cause an error outside the
 	// immediate context (in __relocate_object_a's noexcept-specifier),
 	// so only do it if we know the type can be move-inserted into *this.
 	return _S_nothrow_relocate(__is_move_insertable<_Tp_alloc_type>{});
       }
 
-      static pointer
+      static _GLIBCXX20_CONSTEXPR pointer
       _S_do_relocate(pointer __first, pointer __last, pointer __result,
 		     _Tp_alloc_type& __alloc, true_type) noexcept
       {
 	return std::__relocate_a(__first, __last, __result, __alloc);
       }
 
-      static pointer
+      static _GLIBCXX20_CONSTEXPR pointer
       _S_do_relocate(pointer, pointer, pointer __result,
 		     _Tp_alloc_type&, false_type) noexcept
       { return __result; }
 
-      static pointer
+      static _GLIBCXX20_CONSTEXPR pointer
       _S_relocate(pointer __first, pointer __last, pointer __result,
 		  _Tp_alloc_type& __alloc) noexcept
       {
@@ -494,6 +530,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  @param  __a  An allocator object.
        */
       explicit
+      _GLIBCXX20_CONSTEXPR
       vector(const allocator_type& __a) _GLIBCXX_NOEXCEPT
       : _Base(__a) { }
 
@@ -507,6 +544,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  constructed elements.
        */
       explicit
+      _GLIBCXX20_CONSTEXPR
       vector(size_type __n, const allocator_type& __a = allocator_type())
       : _Base(_S_check_init_len(__n, __a), __a)
       { _M_default_initialize(__n); }
@@ -519,6 +557,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *
        *  This constructor fills the %vector with @a __n copies of @a __value.
        */
+      _GLIBCXX20_CONSTEXPR
       vector(size_type __n, const value_type& __value,
 	     const allocator_type& __a = allocator_type())
       : _Base(_S_check_init_len(__n, __a), __a)
@@ -550,6 +589,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  The newly-created %vector uses a copy of the allocator object used
        *  by @a __x (unless the allocator traits dictate a different object).
        */
+      _GLIBCXX20_CONSTEXPR
       vector(const vector& __x)
       : _Base(__x.size(),
 	_Alloc_traits::_S_select_on_copy(__x._M_get_Tp_allocator()))
@@ -572,6 +612,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       vector(vector&&) noexcept = default;
 
       /// Copy constructor with alternative allocator
+      _GLIBCXX20_CONSTEXPR
       vector(const vector& __x, const __type_identity_t<allocator_type>& __a)
       : _Base(__x.size(), __a)
       {
@@ -582,10 +623,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
     private:
+      _GLIBCXX20_CONSTEXPR
       vector(vector&& __rv, const allocator_type& __m, true_type) noexcept
       : _Base(__m, std::move(__rv))
       { }
 
+      _GLIBCXX20_CONSTEXPR
       vector(vector&& __rv, const allocator_type& __m, false_type)
       : _Base(__m)
       {
@@ -604,6 +647,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
     public:
       /// Move constructor with alternative allocator
+      _GLIBCXX20_CONSTEXPR
       vector(vector&& __rv, const __type_identity_t<allocator_type>& __m)
       noexcept( noexcept(
 	vector(std::declval<vector&&>(), std::declval<const allocator_type&>(),
@@ -622,6 +666,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  This will call the element type's copy constructor N times
        *  (where N is @a __l.size()) and do no memory reallocation.
        */
+      _GLIBCXX20_CONSTEXPR
       vector(initializer_list<value_type> __l,
 	     const allocator_type& __a = allocator_type())
       : _Base(__a)
@@ -650,6 +695,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #if __cplusplus >= 201103L
       template<typename _InputIterator,
 	       typename = std::_RequireInputIter<_InputIterator>>
+	_GLIBCXX20_CONSTEXPR
 	vector(_InputIterator __first, _InputIterator __last,
 	       const allocator_type& __a = allocator_type())
 	: _Base(__a)
@@ -675,6 +721,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  not touched in any way.  Managing the pointer is the user's
        *  responsibility.
        */
+      _GLIBCXX20_CONSTEXPR
       ~vector() _GLIBCXX_NOEXCEPT
       {
 	std::_Destroy(this->_M_impl._M_start, this->_M_impl._M_finish,
@@ -691,6 +738,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *
        *  Whether the allocator is copied depends on the allocator traits.
        */
+      _GLIBCXX20_CONSTEXPR
       vector&
       operator=(const vector& __x);
 
@@ -705,6 +753,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *
        *  Whether the allocator is moved depends on the allocator traits.
        */
+      _GLIBCXX20_CONSTEXPR
       vector&
       operator=(vector&& __x) noexcept(_Alloc_traits::_S_nothrow_move())
       {
@@ -726,6 +775,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  that the resulting %vector's size is the same as the number
        *  of elements assigned.
        */
+      _GLIBCXX20_CONSTEXPR
       vector&
       operator=(initializer_list<value_type> __l)
       {
@@ -745,6 +795,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  %vector and that the resulting %vector's size is the same as
        *  the number of elements assigned.
        */
+      _GLIBCXX20_CONSTEXPR
       void
       assign(size_type __n, const value_type& __val)
       { _M_fill_assign(__n, __val); }
@@ -764,6 +815,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #if __cplusplus >= 201103L
       template<typename _InputIterator,
 	       typename = std::_RequireInputIter<_InputIterator>>
+	_GLIBCXX20_CONSTEXPR
 	void
 	assign(_InputIterator __first, _InputIterator __last)
 	{ _M_assign_dispatch(__first, __last, __false_type()); }
@@ -790,6 +842,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  that the resulting %vector's size is the same as the number
        *  of elements assigned.
        */
+      _GLIBCXX20_CONSTEXPR
       void
       assign(initializer_list<value_type> __l)
       {
@@ -807,7 +860,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  element in the %vector.  Iteration is done in ordinary
        *  element order.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       iterator
       begin() _GLIBCXX_NOEXCEPT
       { return iterator(this->_M_impl._M_start); }
@@ -817,7 +870,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  first element in the %vector.  Iteration is done in ordinary
        *  element order.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_iterator
       begin() const _GLIBCXX_NOEXCEPT
       { return const_iterator(this->_M_impl._M_start); }
@@ -827,7 +880,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  element in the %vector.  Iteration is done in ordinary
        *  element order.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       iterator
       end() _GLIBCXX_NOEXCEPT
       { return iterator(this->_M_impl._M_finish); }
@@ -837,7 +890,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  the last element in the %vector.  Iteration is done in
        *  ordinary element order.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_iterator
       end() const _GLIBCXX_NOEXCEPT
       { return const_iterator(this->_M_impl._M_finish); }
@@ -847,7 +900,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  last element in the %vector.  Iteration is done in reverse
        *  element order.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       reverse_iterator
       rbegin() _GLIBCXX_NOEXCEPT
       { return reverse_iterator(end()); }
@@ -857,7 +910,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  to the last element in the %vector.  Iteration is done in
        *  reverse element order.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_reverse_iterator
       rbegin() const _GLIBCXX_NOEXCEPT
       { return const_reverse_iterator(end()); }
@@ -867,7 +920,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  before the first element in the %vector.  Iteration is done
        *  in reverse element order.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       reverse_iterator
       rend() _GLIBCXX_NOEXCEPT
       { return reverse_iterator(begin()); }
@@ -877,7 +930,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  to one before the first element in the %vector.  Iteration
        *  is done in reverse element order.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_reverse_iterator
       rend() const _GLIBCXX_NOEXCEPT
       { return const_reverse_iterator(begin()); }
@@ -888,7 +941,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  first element in the %vector.  Iteration is done in ordinary
        *  element order.
        */
-      [[__nodiscard__]]
+      [[__nodiscard__]] _GLIBCXX20_CONSTEXPR
       const_iterator
       cbegin() const noexcept
       { return const_iterator(this->_M_impl._M_start); }
@@ -898,7 +951,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  the last element in the %vector.  Iteration is done in
        *  ordinary element order.
        */
-      [[__nodiscard__]]
+      [[__nodiscard__]] _GLIBCXX20_CONSTEXPR
       const_iterator
       cend() const noexcept
       { return const_iterator(this->_M_impl._M_finish); }
@@ -908,7 +961,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  to the last element in the %vector.  Iteration is done in
        *  reverse element order.
        */
-      [[__nodiscard__]]
+      [[__nodiscard__]] _GLIBCXX20_CONSTEXPR
       const_reverse_iterator
       crbegin() const noexcept
       { return const_reverse_iterator(end()); }
@@ -918,7 +971,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  to one before the first element in the %vector.  Iteration
        *  is done in reverse element order.
        */
-      [[__nodiscard__]]
+      [[__nodiscard__]] _GLIBCXX20_CONSTEXPR
       const_reverse_iterator
       crend() const noexcept
       { return const_reverse_iterator(begin()); }
@@ -926,13 +979,13 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
       // [23.2.4.2] capacity
       /**  Returns the number of elements in the %vector.  */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       size_type
       size() const _GLIBCXX_NOEXCEPT
       { return size_type(this->_M_impl._M_finish - this->_M_impl._M_start); }
 
       /**  Returns the size() of the largest possible %vector.  */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       size_type
       max_size() const _GLIBCXX_NOEXCEPT
       { return _S_max_size(_M_get_Tp_allocator()); }
@@ -947,6 +1000,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  %vector's current size the %vector is truncated, otherwise
        *  default constructed elements are appended.
        */
+      _GLIBCXX20_CONSTEXPR
       void
       resize(size_type __new_size)
       {
@@ -967,6 +1021,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  the %vector is extended and new elements are populated with
        *  given data.
        */
+      _GLIBCXX20_CONSTEXPR
       void
       resize(size_type __new_size, const value_type& __x)
       {
@@ -987,6 +1042,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  the %vector is extended and new elements are populated with
        *  given data.
        */
+      _GLIBCXX20_CONSTEXPR
       void
       resize(size_type __new_size, value_type __x = value_type())
       {
@@ -999,6 +1055,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
 #if __cplusplus >= 201103L
       /**  A non-binding request to reduce capacity() to size().  */
+      _GLIBCXX20_CONSTEXPR
       void
       shrink_to_fit()
       { _M_shrink_to_fit(); }
@@ -1008,7 +1065,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  Returns the total number of elements that the %vector can
        *  hold before needing to allocate more memory.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       size_type
       capacity() const _GLIBCXX_NOEXCEPT
       { return size_type(this->_M_impl._M_end_of_storage
@@ -1018,7 +1075,8 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  Returns true if the %vector is empty.  (Thus begin() would
        *  equal end().)
        */
-      _GLIBCXX_NODISCARD bool
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
+      bool
       empty() const _GLIBCXX_NOEXCEPT
       { return begin() == end(); }
 
@@ -1039,6 +1097,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  %advance, and thus prevent a possible reallocation of memory
        *  and copying of %vector data.
        */
+      _GLIBCXX20_CONSTEXPR
       void
       reserve(size_type __n);
 
@@ -1054,7 +1113,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  out_of_range lookups are not defined. (For checked lookups
        *  see at().)
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       reference
       operator[](size_type __n) _GLIBCXX_NOEXCEPT
       {
@@ -1073,7 +1132,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  out_of_range lookups are not defined. (For checked lookups
        *  see at().)
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_reference
       operator[](size_type __n) const _GLIBCXX_NOEXCEPT
       {
@@ -1083,6 +1142,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
     protected:
       /// Safety check used only from at().
+      _GLIBCXX20_CONSTEXPR
       void
       _M_range_check(size_type __n) const
       {
@@ -1105,6 +1165,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  is first checked that it is in the range of the vector.  The
        *  function throws out_of_range if the check fails.
        */
+      _GLIBCXX20_CONSTEXPR
       reference
       at(size_type __n)
       {
@@ -1123,6 +1184,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  is first checked that it is in the range of the vector.  The
        *  function throws out_of_range if the check fails.
        */
+      _GLIBCXX20_CONSTEXPR
       const_reference
       at(size_type __n) const
       {
@@ -1134,7 +1196,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  Returns a read/write reference to the data at the first
        *  element of the %vector.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       reference
       front() _GLIBCXX_NOEXCEPT
       {
@@ -1146,7 +1208,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  Returns a read-only (constant) reference to the data at the first
        *  element of the %vector.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_reference
       front() const _GLIBCXX_NOEXCEPT
       {
@@ -1158,7 +1220,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  Returns a read/write reference to the data at the last
        *  element of the %vector.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       reference
       back() _GLIBCXX_NOEXCEPT
       {
@@ -1170,7 +1232,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  Returns a read-only (constant) reference to the data at the
        *  last element of the %vector.
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const_reference
       back() const _GLIBCXX_NOEXCEPT
       {
@@ -1185,12 +1247,12 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *   Returns a pointer such that [data(), data() + size()) is a valid
        *   range.  For a non-empty %vector, data() == &front().
        */
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       _Tp*
       data() _GLIBCXX_NOEXCEPT
       { return _M_data_ptr(this->_M_impl._M_start); }
 
-      _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD _GLIBCXX20_CONSTEXPR
       const _Tp*
       data() const _GLIBCXX_NOEXCEPT
       { return _M_data_ptr(this->_M_impl._M_start); }
@@ -1206,6 +1268,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  done in constant time if the %vector has preallocated space
        *  available.
        */
+      _GLIBCXX20_CONSTEXPR
       void
       push_back(const value_type& __x)
       {
@@ -1222,12 +1285,14 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
 #if __cplusplus >= 201103L
+      _GLIBCXX20_CONSTEXPR
       void
       push_back(value_type&& __x)
       { emplace_back(std::move(__x)); }
 
       template<typename... _Args>
 #if __cplusplus > 201402L
+	_GLIBCXX20_CONSTEXPR
 	reference
 #else
 	void
@@ -1244,6 +1309,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  data is needed, it should be retrieved before pop_back() is
        *  called.
        */
+      _GLIBCXX20_CONSTEXPR
       void
       pop_back() _GLIBCXX_NOEXCEPT
       {
@@ -1267,6 +1333,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  std::list.
        */
       template<typename... _Args>
+	_GLIBCXX20_CONSTEXPR
 	iterator
 	emplace(const_iterator __position, _Args&&... __args)
 	{ return _M_emplace_aux(__position, std::forward<_Args>(__args)...); }
@@ -1282,6 +1349,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  could be expensive for a %vector and if it is frequently
        *  used the user should consider using std::list.
        */
+      _GLIBCXX20_CONSTEXPR
       iterator
       insert(const_iterator __position, const value_type& __x);
 #else
@@ -1312,6 +1380,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  could be expensive for a %vector and if it is frequently
        *  used the user should consider using std::list.
        */
+      _GLIBCXX20_CONSTEXPR
       iterator
       insert(const_iterator __position, value_type&& __x)
       { return _M_insert_rval(__position, std::move(__x)); }
@@ -1329,6 +1398,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  %vector and if it is frequently used the user should
        *  consider using std::list.
        */
+      _GLIBCXX20_CONSTEXPR
       iterator
       insert(const_iterator __position, initializer_list<value_type> __l)
       {
@@ -1354,6 +1424,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  %vector and if it is frequently used the user should
        *  consider using std::list.
        */
+      _GLIBCXX20_CONSTEXPR
       iterator
       insert(const_iterator __position, size_type __n, const value_type& __x)
       {
@@ -1398,6 +1469,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        */
       template<typename _InputIterator,
 	       typename = std::_RequireInputIter<_InputIterator>>
+	_GLIBCXX20_CONSTEXPR
 	iterator
 	insert(const_iterator __position, _InputIterator __first,
 	       _InputIterator __last)
@@ -1448,6 +1520,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  the pointed-to memory is not touched in any way.  Managing
        *  the pointer is the user's responsibility.
        */
+      _GLIBCXX20_CONSTEXPR
       iterator
 #if __cplusplus >= 201103L
       erase(const_iterator __position)
@@ -1475,6 +1548,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  pointers, the pointed-to memory is not touched in any way.
        *  Managing the pointer is the user's responsibility.
        */
+      _GLIBCXX20_CONSTEXPR
       iterator
 #if __cplusplus >= 201103L
       erase(const_iterator __first, const_iterator __last)
@@ -1499,6 +1573,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *
        *  Whether the allocators are swapped depends on the allocator traits.
        */
+      _GLIBCXX20_CONSTEXPR
       void
       swap(vector& __x) _GLIBCXX_NOEXCEPT
       {
@@ -1517,6 +1592,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  pointed-to memory is not touched in any way.  Managing the pointer is
        *  the user's responsibility.
        */
+      _GLIBCXX20_CONSTEXPR
       void
       clear() _GLIBCXX_NOEXCEPT
       { _M_erase_at_end(this->_M_impl._M_start); }
@@ -1527,6 +1603,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
        *  obtain @a n bytes of memory, and then copies [first,last) into it.
        */
       template<typename _ForwardIterator>
+	_GLIBCXX20_CONSTEXPR
 	pointer
 	_M_allocate_and_copy(size_type __n,
 			     _ForwardIterator __first, _ForwardIterator __last)
@@ -1577,6 +1654,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
       // Called by the second initialize_dispatch above
       template<typename _InputIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_range_initialize(_InputIterator __first, _InputIterator __last,
 			    std::input_iterator_tag)
@@ -1596,6 +1674,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
       // Called by the second initialize_dispatch above
       template<typename _ForwardIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_range_initialize(_ForwardIterator __first, _ForwardIterator __last,
 			    std::forward_iterator_tag)
@@ -1612,6 +1691,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
       // Called by the first initialize_dispatch above and by the
       // vector(n,value,a) constructor.
+      _GLIBCXX20_CONSTEXPR
       void
       _M_fill_initialize(size_type __n, const value_type& __value)
       {
@@ -1622,6 +1702,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
 #if __cplusplus >= 201103L
       // Called by the vector(n) constructor.
+      _GLIBCXX20_CONSTEXPR
       void
       _M_default_initialize(size_type __n)
       {
@@ -1639,12 +1720,14 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       // _GLIBCXX_RESOLVE_LIB_DEFECTS
       // 438. Ambiguity in the "do the right thing" clause
       template<typename _Integer>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_assign_dispatch(_Integer __n, _Integer __val, __true_type)
 	{ _M_fill_assign(__n, __val); }
 
       // Called by the range assign to implement [23.1.1]/9
       template<typename _InputIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_assign_dispatch(_InputIterator __first, _InputIterator __last,
 			   __false_type)
@@ -1652,18 +1735,21 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
       // Called by the second assign_dispatch above
       template<typename _InputIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_assign_aux(_InputIterator __first, _InputIterator __last,
 		      std::input_iterator_tag);
 
       // Called by the second assign_dispatch above
       template<typename _ForwardIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_assign_aux(_ForwardIterator __first, _ForwardIterator __last,
 		      std::forward_iterator_tag);
 
       // Called by assign(n,t), and the range assign when it turns out
       // to be the same thing.
+      _GLIBCXX20_CONSTEXPR
       void
       _M_fill_assign(size_type __n, const value_type& __val);
 
@@ -1674,6 +1760,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       // _GLIBCXX_RESOLVE_LIB_DEFECTS
       // 438. Ambiguity in the "do the right thing" clause
       template<typename _Integer>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_insert_dispatch(iterator __pos, _Integer __n, _Integer __val,
 			   __true_type)
@@ -1681,6 +1768,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
       // Called by the range insert to implement [23.1.1]/9
       template<typename _InputIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_insert_dispatch(iterator __pos, _InputIterator __first,
 			   _InputIterator __last, __false_type)
@@ -1691,26 +1779,31 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
       // Called by the second insert_dispatch above
       template<typename _InputIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_range_insert(iterator __pos, _InputIterator __first,
 			_InputIterator __last, std::input_iterator_tag);
 
       // Called by the second insert_dispatch above
       template<typename _ForwardIterator>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_range_insert(iterator __pos, _ForwardIterator __first,
 			_ForwardIterator __last, std::forward_iterator_tag);
 
       // Called by insert(p,n,x), and the range insert when it turns out to be
       // the same thing.
+      _GLIBCXX20_CONSTEXPR
       void
       _M_fill_insert(iterator __pos, size_type __n, const value_type& __x);
 
 #if __cplusplus >= 201103L
       // Called by resize(n).
+      _GLIBCXX20_CONSTEXPR
       void
       _M_default_append(size_type __n);
 
+      _GLIBCXX20_CONSTEXPR
       bool
       _M_shrink_to_fit();
 #endif
@@ -1728,53 +1821,69 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       struct _Temporary_value
       {
 	template<typename... _Args>
-	  explicit
+	  _GLIBCXX20_CONSTEXPR explicit
 	  _Temporary_value(vector* __vec, _Args&&... __args) : _M_this(__vec)
 	  {
 	    _Alloc_traits::construct(_M_this->_M_impl, _M_ptr(),
 				     std::forward<_Args>(__args)...);
 	  }
 
+	_GLIBCXX20_CONSTEXPR
 	~_Temporary_value()
 	{ _Alloc_traits::destroy(_M_this->_M_impl, _M_ptr()); }
 
-	value_type&
-	_M_val() { return *_M_ptr(); }
+	_GLIBCXX20_CONSTEXPR value_type&
+	_M_val() noexcept { return _M_storage._M_val; }
 
       private:
-	_Tp*
-	_M_ptr() { return reinterpret_cast<_Tp*>(&__buf); }
+	_GLIBCXX20_CONSTEXPR _Tp*
+	_M_ptr() noexcept { return std::__addressof(_M_storage._M_val); }
 
-	vector* _M_this;
-	typename aligned_storage<sizeof(_Tp), alignof(_Tp)>::type __buf;
+	union _Storage
+	{
+	  constexpr _Storage() : _M_byte() { }
+	  _GLIBCXX20_CONSTEXPR ~_Storage() { }
+	  _Storage& operator=(const _Storage&) = delete;
+	  unsigned char _M_byte;
+	  _Tp _M_val;
+	};
+
+	vector*  _M_this;
+	_Storage _M_storage;
       };
 
       // Called by insert(p,x) and other functions when insertion needs to
       // reallocate or move existing elements. _Arg is either _Tp& or _Tp.
       template<typename _Arg>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_insert_aux(iterator __position, _Arg&& __arg);
 
       template<typename... _Args>
+	_GLIBCXX20_CONSTEXPR
 	void
 	_M_realloc_insert(iterator __position, _Args&&... __args);
 
       // Either move-construct at the end, or forward to _M_insert_aux.
+      _GLIBCXX20_CONSTEXPR
       iterator
       _M_insert_rval(const_iterator __position, value_type&& __v);
 
       // Try to emplace at the end, otherwise forward to _M_insert_aux.
       template<typename... _Args>
+	_GLIBCXX20_CONSTEXPR
 	iterator
 	_M_emplace_aux(const_iterator __position, _Args&&... __args);
 
       // Emplacing an rvalue of the correct type can use _M_insert_rval.
+      _GLIBCXX20_CONSTEXPR
       iterator
       _M_emplace_aux(const_iterator __position, value_type&& __v)
       { return _M_insert_rval(__position, std::move(__v)); }
 #endif
 
       // Called by _M_fill_insert, _M_insert_aux etc.
+      _GLIBCXX20_CONSTEXPR
       size_type
       _M_check_len(size_type __n, const char* __s) const
       {
@@ -1786,7 +1895,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
       // Called by constructors to check initial size.
-      static size_type
+      static _GLIBCXX20_CONSTEXPR size_type
       _S_check_init_len(size_type __n, const allocator_type& __a)
       {
 	if (__n > _S_max_size(_Tp_alloc_type(__a)))
@@ -1795,7 +1904,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	return __n;
       }
 
-      static size_type
+      static _GLIBCXX20_CONSTEXPR size_type
       _S_max_size(const _Tp_alloc_type& __a) _GLIBCXX_NOEXCEPT
       {
 	// std::distance(begin(), end()) cannot be greater than PTRDIFF_MAX,
@@ -1811,6 +1920,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
       // Called by erase(q1,q2), clear(), resize(), _M_fill_assign,
       // _M_assign_aux.
+      _GLIBCXX20_CONSTEXPR
       void
       _M_erase_at_end(pointer __pos) _GLIBCXX_NOEXCEPT
       {
@@ -1823,9 +1933,11 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 	  }
       }
 
+      _GLIBCXX20_CONSTEXPR
       iterator
       _M_erase(iterator __position);
 
+      _GLIBCXX20_CONSTEXPR
       iterator
       _M_erase(iterator __first, iterator __last);
 
@@ -1834,6 +1946,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       // Constant-time move assignment when source object's memory can be
       // moved, either because the source's allocator will move too
       // or because the allocators are equal.
+      _GLIBCXX20_CONSTEXPR
       void
       _M_move_assign(vector&& __x, true_type) noexcept
       {
@@ -1845,6 +1958,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
       // Do move assignment when it might not be possible to move source
       // object's memory, resulting in a linear-time operation.
+      _GLIBCXX20_CONSTEXPR
       void
       _M_move_assign(vector&& __x, false_type)
       {
@@ -1863,12 +1977,14 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #endif
 
       template<typename _Up>
+	_GLIBCXX20_CONSTEXPR
 	_Up*
 	_M_data_ptr(_Up* __ptr) const _GLIBCXX_NOEXCEPT
 	{ return __ptr; }
 
 #if __cplusplus >= 201103L
       template<typename _Ptr>
+	_GLIBCXX20_CONSTEXPR
 	typename std::pointer_traits<_Ptr>::element_type*
 	_M_data_ptr(_Ptr __ptr) const
 	{ return empty() ? nullptr : std::__to_address(__ptr); }
@@ -1911,6 +2027,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
    *  and if corresponding elements compare equal.
   */
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     inline bool
     operator==(const vector<_Tp, _Alloc>& __x, const vector<_Tp, _Alloc>& __y)
     { return (__x.size() == __y.size()
@@ -1929,6 +2046,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
    *  `<` and `>=` etc.
   */
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     inline __detail::__synth3way_t<_Tp>
     operator<=>(const vector<_Tp, _Alloc>& __x, const vector<_Tp, _Alloc>& __y)
     {
@@ -1981,6 +2099,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   /// See std::vector::swap().
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     inline void
     swap(vector<_Tp, _Alloc>& __x, vector<_Tp, _Alloc>& __y)
     _GLIBCXX_NOEXCEPT_IF(noexcept(__x.swap(__y)))
diff --git a/libstdc++-v3/include/bits/uses_allocator.h b/libstdc++-v3/include/bits/uses_allocator.h
index 358c571..bd44670 100644
--- a/libstdc++-v3/include/bits/uses_allocator.h
+++ b/libstdc++-v3/include/bits/uses_allocator.h
@@ -35,6 +35,7 @@
 namespace std _GLIBCXX_VISIBILITY(default)
 {
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
+/// @cond undocumented
 
   // This is used for std::experimental::erased_type from Library Fundamentals.
   struct __erased_type { };
@@ -188,6 +189,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  std::forward<_Args>(__args)...);
     }
 
+/// @endcond
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace std
 
diff --git a/libstdc++-v3/include/bits/utility.h b/libstdc++-v3/include/bits/utility.h
index fce52a4..ec5ed04 100644
--- a/libstdc++-v3/include/bits/utility.h
+++ b/libstdc++-v3/include/bits/utility.h
@@ -70,6 +70,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     struct tuple_size<const volatile __enable_if_has_tuple_size<_Tp>>
     : public tuple_size<_Tp> { };
 
+#if __cplusplus >= 201703L
+  template<typename _Tp>
+    inline constexpr size_t tuple_size_v = tuple_size<_Tp>::value;
+#endif
+
   /// Gives the type of the ith element of a given tuple type.
   template<size_t __i, typename _Tp>
     struct tuple_element;
@@ -97,8 +102,33 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     };
 
 #if __cplusplus >= 201402L
-// The standard says this macro and alias template should be in <tuple>
-// but we define them here, to be available in <utility> and <array> too.
+
+  // Return the index of _Tp in _Types, if it occurs exactly once.
+  // Otherwise, return sizeof...(_Types).
+  template<typename _Tp, typename... _Types>
+    constexpr size_t
+    __find_uniq_type_in_pack()
+    {
+      constexpr size_t __sz = sizeof...(_Types);
+      constexpr bool __found[__sz] = { __is_same(_Tp, _Types) ... };
+      size_t __n = __sz;
+      for (size_t __i = 0; __i < __sz; ++__i)
+	{
+	  if (__found[__i])
+	    {
+	      if (__n < __sz) // more than one _Tp found
+		return __sz;
+	      __n = __i;
+	    }
+	}
+      return __n;
+    }
+
+// The standard says this macro and alias template should be in <tuple> but we
+// we define them here, to be available in <array>, <utility> and <ranges> too.
+// _GLIBCXX_RESOLVE_LIB_DEFECTS
+// 3378. tuple_size_v/tuple_element_t should be available when
+//       tuple_size/tuple_element are
 #define __cpp_lib_tuple_element_t 201402L
 
   template<size_t __i, typename _Tp>
@@ -195,6 +225,45 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #endif // C++17
 #endif // C++14
 
+  template<size_t _Np, typename... _Types>
+    struct _Nth_type
+    { };
+
+  template<typename _Tp0, typename... _Rest>
+    struct _Nth_type<0, _Tp0, _Rest...>
+    { using type = _Tp0; };
+
+  template<typename _Tp0, typename _Tp1, typename... _Rest>
+    struct _Nth_type<1, _Tp0, _Tp1, _Rest...>
+    { using type = _Tp1; };
+
+  template<typename _Tp0, typename _Tp1, typename _Tp2, typename... _Rest>
+    struct _Nth_type<2, _Tp0, _Tp1, _Tp2, _Rest...>
+    { using type = _Tp2; };
+
+  template<size_t _Np, typename _Tp0, typename _Tp1, typename _Tp2,
+	   typename... _Rest>
+#if __cpp_concepts
+    requires (_Np >= 3)
+#endif
+    struct _Nth_type<_Np, _Tp0, _Tp1, _Tp2, _Rest...>
+    : _Nth_type<_Np - 3, _Rest...>
+    { };
+
+#if ! __cpp_concepts // Need additional specializations to avoid ambiguities.
+  template<typename _Tp0, typename _Tp1, typename... _Rest>
+    struct _Nth_type<0, _Tp0, _Tp1, _Rest...>
+    { using type = _Tp0; };
+
+  template<typename _Tp0, typename _Tp1, typename _Tp2, typename... _Rest>
+    struct _Nth_type<0, _Tp0, _Tp1, _Tp2, _Rest...>
+    { using type = _Tp0; };
+
+  template<typename _Tp0, typename _Tp1, typename _Tp2, typename... _Rest>
+    struct _Nth_type<1, _Tp0, _Tp1, _Tp2, _Rest...>
+    { using type = _Tp1; };
+#endif
+
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace
 
diff --git a/libstdc++-v3/include/bits/vector.tcc b/libstdc++-v3/include/bits/vector.tcc
index 16366e0..7e3115c 100644
--- a/libstdc++-v3/include/bits/vector.tcc
+++ b/libstdc++-v3/include/bits/vector.tcc
@@ -62,6 +62,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     void
     vector<_Tp, _Alloc>::
     reserve(size_type __n)
@@ -102,6 +103,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
   template<typename _Tp, typename _Alloc>
     template<typename... _Args>
 #if __cplusplus > 201402L
+      _GLIBCXX20_CONSTEXPR
       typename vector<_Tp, _Alloc>::reference
 #else
       void
@@ -126,6 +128,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #endif
 
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     typename vector<_Tp, _Alloc>::iterator
     vector<_Tp, _Alloc>::
 #if __cplusplus >= 201103L
@@ -167,6 +170,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     typename vector<_Tp, _Alloc>::iterator
     vector<_Tp, _Alloc>::
     _M_erase(iterator __position)
@@ -180,6 +184,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     typename vector<_Tp, _Alloc>::iterator
     vector<_Tp, _Alloc>::
     _M_erase(iterator __first, iterator __last)
@@ -194,6 +199,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     vector<_Tp, _Alloc>&
     vector<_Tp, _Alloc>::
     operator=(const vector<_Tp, _Alloc>& __x)
@@ -253,6 +259,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     void
     vector<_Tp, _Alloc>::
     _M_fill_assign(size_t __n, const value_type& __val)
@@ -278,6 +285,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   template<typename _Tp, typename _Alloc>
     template<typename _InputIterator>
+      _GLIBCXX20_CONSTEXPR
       void
       vector<_Tp, _Alloc>::
       _M_assign_aux(_InputIterator __first, _InputIterator __last,
@@ -296,6 +304,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   template<typename _Tp, typename _Alloc>
     template<typename _ForwardIterator>
+      _GLIBCXX20_CONSTEXPR
       void
       vector<_Tp, _Alloc>::
       _M_assign_aux(_ForwardIterator __first, _ForwardIterator __last,
@@ -336,6 +345,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
 #if __cplusplus >= 201103L
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     auto
     vector<_Tp, _Alloc>::
     _M_insert_rval(const_iterator __position, value_type&& __v) -> iterator
@@ -360,6 +370,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   template<typename _Tp, typename _Alloc>
     template<typename... _Args>
+      _GLIBCXX20_CONSTEXPR
       auto
       vector<_Tp, _Alloc>::
       _M_emplace_aux(const_iterator __position, _Args&&... __args)
@@ -391,6 +402,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   template<typename _Tp, typename _Alloc>
     template<typename _Arg>
+      _GLIBCXX20_CONSTEXPR
       void
       vector<_Tp, _Alloc>::
       _M_insert_aux(iterator __position, _Arg&& __arg)
@@ -422,6 +434,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 #if __cplusplus >= 201103L
   template<typename _Tp, typename _Alloc>
     template<typename... _Args>
+      _GLIBCXX20_CONSTEXPR
       void
       vector<_Tp, _Alloc>::
       _M_realloc_insert(iterator __position, _Args&&... __args)
@@ -505,6 +518,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     void
     vector<_Tp, _Alloc>::
     _M_fill_insert(iterator __position, size_type __n, const value_type& __x)
@@ -607,6 +621,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
 #if __cplusplus >= 201103L
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     void
     vector<_Tp, _Alloc>::
     _M_default_append(size_type __n)
@@ -683,6 +698,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
   template<typename _Tp, typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     bool
     vector<_Tp, _Alloc>::
     _M_shrink_to_fit()
@@ -696,6 +712,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   template<typename _Tp, typename _Alloc>
     template<typename _InputIterator>
+      _GLIBCXX20_CONSTEXPR
       void
       vector<_Tp, _Alloc>::
       _M_range_insert(iterator __pos, _InputIterator __first,
@@ -717,6 +734,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   template<typename _Tp, typename _Alloc>
     template<typename _ForwardIterator>
+      _GLIBCXX20_CONSTEXPR
       void
       vector<_Tp, _Alloc>::
       _M_range_insert(iterator __position, _ForwardIterator __first,
@@ -806,6 +824,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   // vector<bool>
   template<typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     void
     vector<bool, _Alloc>::
     _M_reallocate(size_type __n)
@@ -820,6 +839,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
   template<typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     void
     vector<bool, _Alloc>::
     _M_fill_insert(iterator __position, size_type __n, bool __x)
@@ -852,6 +872,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   template<typename _Alloc>
     template<typename _ForwardIterator>
+      _GLIBCXX20_CONSTEXPR
       void
       vector<bool, _Alloc>::
       _M_insert_range(iterator __position, _ForwardIterator __first, 
@@ -886,6 +907,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
       }
 
   template<typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     void
     vector<bool, _Alloc>::
     _M_insert_aux(iterator __position, bool __x)
@@ -914,6 +936,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
   template<typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     typename vector<bool, _Alloc>::iterator
     vector<bool, _Alloc>::
     _M_erase(iterator __position)
@@ -925,6 +948,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
     }
 
   template<typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     typename vector<bool, _Alloc>::iterator
     vector<bool, _Alloc>::
     _M_erase(iterator __first, iterator __last)
@@ -936,6 +960,7 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
 #if __cplusplus >= 201103L
   template<typename _Alloc>
+    _GLIBCXX20_CONSTEXPR
     bool
     vector<bool, _Alloc>::
     _M_shrink_to_fit()
@@ -974,9 +999,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     operator()(const _GLIBCXX_STD_C::vector<bool, _Alloc>& __b) const noexcept
     {
       size_t __hash = 0;
-      using _GLIBCXX_STD_C::_S_word_bit;
-      using _GLIBCXX_STD_C::_Bit_type;
-
       const size_t __words = __b.size() / _S_word_bit;
       if (__words)
 	{
diff --git a/libstdc++-v3/include/debug/deque b/libstdc++-v3/include/debug/deque
index 8e48111..a73d6c3 100644
--- a/libstdc++-v3/include/debug/deque
+++ b/libstdc++-v3/include/debug/deque
@@ -156,15 +156,7 @@ namespace __debug
       deque(_Base_ref __x)
       : _Base(__x._M_ref) { }
 
-#if __cplusplus < 201103L
-      deque&
-      operator=(const deque& __x)
-      {
-	this->_M_safe() = __x;
-	_M_base() = __x;
-	return *this;
-      }
-#else
+#if __cplusplus >= 201103L
       deque&
       operator=(const deque&) = default;
 
@@ -174,7 +166,7 @@ namespace __debug
       deque&
       operator=(initializer_list<value_type> __l)
       {
-	_M_base() = __l;
+	_Base::operator=(__l);
 	this->_M_invalidate_all();
 	return *this;
       }
@@ -352,7 +344,7 @@ namespace __debug
       operator[](size_type __n) _GLIBCXX_NOEXCEPT
       {
 	__glibcxx_check_subscript(__n);
-	return _M_base()[__n];
+	return _Base::operator[](__n);
       }
 
       _GLIBCXX_NODISCARD
@@ -360,7 +352,7 @@ namespace __debug
       operator[](size_type __n) const _GLIBCXX_NOEXCEPT
       {
 	__glibcxx_check_subscript(__n);
-	return _M_base()[__n];
+	return _Base::operator[](__n);
       }
 
       using _Base::at;
diff --git a/libstdc++-v3/include/debug/forward_list b/libstdc++-v3/include/debug/forward_list
index cae5b5f..6ed4853 100644
--- a/libstdc++-v3/include/debug/forward_list
+++ b/libstdc++-v3/include/debug/forward_list
@@ -242,8 +242,8 @@ namespace __debug
 	noexcept(
 	  std::is_nothrow_constructible<_Base,
 	    _Base, const allocator_type&>::value )
-      : _Safe(std::move(__list._M_safe()), __al),
-	_Base(std::move(__list._M_base()), __al)
+      : _Safe(std::move(__list), __al),
+	_Base(std::move(__list), __al)
       { }
 
       explicit
@@ -287,7 +287,7 @@ namespace __debug
       forward_list&
       operator=(std::initializer_list<_Tp> __il)
       {
-	_M_base() = __il;
+	_Base::operator=(__il);
 	this->_M_invalidate_all();
 	return *this;
       }
@@ -584,7 +584,7 @@ namespace __debug
 	    return __it != __list._M_base().cbefore_begin()
 		   && __it != __list._M_base().end();
 	  });
-	_Base::splice_after(__pos.base(), std::move(__list._M_base()));
+	_Base::splice_after(__pos.base(), std::move(__list));
       }
 
       void
@@ -613,8 +613,7 @@ namespace __debug
 	_Base_const_iterator __next = std::next(__i.base());
 	this->_M_transfer_from_if(__list, [__next](_Base_const_iterator __it)
 	  { return __it == __next; });
-	_Base::splice_after(__pos.base(), std::move(__list._M_base()),
-			    __i.base());
+	_Base::splice_after(__pos.base(), std::move(__list), __i.base());
       }
 
       void
@@ -669,7 +668,7 @@ namespace __debug
 	      { return __it == __tmp; });
 	  }
 
-	_Base::splice_after(__pos.base(), std::move(__list._M_base()),
+	_Base::splice_after(__pos.base(), std::move(__list),
 			    __before.base(), __last.base());
       }
 
@@ -710,7 +709,7 @@ namespace __debug
 		this->_M_invalidate_if([__next](_Base_const_iterator __it)
 				       { return __it == __next; });
 		__to_destroy.splice_after(__to_destroy.cbefore_begin(),
-					  _M_base(), __old);
+					  *this, __old);
 		__x = __old;
 		_GLIBCXX20_ONLY( __removed++ );
 	      }
@@ -739,7 +738,7 @@ namespace __debug
 		  this->_M_invalidate_if([__x](_Base_const_iterator __it)
 					 { return __it == __x; });
 		  __to_destroy.splice_after(__to_destroy.cbefore_begin(),
-					    _M_base(), __old);
+					    *this, __old);
 		  __x = __old;
 		  _GLIBCXX20_ONLY( __removed++ );
 		}
@@ -777,7 +776,7 @@ namespace __debug
 		  this->_M_invalidate_if([__next](_Base_const_iterator __it)
 					 { return __it == __next; });
 		  __to_destroy.splice_after(__to_destroy.cbefore_begin(),
-					    _M_base(), __first);
+					    *this, __first);
 		  __next = __first;
 		  _GLIBCXX20_ONLY( __removed++ );
 		}
@@ -804,7 +803,7 @@ namespace __debug
 	      return __it != __list._M_base().cbefore_begin()
 		     && __it != __list._M_base().cend();
 	    });
-	  _Base::merge(std::move(__list._M_base()));
+	  _Base::merge(std::move(__list));
 	}
       }
 
@@ -827,7 +826,7 @@ namespace __debug
 		return __it != __list._M_base().cbefore_begin()
 		       && __it != __list._M_base().cend();
 	      });
-	    _Base::merge(std::move(__list._M_base()), __comp);
+	    _Base::merge(std::move(__list), __comp);
 	  }
 	}
 
diff --git a/libstdc++-v3/include/debug/list b/libstdc++-v3/include/debug/list
index de30edb..383482c 100644
--- a/libstdc++-v3/include/debug/list
+++ b/libstdc++-v3/include/debug/list
@@ -122,8 +122,8 @@ namespace __debug
 	noexcept(
 	  std::is_nothrow_constructible<_Base,
 	    _Base, const allocator_type&>::value )
-      : _Safe(std::move(__x._M_safe()), __a),
-	_Base(std::move(__x._M_base()), __a) { }
+      : _Safe(std::move(__x), __a),
+	_Base(std::move(__x), __a) { }
 #endif
 
       explicit
@@ -161,15 +161,7 @@ namespace __debug
       list(_Base_ref __x)
       : _Base(__x._M_ref) { }
 
-#if __cplusplus < 201103L
-      list&
-      operator=(const list& __x)
-      {
-	this->_M_safe() = __x;
-	_M_base() = __x;
-	return *this;
-      }
-#else
+#if __cplusplus >= 201103L
       list&
       operator=(const list&) = default;
 
@@ -180,7 +172,7 @@ namespace __debug
       operator=(initializer_list<value_type> __l)
       {
 	this->_M_invalidate_all();
-	_M_base() = __l;
+	_Base::operator=(__l);
 	return *this;
       }
 
@@ -591,7 +583,7 @@ namespace __debug
 			      _M_message(__gnu_debug::__msg_self_splice)
 			      ._M_sequence(*this, "this"));
 	this->_M_transfer_from_if(__x, _Not_equal(__x._M_base().end()));
-	_Base::splice(__position.base(), _GLIBCXX_MOVE(__x._M_base()));
+	_Base::splice(__position.base(), _GLIBCXX_MOVE(__x));
       }
 
 #if __cplusplus >= 201103L
@@ -622,7 +614,7 @@ namespace __debug
 	// _GLIBCXX_RESOLVE_LIB_DEFECTS
 	// 250. splicing invalidates iterators
 	this->_M_transfer_from_if(__x, _Equal(__i.base()));
-	_Base::splice(__position.base(), _GLIBCXX_MOVE(__x._M_base()),
+	_Base::splice(__position.base(), _GLIBCXX_MOVE(__x),
 		      __i.base());
       }
 
@@ -670,7 +662,7 @@ namespace __debug
 	    this->_M_transfer_from_if(__x, _Equal(__tmp));
 	  }
 
-	_Base::splice(__position.base(), _GLIBCXX_MOVE(__x._M_base()),
+	_Base::splice(__position.base(), _GLIBCXX_MOVE(__x),
 		      __first.base(), __last.base());
       }
 
@@ -717,7 +709,7 @@ namespace __debug
 		// 526. Is it undefined if a function in the standard changes
 		// in parameters?
 		this->_M_invalidate_if(_Equal(__first));
-		__to_destroy.splice(__to_destroy.begin(), _M_base(), __first);
+		__to_destroy.splice(__to_destroy.begin(), *this, __first);
 #if !_GLIBCXX_USE_CXX11_ABI
 		_GLIBCXX20_ONLY( __removed++ );
 #endif
@@ -751,7 +743,7 @@ namespace __debug
 	    if (__pred(*__x))
 	      {
 		this->_M_invalidate_if(_Equal(__x));
-		__to_destroy.splice(__to_destroy.begin(), _M_base(), __x);
+		__to_destroy.splice(__to_destroy.begin(), *this, __x);
 #if !_GLIBCXX_USE_CXX11_ABI
 		_GLIBCXX20_ONLY( __removed++ );
 #endif
@@ -788,7 +780,7 @@ namespace __debug
 	  if (*__first == *__next)
 	    {
 	      this->_M_invalidate_if(_Equal(__next));
-	      __to_destroy.splice(__to_destroy.begin(), _M_base(), __next);
+	      __to_destroy.splice(__to_destroy.begin(), *this, __next);
 	      __next = __first;
 #if !_GLIBCXX_USE_CXX11_ABI
 	      _GLIBCXX20_ONLY( __removed++ );
@@ -826,7 +818,7 @@ namespace __debug
 	    if (__binary_pred(*__first, *__next))
 	      {
 		this->_M_invalidate_if(_Equal(__next));
-		__to_destroy.splice(__to_destroy.begin(), _M_base(), __next);
+		__to_destroy.splice(__to_destroy.begin(), *this, __next);
 		__next = __first;
 #if !_GLIBCXX_USE_CXX11_ABI
 		_GLIBCXX20_ONLY( __removed++ );
@@ -859,7 +851,7 @@ namespace __debug
 	    __glibcxx_check_sorted(_Base::begin(), _Base::end());
 	    __glibcxx_check_sorted(__x.begin().base(), __x.end().base());
 	    this->_M_transfer_from_if(__x, _Not_equal(__x._M_base().end()));
-	    _Base::merge(_GLIBCXX_MOVE(__x._M_base()));
+	    _Base::merge(_GLIBCXX_MOVE(__x));
 	  }
       }
 
@@ -886,7 +878,7 @@ namespace __debug
 	      __glibcxx_check_sorted_pred(__x.begin().base(), __x.end().base(),
 					  __comp);
 	      this->_M_transfer_from_if(__x, _Not_equal(__x._M_base().end()));
-	      _Base::merge(_GLIBCXX_MOVE(__x._M_base()), __comp);
+	      _Base::merge(_GLIBCXX_MOVE(__x), __comp);
 	    }
 	}
 
diff --git a/libstdc++-v3/include/debug/map.h b/libstdc++-v3/include/debug/map.h
index 9e142cf..c62f0b5 100644
--- a/libstdc++-v3/include/debug/map.h
+++ b/libstdc++-v3/include/debug/map.h
@@ -118,9 +118,9 @@ namespace __debug
       : _Base(__m, __a) { }
 
       map(map&& __m, const __type_identity_t<allocator_type>& __a)
-      noexcept( noexcept(_Base(std::move(__m._M_base()), __a)) )
-      : _Safe(std::move(__m._M_safe()), __a),
-	_Base(std::move(__m._M_base()), __a) { }
+      noexcept( noexcept(_Base(std::move(__m), __a)) )
+      : _Safe(std::move(__m), __a),
+	_Base(std::move(__m), __a) { }
 
       map(initializer_list<value_type> __l, const allocator_type& __a)
       : _Base(__l, __a) { }
@@ -152,15 +152,7 @@ namespace __debug
 		__gnu_debug::__base(__last),
 		__comp, __a) { }
 
-#if __cplusplus < 201103L
-      map&
-      operator=(const map& __x)
-      {
-	this->_M_safe() = __x;
-	_M_base() = __x;
-	return *this;
-      }
-#else
+#if __cplusplus >= 201103L
       map&
       operator=(const map&) = default;
 
@@ -170,7 +162,7 @@ namespace __debug
       map&
       operator=(initializer_list<value_type> __l)
       {
-	_M_base() = __l;
+	_Base::operator=(__l);
 	this->_M_invalidate_all();
 	return *this;
       }
diff --git a/libstdc++-v3/include/debug/multimap.h b/libstdc++-v3/include/debug/multimap.h
index a05b8a8..5f0f1fa 100644
--- a/libstdc++-v3/include/debug/multimap.h
+++ b/libstdc++-v3/include/debug/multimap.h
@@ -119,9 +119,9 @@ namespace __debug
       : _Base(__m, __a) { }
 
       multimap(multimap&& __m, const __type_identity_t<allocator_type>& __a)
-      noexcept( noexcept(_Base(std::move(__m._M_base()), __a)) )
-      : _Safe(std::move(__m._M_safe()), __a),
-	_Base(std::move(__m._M_base()), __a) { }
+      noexcept( noexcept(_Base(std::move(__m), __a)) )
+      : _Safe(std::move(__m), __a),
+	_Base(std::move(__m), __a) { }
 
       multimap(initializer_list<value_type> __l, const allocator_type& __a)
       : _Base(__l, __a) { }
@@ -152,15 +152,7 @@ namespace __debug
       multimap(_Base_ref __x)
       : _Base(__x._M_ref) { }
 
-#if __cplusplus < 201103L
-      multimap&
-      operator=(const multimap& __x)
-      {
-	this->_M_safe() = __x;
-	_M_base() = __x;
-	return *this;
-      }
-#else
+#if __cplusplus >= 201103L
       multimap&
       operator=(const multimap&) = default;
 
@@ -170,7 +162,7 @@ namespace __debug
       multimap&
       operator=(initializer_list<value_type> __l)
       {
-	_M_base() = __l;
+	_Base::operator=(__l);
 	this->_M_invalidate_all();
 	return *this;
       }
diff --git a/libstdc++-v3/include/debug/multiset.h b/libstdc++-v3/include/debug/multiset.h
index a312ccf..7729fc1 100644
--- a/libstdc++-v3/include/debug/multiset.h
+++ b/libstdc++-v3/include/debug/multiset.h
@@ -118,9 +118,9 @@ namespace __debug
       : _Base(__m, __a) { }
 
       multiset(multiset&& __m, const __type_identity_t<allocator_type>& __a)
-      noexcept( noexcept(_Base(std::move(__m._M_base()), __a)) )
-      : _Safe(std::move(__m._M_safe()), __a),
-	_Base(std::move(__m._M_base()), __a) { }
+      noexcept( noexcept(_Base(std::move(__m), __a)) )
+      : _Safe(std::move(__m), __a),
+	_Base(std::move(__m), __a) { }
 
       multiset(initializer_list<value_type> __l, const allocator_type& __a)
 	: _Base(__l, __a)
@@ -152,15 +152,7 @@ namespace __debug
       multiset(_Base_ref __x)
       : _Base(__x._M_ref) { }
 
-#if __cplusplus < 201103L
-      multiset&
-      operator=(const multiset& __x)
-      {
-	this->_M_safe() = __x;
-	_M_base() = __x;
-	return *this;
-      }
-#else
+#if __cplusplus >= 201103L
       multiset&
       operator=(const multiset&) = default;
 
@@ -170,7 +162,7 @@ namespace __debug
       multiset&
       operator=(initializer_list<value_type> __l)
       {
-	_M_base() = __l;
+	_Base::operator=(__l);
 	this->_M_invalidate_all();
 	return *this;
       }
diff --git a/libstdc++-v3/include/debug/safe_container.h b/libstdc++-v3/include/debug/safe_container.h
index 97c4716..caba37a 100644
--- a/libstdc++-v3/include/debug/safe_container.h
+++ b/libstdc++-v3/include/debug/safe_container.h
@@ -48,10 +48,6 @@ namespace __gnu_debug
       { return *static_cast<_SafeContainer*>(this); }
 
     protected:
-      _Safe_container&
-      _M_safe() _GLIBCXX_NOEXCEPT
-      { return *this; }
-
 #if __cplusplus >= 201103L
       _Safe_container() = default;
       _Safe_container(const _Safe_container&) = default;
@@ -78,7 +74,6 @@ namespace __gnu_debug
       { }
 #endif
 
-    public:
       // Copy assignment invalidate all iterators.
       _Safe_container&
       operator=(const _Safe_container&) _GLIBCXX_NOEXCEPT
diff --git a/libstdc++-v3/include/debug/safe_unordered_container.h b/libstdc++-v3/include/debug/safe_unordered_container.h
index aae1e2d..ce9d9ea 100644
--- a/libstdc++-v3/include/debug/safe_unordered_container.h
+++ b/libstdc++-v3/include/debug/safe_unordered_container.h
@@ -72,6 +72,96 @@ namespace __gnu_debug
 		{ return __it != __local_end; });
       }
 
+#if __cplusplus > 201402L
+      template<typename _ExtractKey, typename _Source>
+	struct _UContInvalidatePred
+	{
+	  template<typename _Iterator>
+	    bool
+	    operator()(_Iterator __it) const
+	    { return _M_source.count(_ExtractKey{}(*__it)) == 0; }
+
+	  const _Source& _M_source;
+	};
+
+      template<typename _ExtractKey, typename _Source>
+	struct _UMContInvalidatePred
+	{
+	  template<typename _Iterator>
+	    bool
+	    operator()(_Iterator __it) const
+	    {
+	      auto __rng =
+		_M_source._M_base().equal_range(_ExtractKey{}(*__it));
+	      for (auto __rit = __rng.first;
+		   __rit != __rng.second; ++__rit)
+		{
+		  if (__it == __rit)
+		    return false;
+		}
+
+	      return true;
+	    }
+
+	  const _Source& _M_source;
+	};
+
+      template<typename _Source, typename _InvalidatePred>
+	struct _UContMergeGuard
+	{
+	  _UContMergeGuard(_Source& __src) noexcept
+	  : _M_source(__src), _M_size(__src.size()), _M_pred { __src }
+	  { }
+
+	  _UContMergeGuard(const _UContMergeGuard&) = delete;
+
+	  ~_UContMergeGuard()
+	  {
+	    const std::size_t __size = _M_source.size();
+	    if (__size == _M_size)
+	      return;
+
+	    __try
+	      {
+		if (__size == 0)
+		  _M_source._M_invalidate_all();
+		else
+		  {
+		    _M_source._M_invalidate_if(_M_pred);
+		    _M_source._M_invalidate_local_if(_M_pred);
+		  }
+	      }
+	    __catch(...)
+	      {
+		_M_source._M_invalidate_all();
+	      }
+	  }
+
+	  _Source& _M_source;
+	  const std::size_t _M_size;
+	  _InvalidatePred _M_pred;
+	};
+
+      template<typename _ExtractKey, typename _Source>
+	static _UContMergeGuard<_Source,
+				_UContInvalidatePred<_ExtractKey, _Source>>
+	_S_uc_guard(_ExtractKey, _Source& __src)
+	{
+	  typedef _UContInvalidatePred<_ExtractKey, _Source> _InvalidatePred;
+	  return _UContMergeGuard<_Source, _InvalidatePred>(__src);
+	}
+
+      template<typename _ExtractKey, typename _Source>
+	static _UContMergeGuard<_Source,
+				_UMContInvalidatePred<_ExtractKey, _Source>>
+	_S_umc_guard(_ExtractKey, _Source& __src)
+	{
+	  typedef _UMContInvalidatePred<_ExtractKey, _Source> _InvalidatePred;
+	  return _UContMergeGuard<_Source, _InvalidatePred>(__src);
+	}
+#endif // C++17
+
+    public:
       void
       _M_invalidate_all()
       {
diff --git a/libstdc++-v3/include/debug/set.h b/libstdc++-v3/include/debug/set.h
index 01da942..39142ae 100644
--- a/libstdc++-v3/include/debug/set.h
+++ b/libstdc++-v3/include/debug/set.h
@@ -117,9 +117,9 @@ namespace __debug
       : _Base(__x, __a) { }
 
       set(set&& __x, const __type_identity_t<allocator_type>& __a)
-      noexcept( noexcept(_Base(std::move(__x._M_base()), __a)) )
-      : _Safe(std::move(__x._M_safe()), __a),
-	_Base(std::move(__x._M_base()), __a) { }
+      noexcept( noexcept(_Base(std::move(__x), __a)) )
+      : _Safe(std::move(__x), __a),
+	_Base(std::move(__x), __a) { }
 
       set(initializer_list<value_type> __l, const allocator_type& __a)
       : _Base(__l, __a) { }
@@ -150,15 +150,7 @@ namespace __debug
       set(_Base_ref __x)
       : _Base(__x._M_ref) { }
 
-#if __cplusplus < 201103L
-      set&
-      operator=(const set& __x)
-      {
-	this->_M_safe() = __x;
-	_M_base() = __x;
-	return *this;
-      }
-#else
+#if __cplusplus >= 201103L
       set&
       operator=(const set&) = default;
 
@@ -168,7 +160,7 @@ namespace __debug
       set&
       operator=(initializer_list<value_type> __l)
       {
-	_M_base() = __l;
+	_Base::operator=(__l);
 	this->_M_invalidate_all();
 	return *this;
       }
diff --git a/libstdc++-v3/include/debug/stl_iterator.h b/libstdc++-v3/include/debug/stl_iterator.h
index edeb42e..54f7d42 100644
--- a/libstdc++-v3/include/debug/stl_iterator.h
+++ b/libstdc++-v3/include/debug/stl_iterator.h
@@ -35,31 +35,38 @@ namespace __gnu_debug
 {
   // Help Debug mode to see through reverse_iterator.
   template<typename _Iterator>
+    _GLIBCXX20_CONSTEXPR
     inline bool
     __valid_range(const std::reverse_iterator<_Iterator>& __first,
 		  const std::reverse_iterator<_Iterator>& __last,
 		  typename _Distance_traits<_Iterator>::__type& __dist)
-    { return __valid_range(__last.base(), __first.base(), __dist); }
+    {
+      return __gnu_debug::__valid_range(__last.base(), __first.base(), __dist);
+    }
 
   template<typename _Iterator>
+    _GLIBCXX20_CONSTEXPR
     inline typename _Distance_traits<_Iterator>::__type
     __get_distance(const std::reverse_iterator<_Iterator>& __first,
 		   const std::reverse_iterator<_Iterator>& __last)
-    { return __get_distance(__last.base(), __first.base()); }
+    { return __gnu_debug::__get_distance(__last.base(), __first.base()); }
 
   template<typename _Iterator, typename _Size>
+    _GLIBCXX20_CONSTEXPR
     inline bool
     __can_advance(const std::reverse_iterator<_Iterator>& __it, _Size __n)
-    { return __can_advance(__it.base(), -__n); }
+    { return __gnu_debug::__can_advance(__it.base(), -__n); }
 
   template<typename _Iterator, typename _Diff>
+    _GLIBCXX20_CONSTEXPR
     inline bool
     __can_advance(const std::reverse_iterator<_Iterator>& __it,
 		  const std::pair<_Diff, _Distance_precision>& __dist,
 		  int __way)
-    { return __can_advance(__it.base(), __dist, -__way); }
+    { return __gnu_debug::__can_advance(__it.base(), __dist, -__way); }
 
   template<typename _Iterator, typename _Sequence>
+    _GLIBCXX20_CONSTEXPR
     inline std::reverse_iterator<_Iterator>
     __base(const std::reverse_iterator<_Safe_iterator<
 	     _Iterator, _Sequence, std::random_access_iterator_tag> >& __it)
@@ -82,6 +89,7 @@ namespace __gnu_debug
     }
 #else
   template<typename _Iterator>
+    _GLIBCXX20_CONSTEXPR
     inline auto
     __unsafe(const std::reverse_iterator<_Iterator>& __it)
     -> decltype(std::__make_reverse_iterator(__unsafe(__it.base())))
@@ -91,37 +99,45 @@ namespace __gnu_debug
 #if __cplusplus >= 201103L
   // Help Debug mode to see through move_iterator.
   template<typename _Iterator>
+    _GLIBCXX20_CONSTEXPR
     inline bool
     __valid_range(const std::move_iterator<_Iterator>& __first,
 		  const std::move_iterator<_Iterator>& __last,
 		  typename _Distance_traits<_Iterator>::__type& __dist)
-    { return __valid_range(__first.base(), __last.base(), __dist); }
+    {
+      return __gnu_debug::__valid_range(__first.base(), __last.base(), __dist);
+    }
 
   template<typename _Iterator>
+    _GLIBCXX20_CONSTEXPR
     inline typename _Distance_traits<_Iterator>::__type
     __get_distance(const std::move_iterator<_Iterator>& __first,
 		   const std::move_iterator<_Iterator>& __last)
-    { return __get_distance(__first.base(), __last.base()); }
+    { return __gnu_debug::__get_distance(__first.base(), __last.base()); }
 
   template<typename _Iterator, typename _Size>
+    _GLIBCXX20_CONSTEXPR
     inline bool
     __can_advance(const std::move_iterator<_Iterator>& __it, _Size __n)
-    { return __can_advance(__it.base(), __n); }
+    { return __gnu_debug::__can_advance(__it.base(), __n); }
 
   template<typename _Iterator, typename _Diff>
+    _GLIBCXX20_CONSTEXPR
     inline bool
     __can_advance(const std::move_iterator<_Iterator>& __it,
 		  const std::pair<_Diff, _Distance_precision>& __dist,
 		  int __way)
-    { return __can_advance(__it.base(), __dist, __way); }
+    { return __gnu_debug::__can_advance(__it.base(), __dist, __way); }
 
   template<typename _Iterator>
+    _GLIBCXX20_CONSTEXPR
     inline auto
     __unsafe(const std::move_iterator<_Iterator>& __it)
     -> decltype(std::make_move_iterator(__unsafe(__it.base())))
     { return std::make_move_iterator(__unsafe(__it.base())); }
 
   template<typename _Iterator>
+    _GLIBCXX20_CONSTEXPR
     inline auto
     __base(const std::move_iterator<_Iterator>& __it)
     -> decltype(std::make_move_iterator(__base(__it.base())))
diff --git a/libstdc++-v3/include/debug/string b/libstdc++-v3/include/debug/string
index a838952..dd16527 100644
--- a/libstdc++-v3/include/debug/string
+++ b/libstdc++-v3/include/debug/string
@@ -160,8 +160,8 @@ namespace __gnu_debug
       basic_string(basic_string&& __s, const _Allocator& __a)
       noexcept(
 	std::is_nothrow_constructible<_Base, _Base, const _Allocator&>::value )
-      : _Safe(std::move(__s._M_safe()), __a),
-	_Base(std::move(__s._M_base()), __a)
+      : _Safe(std::move(__s), __a),
+	_Base(std::move(__s), __a)
       { }
 
       ~basic_string() = default;
@@ -201,15 +201,7 @@ namespace __gnu_debug
 		  __glibcxx_check_valid_constructor_range(__begin, __end)),
 		__gnu_debug::__base(__end), __a) { }
 
-#if __cplusplus < 201103L
-      basic_string&
-      operator=(const basic_string& __str)
-      {
-	this->_M_safe() = __str;
-	_M_base() = __str;
-	return *this;
-      }
-#else
+#if __cplusplus >= 201103L
       basic_string&
       operator=(const basic_string&) = default;
 
@@ -221,7 +213,7 @@ namespace __gnu_debug
       operator=(const _CharT* __s)
       {
 	__glibcxx_check_string(__s);
-	_M_base() = __s;
+	_Base::operator=(__s);
 	this->_M_invalidate_all();
 	return *this;
       }
@@ -229,7 +221,7 @@ namespace __gnu_debug
       basic_string&
       operator=(_CharT __c)
       {
-	_M_base() = __c;
+	_Base::operator=(__c);
 	this->_M_invalidate_all();
 	return *this;
       }
@@ -238,7 +230,7 @@ namespace __gnu_debug
       basic_string&
       operator=(std::initializer_list<_CharT> __l)
       {
-	_M_base() = __l;
+	_Base::operator=(__l);
 	this->_M_invalidate_all();
 	return *this;
       }
@@ -349,7 +341,7 @@ namespace __gnu_debug
 			      ._M_sequence(*this, "this")
 			      ._M_integer(__pos, "__pos")
 			      ._M_integer(this->size(), "size"));
-	return _M_base()[__pos];
+	return _Base::operator[](__pos);
       }
 
       reference
@@ -365,7 +357,7 @@ namespace __gnu_debug
 			      ._M_integer(__pos, "__pos")
 			      ._M_integer(this->size(), "size"));
 #endif
-	return _M_base()[__pos];
+	return _Base::operator[](__pos);
       }
 
       using _Base::at;
@@ -379,7 +371,7 @@ namespace __gnu_debug
       basic_string&
       operator+=(const basic_string& __str)
       {
-	_M_base() += __str;
+	_Base::operator+=(__str);
 	this->_M_invalidate_all();
 	return *this;
       }
@@ -388,7 +380,7 @@ namespace __gnu_debug
       operator+=(const _CharT* __s)
       {
 	__glibcxx_check_string(__s);
-	_M_base() += __s;
+	_Base::operator+=(__s);
 	this->_M_invalidate_all();
 	return *this;
       }
@@ -396,7 +388,7 @@ namespace __gnu_debug
       basic_string&
       operator+=(_CharT __c)
       {
-	_M_base() += __c;
+	_Base::operator+=(__c);
 	this->_M_invalidate_all();
 	return *this;
       }
@@ -405,7 +397,7 @@ namespace __gnu_debug
       basic_string&
       operator+=(std::initializer_list<_CharT> __l)
       {
-	_M_base() += __l;
+	_Base::operator+=(__l);
 	this->_M_invalidate_all();
 	return *this;
       }
diff --git a/libstdc++-v3/include/debug/unordered_map b/libstdc++-v3/include/debug/unordered_map
index bb697d3..64cc8ba 100644
--- a/libstdc++-v3/include/debug/unordered_map
+++ b/libstdc++-v3/include/debug/unordered_map
@@ -97,7 +97,12 @@ namespace __debug
 
       typedef typename _Base::key_type			key_type;
       typedef typename _Base::value_type		value_type;
+      typedef typename _Base::mapped_type		mapped_type;
 
+      typedef typename _Base::pointer			pointer;
+      typedef typename _Base::const_pointer		const_pointer;
+      typedef typename _Base::reference			reference;
+      typedef typename _Base::const_reference		const_reference;
       typedef __gnu_debug::_Safe_iterator<
 	_Base_iterator, unordered_map>			iterator;
       typedef __gnu_debug::_Safe_iterator<
@@ -106,6 +111,7 @@ namespace __debug
 	_Base_local_iterator, unordered_map>		local_iterator;
       typedef __gnu_debug::_Safe_local_iterator<
 	_Base_const_local_iterator, unordered_map>	const_local_iterator;
+      typedef typename _Base::difference_type		difference_type;
 
       unordered_map() = default;
 
@@ -144,9 +150,9 @@ namespace __debug
 
       unordered_map(unordered_map&& __umap,
 		    const allocator_type& __a)
-      noexcept( noexcept(_Base(std::move(__umap._M_base()), __a)) )
-      : _Safe(std::move(__umap._M_safe()), __a),
-	_Base(std::move(__umap._M_base()), __a) { }
+      noexcept( noexcept(_Base(std::move(__umap), __a)) )
+      : _Safe(std::move(__umap), __a),
+	_Base(std::move(__umap), __a) { }
 
       unordered_map(initializer_list<value_type> __l,
 		    size_type __n = 0,
@@ -204,11 +210,16 @@ namespace __debug
       unordered_map&
       operator=(initializer_list<value_type> __l)
       {
-	_M_base() = __l;
+	_Base::operator=(__l);
 	this->_M_invalidate_all();
 	return *this;
       }
 
+      using _Base::get_allocator;
+      using _Base::empty;
+      using _Base::size;
+      using _Base::max_size;
+
       void
       swap(unordered_map& __x)
 	noexcept( noexcept(declval<_Base&>().swap(__x)) )
@@ -291,6 +302,10 @@ namespace __debug
 	return { _Base::cend(__b), this };
       }
 
+      using _Base::bucket_count;
+      using _Base::max_bucket_count;
+      using _Base::bucket;
+
       size_type
       bucket_size(size_type __b) const
       {
@@ -298,6 +313,8 @@ namespace __debug
 	return _Base::bucket_size(__b);
       }
 
+      using _Base::load_factor;
+
       float
       max_load_factor() const noexcept
       { return _Base::max_load_factor(); }
@@ -538,9 +555,38 @@ namespace __debug
 	return { _Base::insert(__hint.base(), std::move(__nh)), this };
       }
 
-      using _Base::merge;
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_map<_Key, _Tp, _H2, _P2, _Alloc>& __source)
+	{
+	  auto __guard
+	    = _Safe::_S_uc_guard(std::__detail::_Select1st{}, __source);
+	  _Base::merge(__source);
+	}
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_map<_Key, _Tp, _H2, _P2, _Alloc>&& __source)
+	{ merge(__source); }
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_multimap<_Key, _Tp, _H2, _P2, _Alloc>& __source)
+	{
+	  auto __guard
+	    = _Safe::_S_umc_guard(std::__detail::_Select1st{}, __source);
+	  _Base::merge(__source);
+	}
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_multimap<_Key, _Tp, _H2, _P2, _Alloc>&& __source)
+	{ merge(__source); }
 #endif // C++17
 
+      using _Base::hash_function;
+      using _Base::key_eq;
+
       iterator
       find(const key_type& __key)
       { return { _Base::find(__key), this }; }
@@ -567,6 +613,11 @@ namespace __debug
 	{ return { _Base::find(__k), this }; }
 #endif
 
+      using _Base::count;
+#if __cplusplus > 201703L
+      using _Base::contains;
+#endif
+
       std::pair<iterator, iterator>
       equal_range(const key_type& __key)
       {
@@ -605,6 +656,9 @@ namespace __debug
 	}
 #endif
 
+      using _Base::operator[];
+      using _Base::at;
+
       size_type
       erase(const key_type& __key)
       {
@@ -651,6 +705,9 @@ namespace __debug
 	return { __next, this };
       }
 
+      using _Base::rehash;
+      using _Base::reserve;
+
       _Base&
       _M_base() noexcept	{ return *this; }
 
@@ -843,7 +900,12 @@ namespace __debug
 
       typedef typename _Base::key_type			key_type;
       typedef typename _Base::value_type		value_type;
+      typedef typename _Base::mapped_type		mapped_type;
 
+      typedef typename _Base::pointer			pointer;
+      typedef typename _Base::const_pointer		const_pointer;
+      typedef typename _Base::reference			reference;
+      typedef typename _Base::const_reference		const_reference;
       typedef __gnu_debug::_Safe_iterator<
 	_Base_iterator, unordered_multimap>		iterator;
       typedef __gnu_debug::_Safe_iterator<
@@ -852,6 +914,7 @@ namespace __debug
 	_Base_local_iterator, unordered_multimap>	local_iterator;
       typedef __gnu_debug::_Safe_local_iterator<
 	_Base_const_local_iterator, unordered_multimap>	const_local_iterator;
+      typedef typename _Base::difference_type		difference_type;
 
       unordered_multimap() = default;
 
@@ -890,9 +953,9 @@ namespace __debug
 
       unordered_multimap(unordered_multimap&& __umap,
 			 const allocator_type& __a)
-      noexcept( noexcept(_Base(std::move(__umap._M_base()), __a)) )
-      : _Safe(std::move(__umap._M_safe()), __a),
-	_Base(std::move(__umap._M_base()), __a) { }
+      noexcept( noexcept(_Base(std::move(__umap), __a)) )
+      : _Safe(std::move(__umap), __a),
+	_Base(std::move(__umap), __a) { }
 
       unordered_multimap(initializer_list<value_type> __l,
 			 size_type __n = 0,
@@ -947,11 +1010,16 @@ namespace __debug
       unordered_multimap&
       operator=(initializer_list<value_type> __l)
       {
-	this->_M_base() = __l;
+	_Base::operator=(__l);
 	this->_M_invalidate_all();
 	return *this;
       }
 
+      using _Base::get_allocator;
+      using _Base::empty;
+      using _Base::size;
+      using _Base::max_size;
+
       void
       swap(unordered_multimap& __x)
 	noexcept( noexcept(declval<_Base&>().swap(__x)) )
@@ -1034,6 +1102,10 @@ namespace __debug
 	return { _Base::cend(__b), this };
       }
 
+      using _Base::bucket_count;
+      using _Base::max_bucket_count;
+      using _Base::bucket;
+
       size_type
       bucket_size(size_type __b) const
       {
@@ -1192,9 +1264,38 @@ namespace __debug
 	return { _Base::insert(__hint.base(), std::move(__nh)), this };
       }
 
-      using _Base::merge;
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_multimap<_Key, _Tp, _H2, _P2, _Alloc>& __source)
+	{
+	  auto __guard
+	    = _Safe::_S_umc_guard(std::__detail::_Select1st{}, __source);
+	  _Base::merge(__source);
+	}
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_multimap<_Key, _Tp, _H2, _P2, _Alloc>&& __source)
+	{ merge(__source); }
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_map<_Key, _Tp, _H2, _P2, _Alloc>& __source)
+	{
+	  auto __guard
+	    = _Safe::_S_uc_guard(std::__detail::_Select1st{}, __source);
+	  _Base::merge(__source);
+	}
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_map<_Key, _Tp, _H2, _P2, _Alloc>&& __source)
+	{ merge(__source); }
 #endif // C++17
 
+      using _Base::hash_function;
+      using _Base::key_eq;
+
       iterator
       find(const key_type& __key)
       { return { _Base::find(__key), this }; }
@@ -1221,6 +1322,11 @@ namespace __debug
 	{ return { _Base::find(__k), this }; }
 #endif
 
+      using _Base::count;
+#if __cplusplus > 201703L
+      using _Base::contains;
+#endif
+
       std::pair<iterator, iterator>
       equal_range(const key_type& __key)
       {
@@ -1309,6 +1415,9 @@ namespace __debug
 	return { __next, this };
       }
 
+      using _Base::rehash;
+      using _Base::reserve;
+
       _Base&
       _M_base() noexcept { return *this; }
 
diff --git a/libstdc++-v3/include/debug/unordered_set b/libstdc++-v3/include/debug/unordered_set
index c259109..3516af4 100644
--- a/libstdc++-v3/include/debug/unordered_set
+++ b/libstdc++-v3/include/debug/unordered_set
@@ -88,6 +88,7 @@ namespace __debug
 
     public:
       typedef typename _Base::size_type			size_type;
+      typedef typename _Base::difference_type		difference_type;
       typedef typename _Base::hasher			hasher;
       typedef typename _Base::key_equal			key_equal;
       typedef typename _Base::allocator_type		allocator_type;
@@ -95,6 +96,10 @@ namespace __debug
       typedef typename _Base::key_type			key_type;
       typedef typename _Base::value_type		value_type;
 
+      typedef typename _Base::pointer			pointer;
+      typedef typename _Base::const_pointer		const_pointer;
+      typedef typename _Base::reference			reference;
+      typedef typename _Base::const_reference		const_reference;
       typedef __gnu_debug::_Safe_iterator<
 	_Base_iterator, unordered_set>			iterator;
       typedef __gnu_debug::_Safe_iterator<
@@ -141,9 +146,9 @@ namespace __debug
 
       unordered_set(unordered_set&& __uset,
 		    const allocator_type& __a)
-      noexcept( noexcept(_Base(std::move(__uset._M_base()), __a)) )
-      : _Safe(std::move(__uset._M_safe()), __a),
-	_Base(std::move(__uset._M_base()), __a) { }
+      noexcept( noexcept(_Base(std::move(__uset), __a)) )
+      : _Safe(std::move(__uset), __a),
+	_Base(std::move(__uset), __a) { }
 
       unordered_set(initializer_list<value_type> __l,
 		    size_type __n = 0,
@@ -198,11 +203,16 @@ namespace __debug
       unordered_set&
       operator=(initializer_list<value_type> __l)
       {
-	_M_base() = __l;
+	_Base::operator=(__l);
 	this->_M_invalidate_all();
 	return *this;
       }
 
+      using _Base::get_allocator;
+      using _Base::empty;
+      using _Base::size;
+      using _Base::max_size;
+
       void
       swap(unordered_set& __x)
 	noexcept( noexcept(declval<_Base&>().swap(__x)) )
@@ -285,6 +295,9 @@ namespace __debug
 	return { _Base::cend(__b), this };
       }
 
+      using _Base::bucket_count;
+      using _Base::max_bucket_count;
+
       size_type
       bucket_size(size_type __b) const
       {
@@ -292,6 +305,9 @@ namespace __debug
 	return _Base::bucket_size(__b);
       }
 
+      using _Base::bucket;
+      using _Base::load_factor;
+
       float
       max_load_factor() const noexcept
       { return _Base::max_load_factor(); }
@@ -303,6 +319,9 @@ namespace __debug
 	_Base::max_load_factor(__f);
       }
 
+      using _Base::rehash;
+      using _Base::reserve;
+
       template<typename... _Args>
 	std::pair<iterator, bool>
 	emplace(_Args&&... __args)
@@ -423,9 +442,38 @@ namespace __debug
 	return { _Base::insert(__hint.base(), std::move(__nh)), this };
       }
 
-      using _Base::merge;
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_set<_Value, _H2, _P2, _Alloc>& __source)
+	{
+	  auto __guard
+	    = _Safe::_S_uc_guard(std::__detail::_Identity{}, __source);
+	  _Base::merge(__source);
+	}
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_set<_Value, _H2, _P2, _Alloc>&& __source)
+	{ merge(__source); }
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_multiset<_Value, _H2, _P2, _Alloc>& __source)
+	{
+	  auto __guard
+	    = _Safe::_S_umc_guard(std::__detail::_Identity{}, __source);
+	  _Base::merge(__source);
+	}
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_multiset<_Value, _H2, _P2, _Alloc>&& __source)
+	{ merge(__source); }
 #endif // C++17
 
+      using _Base::hash_function;
+      using _Base::key_eq;
+
       iterator
       find(const key_type& __key)
       { return { _Base::find(__key), this }; }
@@ -452,6 +500,12 @@ namespace __debug
 	{ return { _Base::find(__k), this }; }
 #endif
 
+      using _Base::count;
+
+#if __cplusplus > 201703L
+      using _Base::contains;
+#endif
+
       std::pair<iterator, iterator>
       equal_range(const key_type& __key)
       {
@@ -707,6 +761,7 @@ namespace __debug
 
     public:
       typedef typename _Base::size_type			size_type;
+      typedef typename _Base::difference_type		difference_type;
       typedef typename _Base::hasher			hasher;
       typedef typename _Base::key_equal			key_equal;
       typedef typename _Base::allocator_type		allocator_type;
@@ -714,6 +769,10 @@ namespace __debug
       typedef typename _Base::key_type			key_type;
       typedef typename _Base::value_type		value_type;
 
+      typedef typename _Base::pointer			pointer;
+      typedef typename _Base::const_pointer		const_pointer;
+      typedef typename _Base::reference			reference;
+      typedef typename _Base::const_reference		const_reference;
       typedef __gnu_debug::_Safe_iterator<
 	_Base_iterator, unordered_multiset>		iterator;
       typedef __gnu_debug::_Safe_iterator<
@@ -760,9 +819,9 @@ namespace __debug
 
       unordered_multiset(unordered_multiset&& __uset,
 			 const allocator_type& __a)
-      noexcept( noexcept(_Base(std::move(__uset._M_base()), __a)) )
-      : _Safe(std::move(__uset._M_safe()), __a),
-	_Base(std::move(__uset._M_base()), __a) { }
+      noexcept( noexcept(_Base(std::move(__uset), __a)) )
+      : _Safe(std::move(__uset), __a),
+	_Base(std::move(__uset), __a) { }
 
       unordered_multiset(initializer_list<value_type> __l,
 			 size_type __n = 0,
@@ -817,11 +876,16 @@ namespace __debug
       unordered_multiset&
       operator=(initializer_list<value_type> __l)
       {
-	this->_M_base() = __l;
+	_Base::operator=(__l);
 	this->_M_invalidate_all();
 	return *this;
       }
 
+      using _Base::get_allocator;
+      using _Base::empty;
+      using _Base::size;
+      using _Base::max_size;
+
       void
       swap(unordered_multiset& __x)
 	noexcept( noexcept(declval<_Base&>().swap(__x)) )
@@ -904,6 +968,9 @@ namespace __debug
 	return { _Base::cend(__b), this };
       }
 
+      using _Base::bucket_count;
+      using _Base::max_bucket_count;
+
       size_type
       bucket_size(size_type __b) const
       {
@@ -911,6 +978,9 @@ namespace __debug
 	return _Base::bucket_size(__b);
       }
 
+      using _Base::bucket;
+      using _Base::load_factor;
+
       float
       max_load_factor() const noexcept
       { return _Base::max_load_factor(); }
@@ -922,6 +992,9 @@ namespace __debug
 	_Base::max_load_factor(__f);
       }
 
+      using _Base::rehash;
+      using _Base::reserve;
+
       template<typename... _Args>
 	iterator
 	emplace(_Args&&... __args)
@@ -1037,9 +1110,38 @@ namespace __debug
 	return { _Base::insert(__hint.base(), std::move(__nh)), this };
       }
 
-      using _Base::merge;
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_multiset<_Value, _H2, _P2, _Alloc>& __source)
+	{
+	  auto __guard
+	    = _Safe::_S_umc_guard(std::__detail::_Identity{}, __source);
+	  _Base::merge(__source);
+	}
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_multiset<_Value, _H2, _P2, _Alloc>&& __source)
+	{ merge(__source); }
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_set<_Value, _H2, _P2, _Alloc>& __source)
+	{
+	  auto __guard
+	    = _Safe::_S_uc_guard(std::__detail::_Identity{}, __source);
+	  _Base::merge(__source);
+	}
+
+      template<typename _H2, typename _P2>
+	void
+	merge(unordered_set<_Value, _H2, _P2, _Alloc>&& __source)
+	{ merge(__source); }
 #endif // C++17
 
+      using _Base::hash_function;
+      using _Base::key_eq;
+
       iterator
       find(const key_type& __key)
       { return { _Base::find(__key), this }; }
@@ -1066,6 +1168,12 @@ namespace __debug
 	{ return { _Base::find(__k), this }; }
 #endif
 
+      using _Base::count;
+
+#if __cplusplus > 201703L
+      using _Base::contains;
+#endif
+
       std::pair<iterator, iterator>
       equal_range(const key_type& __key)
       {
diff --git a/libstdc++-v3/include/debug/vector b/libstdc++-v3/include/debug/vector
index 03fd940..ed10e82 100644
--- a/libstdc++-v3/include/debug/vector
+++ b/libstdc++-v3/include/debug/vector
@@ -71,18 +71,18 @@ namespace __gnu_debug
 	: _M_guaranteed_capacity(__n)
       { }
 
-#if __cplusplus >= 201103L
-      _Safe_vector(_Safe_vector&& __x) noexcept
-	: _Safe_vector()
-      { __x._M_guaranteed_capacity = 0; }
-
       _Safe_vector&
-      operator=(const _Safe_vector&) noexcept
+      operator=(const _Safe_vector&) _GLIBCXX_NOEXCEPT
       {
 	_M_update_guaranteed_capacity();
 	return *this;
       }
 
+#if __cplusplus >= 201103L
+      _Safe_vector(_Safe_vector&& __x) noexcept
+	: _Safe_vector()
+      { __x._M_guaranteed_capacity = 0; }
+
       _Safe_vector&
       operator=(_Safe_vector&& __x) noexcept
       {
@@ -219,8 +219,8 @@ namespace __debug
       noexcept(
 	std::is_nothrow_constructible<_Base,
 	  _Base, const allocator_type&>::value )
-      : _Safe(std::move(__x._M_safe()), __a),
-	_Base(std::move(__x._M_base()), __a),
+      : _Safe(std::move(__x), __a),
+	_Base(std::move(__x), __a),
 	_Safe_vector(std::move(__x)) { }
 
       vector(initializer_list<value_type> __l,
@@ -234,16 +234,7 @@ namespace __debug
       vector(_Base_ref __x)
       : _Base(__x._M_ref) { }
 
-#if __cplusplus < 201103L
-      vector&
-      operator=(const vector& __x)
-      {
-	this->_M_safe() = __x;
-	_M_base() = __x;
-	this->_M_update_guaranteed_capacity();
-	return *this;
-      }
-#else
+#if __cplusplus >= 201103L
       vector&
       operator=(const vector&) = default;
 
@@ -253,7 +244,7 @@ namespace __debug
       vector&
       operator=(initializer_list<value_type> __l)
       {
-	_M_base() = __l;
+	_Base::operator=(__l);
 	this->_M_invalidate_all();
 	this->_M_update_guaranteed_capacity();
 	return *this;
@@ -449,7 +440,7 @@ namespace __debug
       operator[](size_type __n) _GLIBCXX_NOEXCEPT
       {
 	__glibcxx_check_subscript(__n);
-	return _M_base()[__n];
+	return _Base::operator[](__n);
       }
 
       _GLIBCXX_NODISCARD
@@ -457,7 +448,7 @@ namespace __debug
       operator[](size_type __n) const _GLIBCXX_NOEXCEPT
       {
 	__glibcxx_check_subscript(__n);
-	return _M_base()[__n];
+	return _Base::operator[](__n);
       }
 
       using _Base::at;
diff --git a/libstdc++-v3/include/experimental/internet b/libstdc++-v3/include/experimental/internet
index 65c97de..95b8cdc 100644
--- a/libstdc++-v3/include/experimental/internet
+++ b/libstdc++-v3/include/experimental/internet
@@ -460,7 +460,9 @@ namespace ip
     // constructors:
     constexpr address() noexcept : _M_v4(), _M_is_v4(true) { }
 
+#if __cpp_constexpr_dynamic_alloc
     constexpr
+#endif
     address(const address& __a) noexcept : _M_uninit(), _M_is_v4(__a._M_is_v4)
     {
       if (_M_is_v4)
diff --git a/libstdc++-v3/include/precompiled/stdc++.h b/libstdc++-v3/include/precompiled/stdc++.h
index d2601d7..e1c10e6 100644
--- a/libstdc++-v3/include/precompiled/stdc++.h
+++ b/libstdc++-v3/include/precompiled/stdc++.h
@@ -133,7 +133,7 @@
 #include <variant>
 #endif
 
-#if __cplusplus > 201703L
+#if __cplusplus >= 202002L
 #include <barrier>
 #include <bit>
 #include <compare>
@@ -151,3 +151,7 @@
 #include <syncstream>
 #include <version>
 #endif
+
+#if __cplusplus > 202002L
+#include <spanstream>
+#endif
diff --git a/libstdc++-v3/include/std/any b/libstdc++-v3/include/std/any
index e3d9d77..f75dddf 100644
--- a/libstdc++-v3/include/std/any
+++ b/libstdc++-v3/include/std/any
@@ -72,8 +72,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   /**
    *  @brief A type-safe container of any type.
    *
-   *  An @c any object's state is either empty or it stores a contained object
+   *  An `any` object's state is either empty or it stores a contained object
    *  of CopyConstructible type.
+   *
+   *  @since C++17
    */
   class any
   {
@@ -337,9 +339,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     }
 #endif
 
+    /// @cond undocumented
     template<typename _Tp>
       static constexpr bool __is_valid_cast()
       { return __or_<is_reference<_Tp>, is_copy_constructible<_Tp>>::value; }
+    /// @endcond
 
   private:
     enum _Op {
@@ -356,8 +360,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     void (*_M_manager)(_Op, const any*, _Arg*);
     _Storage _M_storage;
 
+    /// @cond undocumented
     template<typename _Tp>
       friend void* __any_caster(const any* __any);
+    /// @endcond
 
     // Manage in-place contained object.
     template<typename _Tp>
@@ -422,16 +428,21 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   /// Exchange the states of two @c any objects.
   inline void swap(any& __x, any& __y) noexcept { __x.swap(__y); }
 
-  /// Create an any holding a @c _Tp constructed from @c __args.
+  /// Create an `any` holding a `_Tp` constructed from `__args...`.
   template <typename _Tp, typename... _Args>
-    any make_any(_Args&&... __args)
+    inline
+    enable_if_t<is_constructible_v<any, in_place_type_t<_Tp>, _Args...>, any>
+    make_any(_Args&&... __args)
     {
       return any(in_place_type<_Tp>, std::forward<_Args>(__args)...);
     }
 
-  /// Create an any holding a @c _Tp constructed from @c __il and @c __args.
+  /// Create an `any` holding a `_Tp` constructed from `__il` and `__args...`.
   template <typename _Tp, typename _Up, typename... _Args>
-    any make_any(initializer_list<_Up> __il, _Args&&... __args)
+    inline
+    enable_if_t<is_constructible_v<any, in_place_type_t<_Tp>,
+				   initializer_list<_Up>&, _Args...>, any>
+    make_any(initializer_list<_Up> __il, _Args&&... __args)
     {
       return any(in_place_type<_Tp>, __il, std::forward<_Args>(__args)...);
     }
diff --git a/libstdc++-v3/include/std/array b/libstdc++-v3/include/std/array
index 3e12d35..413f8e2 100644
--- a/libstdc++-v3/include/std/array
+++ b/libstdc++-v3/include/std/array
@@ -481,6 +481,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       using type = _Tp;
     };
 
+#if __cplusplus >= 201703L
+  template<typename _Tp, size_t _Nm>
+    inline constexpr size_t tuple_size_v<array<_Tp, _Nm>> = _Nm;
+
+  template<typename _Tp, size_t _Nm>
+    inline constexpr size_t tuple_size_v<const array<_Tp, _Nm>> = _Nm;
+#endif
+
   template<typename _Tp, size_t _Nm>
     struct __is_tuple_like_impl<array<_Tp, _Nm>> : true_type
     { };
diff --git a/libstdc++-v3/include/std/functional b/libstdc++-v3/include/std/functional
index 2db1c05..5cf4417 100644
--- a/libstdc++-v3/include/std/functional
+++ b/libstdc++-v3/include/std/functional
@@ -79,6 +79,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   /** @brief The type of placeholder objects defined by libstdc++.
    *  @ingroup binders
+   *  @since C++11
    */
   template<int _Num> struct _Placeholder { };
 
@@ -90,7 +91,17 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #  define __cpp_lib_constexpr_functional 201907L
 # endif
 
-  /// Invoke a callable object.
+  /** Invoke a callable object.
+   *
+   * `std::invoke` takes a callable object as its first argument and calls it
+   * with the remaining arguments. The callable object can be a pointer or
+   * reference to a function, a lambda closure, a class with `operator()`,
+   * or even a pointer-to-member.  For a pointer-to-member the first argument
+   * must be a reference or pointer to the object that the pointer-to-member
+   * will be applied to.
+   *
+   *  @since C++17
+   */
   template<typename _Callable, typename... _Args>
     inline _GLIBCXX20_CONSTEXPR invoke_result_t<_Callable, _Args...>
     invoke(_Callable&& __fn, _Args&&... __args)
@@ -103,7 +114,13 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #if __cplusplus > 202002L
 # define __cpp_lib_invoke_r 202106L
 
-  /// Invoke a callable object and convert the result to _Res.
+  /** Invoke a callable object and convert the result to `_Res`.
+   *
+   * `std::invoke_r<R>(f, args...)` is equivalent to `std::invoke(f, args...)`
+   * with the result implicitly converted to `R`.
+   *
+   *  @since C++23
+   */
   template<typename _Res, typename _Callable, typename... _Args>
     requires is_invocable_r_v<_Res, _Callable, _Args...>
     constexpr _Res
@@ -116,6 +133,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #endif // C++23
 #endif // C++17
 
+  /// @cond undocumented
+
   template<typename _MemFunPtr,
 	   bool __is_mem_fn = is_member_function_pointer<_MemFunPtr>::value>
     class _Mem_fn_base
@@ -182,13 +201,32 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     {
       using _Mem_fn_base<_Res _Class::*>::_Mem_fn_base;
     };
+  /// @endcond
 
   // _GLIBCXX_RESOLVE_LIB_DEFECTS
   // 2048.  Unnecessary mem_fn overloads
   /**
-   *  @brief Returns a function object that forwards to the member
-   *  pointer @a pm.
-   *  @ingroup functors
+   * @brief Returns a function object that forwards to the member pointer
+   * pointer `pm`.
+   *
+   * This allows a pointer-to-member to be transformed into a function object
+   * that can be called with an object expression as its first argument.
+   *
+   * For a pointer-to-data-member the result must be called with exactly one
+   * argument, the object expression that would be used as the first operand
+   * in a `obj.*memptr` or `objp->*memptr` expression.
+   *
+   * For a pointer-to-member-function the result must be called with an object
+   * expression and any additional arguments to pass to the member function,
+   * as in an expression like `(obj.*memfun)(args...)` or
+   * `(objp->*memfun)(args...)`.
+   *
+   * The object expression can be a pointer, reference, `reference_wrapper`,
+   * or smart pointer, and the call wrapper will dereference it as needed
+   * to apply the pointer-to-member.
+   *
+   * @ingroup functors
+   * @since C++11
    */
   template<typename _Tp, typename _Class>
     _GLIBCXX20_CONSTEXPR
@@ -199,12 +237,15 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     }
 
   /**
-   *  @brief Determines if the given type _Tp is a function object that
-   *  should be treated as a subexpression when evaluating calls to
-   *  function objects returned by bind().
+   * @brief Trait that identifies a bind expression.
+   * 
+   * Determines if the given type `_Tp` is a function object that
+   * should be treated as a subexpression when evaluating calls to
+   * function objects returned by `std::bind`.
    *
-   *  C++11 [func.bind.isbind].
-   *  @ingroup binders
+   * C++11 [func.bind.isbind].
+   * @ingroup binders
+   * @since C++11
    */
   template<typename _Tp>
     struct is_bind_expression
@@ -216,6 +257,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
    *
    *  C++11 [func.bind.isplace].
    *  @ingroup binders
+   *  @since C++11
    */
   template<typename _Tp>
     struct is_placeholder
@@ -232,6 +274,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   /** @namespace std::placeholders
    *  @brief ISO C++ 2011 namespace for std::bind placeholders.
    *  @ingroup binders
+   *  @since C++11
    */
   namespace placeholders
   {
@@ -274,6 +317,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
    *  Partial specialization of is_placeholder that provides the placeholder
    *  number for the placeholder objects defined by libstdc++.
    *  @ingroup binders
+   *  @since C++11
    */
   template<int _Num>
     struct is_placeholder<_Placeholder<_Num> >
@@ -285,6 +329,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     : public integral_constant<int, _Num>
     { };
 
+  /// @cond undocumented
 
   // Like tuple_element_t but SFINAE-friendly.
   template<std::size_t __i, typename _Tuple>
@@ -414,6 +459,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     -> __tuple_element_t<_Ind, tuple<_Tp...>> const volatile&
     { return std::get<_Ind>(const_cast<const tuple<_Tp...>&>(__tuple)); }
 
+  /// @endcond
+
   /// Type of the function object returned from bind().
   template<typename _Signature>
     class _Bind;
@@ -799,6 +846,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   /**
    *  @brief Function template for std::bind.
    *  @ingroup binders
+   *  @since C++11
    */
   template<typename _Func, typename... _BoundArgs>
     inline _GLIBCXX20_CONSTEXPR typename
@@ -823,6 +871,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   /**
    *  @brief Function template for std::bind<R>.
    *  @ingroup binders
+   *  @since C++11
    */
   template<typename _Result, typename _Func, typename... _BoundArgs>
     inline _GLIBCXX20_CONSTEXPR
@@ -923,6 +972,15 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     using _Bind_front_t
       = _Bind_front<decay_t<_Fn>, decay_t<_Args>...>;
 
+  /** Create call wrapper by partial application of arguments to function.
+   *
+   * The result of `std::bind_front(f, args...)` is a function object that
+   * stores `f` and the bound arguments, `args...`. When that function
+   * object is invoked with `call_args...` it returns the result of calling
+   * `f(args..., call_args...)`.
+   *
+   *  @since C++20
+   */
   template<typename _Fn, typename... _Args>
     constexpr _Bind_front_t<_Fn, _Args...>
     bind_front(_Fn&& __fn, _Args&&... __args)
@@ -932,7 +990,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       return _Bind_front_t<_Fn, _Args...>(0, std::forward<_Fn>(__fn),
 					  std::forward<_Args>(__args)...);
     }
-#endif
+#endif // C++20
 
 #if __cplusplus >= 201402L
   /// Generalized negator.
@@ -1003,8 +1061,20 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     struct __is_byte_like<byte, equal_to<void>>
     : true_type { };
 
+  // [func.not_fn] Function template not_fn
 #define __cpp_lib_not_fn 201603
-  /// [func.not_fn] Function template not_fn
+  /** Wrap a function object to create one that negates its result.
+   *
+   * The function template `std::not_fn` creates a "forwarding call wrapper",
+   * which is a function object that wraps another function object and
+   * when called, forwards its arguments to the wrapped function object.
+   *
+   * The result of invoking the wrapper is the negation (using `!`) of
+   * the wrapped function object.
+   *
+   *  @ingroup functors
+   *  @since C++17
+   */
   template<typename _Fn>
     _GLIBCXX20_CONSTEXPR
     inline auto
diff --git a/libstdc++-v3/include/std/memory_resource b/libstdc++-v3/include/std/memory_resource
index 6bca0af..97d37f1 100644
--- a/libstdc++-v3/include/std/memory_resource
+++ b/libstdc++-v3/include/std/memory_resource
@@ -33,6 +33,7 @@
 
 #if __cplusplus >= 201703L
 
+#include <new>
 #include <vector>			// vector
 #include <cstddef>			// size_t, max_align_t, byte
 #include <shared_mutex>			// shared_mutex
@@ -103,7 +104,7 @@ namespace pmr
     void*
     allocate(size_t __bytes, size_t __alignment = _S_max_align)
     __attribute__((__returns_nonnull__,__alloc_size__(2),__alloc_align__(3)))
-    { return do_allocate(__bytes, __alignment); }
+    { return ::operator new(__bytes, do_allocate(__bytes, __alignment)); }
 
     void
     deallocate(void* __p, size_t __bytes, size_t __alignment = _S_max_align)
diff --git a/libstdc++-v3/include/std/ranges b/libstdc++-v3/include/std/ranges
index 30ba060..7521cd6 100644
--- a/libstdc++-v3/include/std/ranges
+++ b/libstdc++-v3/include/std/ranges
@@ -42,6 +42,7 @@
 #include <initializer_list>
 #include <iterator>
 #include <optional>
+#include <span>
 #include <tuple>
 #include <bits/ranges_util.h>
 #include <bits/refwrap.h>
@@ -781,10 +782,27 @@ namespace views
       friend _Iterator;
     };
 
-  template<typename _Val, typename _CharT, typename _Traits>
-    basic_istream_view<_Val, _CharT, _Traits>
-    istream_view(basic_istream<_CharT, _Traits>& __s)
-    { return basic_istream_view<_Val, _CharT, _Traits>{__s}; }
+  template<typename _Val>
+    using istream_view = basic_istream_view<_Val, char>;
+
+  template<typename _Val>
+    using wistream_view = basic_istream_view<_Val, wchar_t>;
+
+namespace views
+{
+  template<typename _Tp>
+    struct _Istream
+    {
+      template<typename _CharT, typename _Traits>
+	[[nodiscard]]
+	constexpr auto
+	operator()(basic_istream<_CharT, _Traits>& __e) const
+	{ return basic_istream_view<_Tp, _CharT, _Traits>(__e); }
+    };
+
+  template<typename _Tp>
+    inline constexpr _Istream<_Tp> istream;
+}
 
   // C++20 24.7 [range.adaptors] Range adaptors
 
@@ -2125,19 +2143,67 @@ namespace views::__adaptor
   {
     namespace __detail
     {
-      template<typename _Range, typename _Tp>
+      template<typename _Range>
+	inline constexpr bool __is_empty_view = false;
+
+      template<typename _Tp>
+	inline constexpr bool __is_empty_view<empty_view<_Tp>> = true;
+
+      template<typename _Range>
+	inline constexpr bool __is_basic_string_view = false;
+
+      template<typename _CharT, typename _Traits>
+	inline constexpr bool __is_basic_string_view<basic_string_view<_CharT, _Traits>>
+	  = true;
+
+      template<typename _Range>
+	inline constexpr bool __is_subrange = false;
+
+      template<typename _Iter, typename _Sent, subrange_kind _Kind>
+	inline constexpr bool __is_subrange<subrange<_Iter, _Sent, _Kind>> = true;
+
+      template<typename _Range>
+	inline constexpr bool __is_iota_view = false;
+
+      template<typename _Winc, typename _Bound>
+	inline constexpr bool __is_iota_view<iota_view<_Winc, _Bound>> = true;
+
+      template<typename _Range, typename _Dp>
 	concept __can_take_view
-	  = requires { take_view(std::declval<_Range>(), std::declval<_Tp>()); };
+	  = requires { take_view(std::declval<_Range>(), std::declval<_Dp>()); };
     } // namespace __detail
 
     struct _Take : __adaptor::_RangeAdaptor<_Take>
     {
-      template<viewable_range _Range, typename _Tp>
-	requires __detail::__can_take_view<_Range, _Tp>
+      template<viewable_range _Range, typename _Dp = range_difference_t<_Range>>
+	requires __detail::__can_take_view<_Range, _Dp>
 	constexpr auto
-	operator() [[nodiscard]] (_Range&& __r, _Tp&& __n) const
+	operator() [[nodiscard]] (_Range&& __r, type_identity_t<_Dp> __n) const
 	{
-	  return take_view(std::forward<_Range>(__r), std::forward<_Tp>(__n));
+	  using _Tp = remove_cvref_t<_Range>;
+	  if constexpr (__detail::__is_empty_view<_Tp>)
+	    return _Tp();
+	  else if constexpr (random_access_range<_Tp>
+			     && sized_range<_Tp>
+			     && (std::__detail::__is_span<_Tp>
+				 || __detail::__is_basic_string_view<_Tp>
+				 || __detail::__is_subrange<_Tp>
+				 || __detail::__is_iota_view<_Tp>))
+	    {
+	      __n = std::min<_Dp>(ranges::distance(__r), __n);
+	      auto __begin = ranges::begin(__r);
+	      auto __end = __begin + __n;
+	      if constexpr (std::__detail::__is_span<_Tp>)
+		return span<typename _Tp::element_type>(__begin, __end);
+	      else if constexpr (__detail::__is_basic_string_view<_Tp>)
+		return _Tp(__begin, __end);
+	      else if constexpr (__detail::__is_subrange<_Tp>)
+		return subrange<iterator_t<_Tp>>(__begin, __end);
+	      else
+		return iota_view(*__begin, *__end);
+	    }
+	  else
+	    return take_view(std::forward<_Range>(__r), __n);
 	}
 
       using _RangeAdaptor<_Take>::operator();
@@ -2372,19 +2438,49 @@ namespace views::__adaptor
   {
     namespace __detail
     {
-      template<typename _Range, typename _Tp>
+      template<typename _Range, typename _Dp>
 	concept __can_drop_view
-	  = requires { drop_view(std::declval<_Range>(), std::declval<_Tp>()); };
+	  = requires { drop_view(std::declval<_Range>(), std::declval<_Dp>()); };
     } // namespace __detail
 
     struct _Drop : __adaptor::_RangeAdaptor<_Drop>
     {
-      template<viewable_range _Range, typename _Tp>
-	requires __detail::__can_drop_view<_Range, _Tp>
+      template<viewable_range _Range, typename _Dp = range_difference_t<_Range>>
+	requires __detail::__can_drop_view<_Range, _Dp>
 	constexpr auto
-	operator() [[nodiscard]] (_Range&& __r, _Tp&& __n) const
+	operator() [[nodiscard]] (_Range&& __r, type_identity_t<_Dp> __n) const
 	{
-	  return drop_view(std::forward<_Range>(__r), std::forward<_Tp>(__n));
+	  using _Tp = remove_cvref_t<_Range>;
+	  if constexpr (__detail::__is_empty_view<_Tp>)
+	    return _Tp();
+	  else if constexpr (random_access_range<_Tp>
+			     && sized_range<_Tp>
+			     && (std::__detail::__is_span<_Tp>
+				 || __detail::__is_basic_string_view<_Tp>
+				 || __detail::__is_iota_view<_Tp>
+				 || __detail::__is_subrange<_Tp>))
+	    {
+	      __n = std::min<_Dp>(ranges::distance(__r), __n);
+	      auto __begin = ranges::begin(__r) + __n;
+	      auto __end = ranges::end(__r);
+	      if constexpr (std::__detail::__is_span<_Tp>)
+		return span<typename _Tp::element_type>(__begin, __end);
+	      else if constexpr (__detail::__is_subrange<_Tp>)
+		{
+		  if constexpr (_Tp::_S_store_size)
+		    {
+		      using ranges::__detail::__to_unsigned_like;
+		      auto __m = ranges::distance(__r) - __n;
+		      return _Tp(__begin, __end, __to_unsigned_like(__m));
+		    }
+		  else
+		    return _Tp(__begin, __end);
+		}
+	      else
+		return _Tp(__begin, __end);
+	    }
+	  else
+	    return drop_view(std::forward<_Range>(__r), __n);
 	}
 
       using _RangeAdaptor<_Drop>::operator();
@@ -3519,7 +3615,9 @@ namespace views::__adaptor
       constexpr auto
       operator() [[nodiscard]] (_Iter __i, iter_difference_t<_Iter> __n) const
       {
-	if constexpr (random_access_iterator<_Iter>)
+	if constexpr (contiguous_iterator<_Iter>)
+	  return span(std::__to_address(__i), __n);
+	else if constexpr (random_access_iterator<_Iter>)
 	  return subrange(__i, __i + __n);
 	else
 	  return subrange(counted_iterator(std::move(__i), __n),
diff --git a/libstdc++-v3/include/std/span b/libstdc++-v3/include/std/span
index af0d24b..0898ea8 100644
--- a/libstdc++-v3/include/std/span
+++ b/libstdc++-v3/include/std/span
@@ -58,16 +58,16 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   namespace __detail
   {
     template<typename _Tp>
-      struct __is_std_span : false_type { };
+      inline constexpr bool __is_span = false;
 
     template<typename _Tp, size_t _Num>
-      struct __is_std_span<span<_Tp, _Num>> : true_type { };
+      inline constexpr bool __is_span<span<_Tp, _Num>> = true;
 
     template<typename _Tp>
-      struct __is_std_array : false_type { };
+      inline constexpr bool __is_std_array = false;
 
     template<typename _Tp, size_t _Num>
-      struct __is_std_array<std::array<_Tp, _Num>> : true_type { };
+      inline constexpr bool __is_std_array<std::array<_Tp, _Num>> = true;
 
     template<size_t _Extent>
       class __extent_storage
@@ -201,11 +201,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	{ }
 
       template<typename _Range>
-	requires ranges::contiguous_range<_Range> && ranges::sized_range<_Range>
-	  && (ranges::borrowed_range<_Range> || is_const_v<element_type>)
-	  && (!__detail::__is_std_span<remove_cvref_t<_Range>>::value)
-	  && (!__detail::__is_std_array<remove_cvref_t<_Range>>::value)
+	requires (!__detail::__is_span<remove_cvref_t<_Range>>)
+	  && (!__detail::__is_std_array<remove_cvref_t<_Range>>)
 	  && (!is_array_v<remove_cvref_t<_Range>>)
+	  && ranges::contiguous_range<_Range> && ranges::sized_range<_Range>
+	  && (ranges::borrowed_range<_Range> || is_const_v<element_type>)
 	  && __is_compatible_ref<ranges::range_reference_t<_Range>>::value
 	constexpr explicit(extent != dynamic_extent)
 	span(_Range&& __range)
diff --git a/libstdc++-v3/include/std/spanstream b/libstdc++-v3/include/std/spanstream
new file mode 100644
index 0000000..240866f
--- /dev/null
+++ b/libstdc++-v3/include/std/spanstream
@@ -0,0 +1,446 @@
+// Streams based on std::span -*- C++ -*-
+
+// Copyright The GNU Toolchain Authors.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+/** @file spanstream
+ *  This is a Standard C++ Library header.
+ */
+
+#ifndef _GLIBCXX_SPANSTREAM
+#define _GLIBCXX_SPANSTREAM 1
+
+#pragma GCC system_header
+
+#if __cplusplus > 202002L
+#include <span>
+#include <streambuf>
+#include <istream>
+#include <ostream>
+#include <bits/ranges_base.h>
+
+#if __cpp_lib_span
+namespace std _GLIBCXX_VISIBILITY(default)
+{
+_GLIBCXX_BEGIN_NAMESPACE_VERSION
+
+#define __cpp_lib_spanstream 202106L
+
+template<typename _CharT, typename _Traits = char_traits<_CharT>>
+  class basic_spanbuf
+  : public basic_streambuf<_CharT, _Traits>
+  {
+    using __streambuf_type = basic_streambuf<_CharT, _Traits>;
+
+  public:
+    using char_type   = _CharT;
+    using int_type    = typename _Traits::int_type;
+    using pos_type    = typename _Traits::pos_type;
+    using off_type    = typename _Traits::off_type;
+    using traits_type = _Traits;
+
+    // [spanbuf.ctor], constructors
+    basic_spanbuf() : basic_spanbuf(ios_base::in | ios_base::out)
+    { }
+
+    explicit
+    basic_spanbuf(ios_base::openmode __which)
+    : __streambuf_type(), _M_mode(__which)
+    { }
+
+    explicit
+    basic_spanbuf(std::span<_CharT> __s,
+		  ios_base::openmode __which = ios_base::in | ios_base::out)
+    : __streambuf_type(), _M_mode(__which)
+    { span(__s); }
+
+    basic_spanbuf(const basic_spanbuf&) = delete;
+
+    /// Move constructor. In this implementation `rhs` is left unchanged.
+    basic_spanbuf(basic_spanbuf&& __rhs)
+    : __streambuf_type(__rhs), _M_mode(__rhs._M_mode)
+    { span(__rhs._M_buf); }
+
+    // [spanbuf.assign], assignment and swap
+    basic_spanbuf& operator=(const basic_spanbuf&) = delete;
+
+    basic_spanbuf&
+    operator=(basic_spanbuf&& __rhs)
+    {
+      basic_spanbuf(std::move(__rhs))->swap(*this);
+      return *this;
+    }
+
+    void
+    swap(basic_spanbuf& __rhs)
+    {
+      __streambuf_type::swap(__rhs);
+      std::swap(_M_mode, __rhs._M_mode);
+      std::swap(_M_buf, __rhs._M_buf);
+    }
+
+    // [spanbuf.members], member functions
+    std::span<_CharT>
+    span() const noexcept
+    {
+      if (_M_mode & ios_base::out)
+	return {this->pbase(), this->pptr()};
+      else
+	return _M_buf;
+    }
+
+    void
+    span(std::span<_CharT> __s) noexcept
+    {
+      _M_buf = __s;
+      if (_M_mode & ios_base::out)
+	{
+	  this->setp(__s.data(), __s.data() + __s.size());
+	  if (_M_mode & ios_base::ate)
+	    this->pbump(__s.size());
+	}
+      if (_M_mode & ios_base::in)
+	this->setg(__s.data(), __s.data(), __s.data() + __s.size());
+    }
+
+  protected:
+    // [spanbuf.virtuals], overridden virtual functions
+    basic_streambuf<_CharT, _Traits>*
+    setbuf(_CharT* __s, streamsize __n) override
+    {
+      span({__s, __n});
+      return this;
+    }
+
+    pos_type
+    seekoff(off_type __off, ios_base::seekdir __way,
+	    ios_base::openmode __which = ios_base::in | ios_base::out) override
+    {
+      pos_type __ret =  pos_type(off_type(-1));
+
+      if (__way == ios_base::beg)
+	{
+	  if (0 <= __off && __off <= _M_buf.size())
+	    {
+	      if (__which & ios_base::in)
+		this->setg(this->eback(), this->eback() + __off, this->egptr());
+
+	      if (__which & ios_base::out)
+		{
+		  this->setp(this->pbase(), this->epptr());
+		  this->pbump(__off);
+		}
+
+	      __ret = pos_type(__off);
+	    }
+	}
+      else
+	{
+	  off_type __base;
+	  __which &= (ios_base::in|ios_base::out);
+
+	  if (__which == ios_base::out)
+	    __base = this->pptr() - this->pbase();
+	  else if (__way == ios_base::cur)
+	    {
+	      if (__which == ios_base::in)
+		__base = this->gptr() - this->eback();
+	      else
+		return __ret;
+	    }
+	  else if (__way == ios_base::end)
+	    __base = _M_buf.size();
+
+	  if (__builtin_add_overflow(__base, __off, &__off))
+	    return __ret;
+
+	  if (__off < 0 || __off > _M_buf.size())
+	    return __ret;
+
+	  if (__which & ios_base::in)
+	    this->setg(this->eback(), this->eback() + __off, this->egptr());
+
+	  if (__which & ios_base::out)
+	    {
+	      this->setp(this->pbase(), this->epptr());
+	      this->pbump(__off);
+	    }
+
+	  __ret = pos_type(__off);
+
+	}
+      return __ret;
+    }
+
+    pos_type
+    seekpos(pos_type __sp,
+	    ios_base::openmode __which = ios_base::in | ios_base::out) override
+    { return seekoff(off_type(__sp), ios_base::beg, __which); }
+
+  private:
+
+    ios_base::openmode _M_mode;
+    std::span<_CharT> _M_buf;
+  };
+
+template<typename _CharT, typename _Traits>
+  inline void
+  swap(basic_spanbuf<_CharT, _Traits>& __x,
+       basic_spanbuf<_CharT, _Traits>& __y)
+  { __x.swap(__y); }
+
+using spanbuf = basic_spanbuf<char>;
+using wspanbuf = basic_spanbuf<wchar_t>;
+
+template<typename _CharT, typename _Traits = char_traits<_CharT>>
+  class basic_ispanstream
+  : public basic_istream<_CharT, _Traits>
+  {
+    using __istream_type = basic_istream<_CharT, _Traits>;
+
+  public:
+    using char_type   = _CharT;
+    using int_type    = typename _Traits::int_type;
+    using pos_type    = typename _Traits::pos_type;
+    using off_type    = typename _Traits::off_type;
+    using traits_type = _Traits;
+
+    // [ispanstream.ctor], constructors
+    explicit
+    basic_ispanstream(std::span<_CharT> __s,
+		      ios_base::openmode __which = ios_base::in)
+    : __istream_type(std::__addressof(_M_sb)),
+      _M_sb(__s, __which | ios_base::in)
+    { }
+
+    basic_ispanstream(const basic_ispanstream&) = delete;
+
+    basic_ispanstream(basic_ispanstream&& __rhs)
+    : __istream_type(std::move(__rhs)), _M_sb(std::move(__rhs._M_sb))
+    {
+      __istream_type::set_rdbuf(std::addressof(_M_sb));
+    }
+
+    template<typename _Ros>
+      requires ranges::borrowed_range<_Ros>
+	&& (!convertible_to<_Ros, std::span<_CharT>>)
+	&& convertible_to<_Ros, std::span<const _CharT>>
+      explicit
+      basic_ispanstream(_Ros&& __s)
+      : __istream_type(std::__addressof(_M_sb)),
+	_M_sb(ios_base::in)
+      {
+	std::span<const _CharT> __sp(std::forward<_Ros>(__s));
+	_M_sb.span({const_cast<_CharT*>(__sp.data()), __sp.size()});
+      }
+
+    // [ispanstream.assign], assignment and swap
+    basic_ispanstream& operator=(const basic_ispanstream&) = delete;
+    basic_ispanstream& operator=(basic_ispanstream&& __rhs) = default;
+
+    void
+    swap(basic_ispanstream& __rhs)
+    {
+      __istream_type::swap(__rhs);
+      _M_sb.swap(__rhs._M_sb);
+    }
+
+    // [ispanstream.members], member functions
+    basic_spanbuf<_CharT, _Traits>*
+    rdbuf() const noexcept
+    {
+      return const_cast<basic_spanbuf<_CharT, _Traits>*>(std::__addressof(_M_sb));
+    }
+
+    std::span<const _CharT>
+    span() const noexcept
+    { return _M_sb.span(); }
+
+    void
+    span(std::span<_CharT> __s) noexcept
+    { return _M_sb.span(__s); }
+
+    template<typename _Ros>
+      requires ranges::borrowed_range<_Ros>
+	&& (!convertible_to<_Ros, std::span<_CharT>>)
+	&& convertible_to<_Ros, std::span<const _CharT>>
+      void
+      span(_Ros&& __s) noexcept
+      {
+	std::span<const _CharT> __sp(std::forward<_Ros>(__s));
+	_M_sb.span({const_cast<_CharT*>(__sp.data()), __sp.size()});
+      }
+
+  private:
+    basic_spanbuf<_CharT, _Traits> _M_sb;
+  };
+
+template<typename _CharT, typename _Traits>
+  inline void
+  swap(basic_ispanstream<_CharT, _Traits>& __x,
+       basic_ispanstream<_CharT, _Traits>& __y)
+  { __x.swap(__y); }
+
+using ispanstream = basic_ispanstream<char>;
+using wispanstream = basic_ispanstream<wchar_t>;
+
+template<typename _CharT, typename _Traits = char_traits<_CharT>>
+  class basic_ospanstream
+  : public basic_ostream<_CharT, _Traits>
+  {
+    using __ostream_type = basic_ostream<_CharT, _Traits>;
+
+  public:
+    using char_type   = _CharT;
+    using int_type    = typename _Traits::int_type;
+    using pos_type    = typename _Traits::pos_type;
+    using off_type    = typename _Traits::off_type;
+    using traits_type = _Traits;
+
+    // [ospanstream.ctor], constructors
+    explicit
+    basic_ospanstream(std::span<_CharT> __s,
+		      ios_base::openmode __which = ios_base::out)
+    : __ostream_type(std::__addressof(_M_sb)),
+      _M_sb(__s, __which | ios_base::in)
+    { }
+
+    basic_ospanstream(const basic_ospanstream&) = delete;
+
+    basic_ospanstream(basic_ospanstream&& __rhs)
+    : __ostream_type(std::move(__rhs)), _M_sb(std::move(__rhs._M_sb))
+    {
+      __ostream_type::set_rdbuf(std::addressof(_M_sb));
+    }
+
+    // [ospanstream.assign], assignment and swap
+    basic_ospanstream& operator=(const basic_ospanstream&) = delete;
+    basic_ospanstream& operator=(basic_ospanstream&& __rhs) = default;
+
+    void
+    swap(basic_ospanstream& __rhs)
+    {
+      __ostream_type::swap(__rhs);
+      _M_sb.swap(__rhs._M_sb);
+    }
+
+    // [ospanstream.members], member functions
+    basic_spanbuf<_CharT, _Traits>*
+    rdbuf() const noexcept
+    {
+      return const_cast<basic_spanbuf<_CharT, _Traits>*>(std::__addressof(_M_sb));
+    }
+
+    std::span<_CharT>
+    span() const noexcept
+    { return _M_sb.span(); }
+
+    void
+    span(std::span<_CharT> __s) noexcept
+    { return _M_sb.span(__s); }
+
+  private:
+    basic_spanbuf<_CharT, _Traits> _M_sb;
+  };
+
+template<typename _CharT, typename _Traits>
+  inline void
+  swap(basic_ospanstream<_CharT, _Traits>& __x,
+       basic_ospanstream<_CharT, _Traits>& __y)
+  { __x.swap(__y); }
+
+using ospanstream = basic_ospanstream<char>;
+using wospanstream = basic_ospanstream<wchar_t>;
+
+template<typename _CharT, typename _Traits = char_traits<_CharT>>
+  class basic_spanstream
+  : public basic_iostream<_CharT, _Traits>
+  {
+    using __iostream_type = basic_iostream<_CharT, _Traits>;
+
+  public:
+    using char_type   = _CharT;
+    using int_type    = typename _Traits::int_type;
+    using pos_type    = typename _Traits::pos_type;
+    using off_type    = typename _Traits::off_type;
+    using traits_type = _Traits;
+
+    // [spanstream.ctor], constructors
+    explicit
+    basic_spanstream(std::span<_CharT> __s,
+		     ios_base::openmode __which = ios_base::out | ios_base::in)
+    : __iostream_type(std::__addressof(_M_sb)),
+      _M_sb(__s, __which)
+    { }
+
+    basic_spanstream(const basic_spanstream&) = delete;
+
+    basic_spanstream(basic_spanstream&& __rhs)
+    : __iostream_type(std::move(__rhs)), _M_sb(std::move(__rhs._M_sb))
+    {
+      __iostream_type::set_rdbuf(std::addressof(_M_sb));
+    }
+
+    // [spanstream.assign], assignment and swap
+    basic_spanstream& operator=(const basic_spanstream&) = delete;
+    basic_spanstream& operator=(basic_spanstream&& __rhs) = default;
+
+    void
+    swap(basic_spanstream& __rhs)
+    {
+      __iostream_type::swap(__rhs);
+      _M_sb.swap(__rhs._M_sb);
+    }
+
+    // [spanstream.members], members
+    basic_spanbuf<_CharT, _Traits>*
+    rdbuf() const noexcept
+    {
+      return const_cast<basic_spanbuf<_CharT, _Traits>*>(std::__addressof(_M_sb));
+    }
+
+    std::span<_CharT>
+    span() const noexcept
+    { return _M_sb.span(); }
+
+    void
+    span(std::span<_CharT> __s) noexcept
+    { return _M_sb.span(__s); }
+
+  private:
+    basic_spanbuf<_CharT, _Traits> _M_sb;
+  };
+
+template<typename _CharT, typename _Traits>
+  inline void
+  swap(basic_spanstream<_CharT, _Traits>& __x,
+       basic_spanstream<_CharT, _Traits>& __y)
+  { __x.swap(__y); }
+
+using spanstream = basic_spanstream<char>;
+using wspanstream = basic_spanstream<wchar_t>;
+
+_GLIBCXX_END_NAMESPACE_VERSION
+} // namespace std
+#endif // __cpp_lib_span
+#endif // C++23
+#endif // _GLIBCXX_SPANSTREAM
diff --git a/libstdc++-v3/include/std/tuple b/libstdc++-v3/include/std/tuple
index aaee0b8..36dc05d 100644
--- a/libstdc++-v3/include/std/tuple
+++ b/libstdc++-v3/include/std/tuple
@@ -1344,36 +1344,23 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     struct tuple_size<tuple<_Elements...>>
     : public integral_constant<size_t, sizeof...(_Elements)> { };
 
-#if __cplusplus > 201402L
-  template <typename _Tp>
-    inline constexpr size_t tuple_size_v = tuple_size<_Tp>::value;
-#endif
+#if __cplusplus >= 201703L
+  template<typename... _Types>
+    inline constexpr size_t tuple_size_v<tuple<_Types...>>
+      = sizeof...(_Types);
 
-  /**
-   * Recursive case for tuple_element: strip off the first element in
-   * the tuple and retrieve the (i-1)th element of the remaining tuple.
-   */
-  template<size_t __i, typename _Head, typename... _Tail>
-    struct tuple_element<__i, tuple<_Head, _Tail...> >
-    : tuple_element<__i - 1, tuple<_Tail...> > { };
+  template<typename... _Types>
+    inline constexpr size_t tuple_size_v<const tuple<_Types...>>
+      = sizeof...(_Types);
+#endif
 
-  /**
-   * Basis case for tuple_element: The first element is the one we're seeking.
-   */
-  template<typename _Head, typename... _Tail>
-    struct tuple_element<0, tuple<_Head, _Tail...> >
+  /// Trait to get the Ith element type from a tuple.
+  template<size_t __i, typename... _Types>
+    struct tuple_element<__i, tuple<_Types...>>
     {
-      typedef _Head type;
-    };
+      static_assert(__i < sizeof...(_Types), "tuple index must be in range");
 
-  /**
-   * Error case for tuple_element: invalid index.
-   */
-  template<size_t __i>
-    struct tuple_element<__i, tuple<>>
-    {
-      static_assert(__i < tuple_size<tuple<>>::value,
-	  "tuple index must be in range");
+      using type = typename _Nth_type<__i, _Types...>::type;
     };
 
   template<size_t __i, typename _Head, typename... _Tail>
@@ -1432,28 +1419,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
 #define __cpp_lib_tuples_by_type 201304L
 
-  // Return the index of _Tp in _Types, if it occurs exactly once.
-  // Otherwise, return sizeof...(_Types).
-  // TODO reuse this for __detail::__variant::__exactly_once.
-  template<typename _Tp, typename... _Types>
-    constexpr size_t
-    __find_uniq_type_in_pack()
-    {
-      constexpr size_t __sz = sizeof...(_Types);
-      constexpr bool __found[__sz] = { __is_same(_Tp, _Types) ... };
-      size_t __n = __sz;
-      for (size_t __i = 0; __i < __sz; ++__i)
-	{
-	  if (__found[__i])
-	    {
-	      if (__n < __sz) // more than one _Tp found
-		return __sz;
-	      __n = __i;
-	    }
-	}
-      return __n;
-    }
-
   /// Return a reference to the unique element of type _Tp of a tuple.
   template <typename _Tp, typename... _Types>
     constexpr _Tp&
diff --git a/libstdc++-v3/include/std/valarray b/libstdc++-v3/include/std/valarray
index 5adc942..9c2f1be 100644
--- a/libstdc++-v3/include/std/valarray
+++ b/libstdc++-v3/include/std/valarray
@@ -137,7 +137,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
 	// _lib.valarray.cons_ construct/destroy:
       ///  Construct an empty array.
-      valarray();
+      valarray() _GLIBCXX_NOTHROW;
 
       ///  Construct an array with @a n elements.
       explicit valarray(size_t);
@@ -270,11 +270,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
        *  @param  __i  Index of element to return.
        *  @return  Reference to the i'th element.
        */
-      _Tp&                operator[](size_t __i);
+      _Tp&                operator[](size_t __i) _GLIBCXX_NOTHROW;
 
       // _GLIBCXX_RESOLVE_LIB_DEFECTS
       // 389. Const overload of valarray::operator[] returns by value.
-      const _Tp&          operator[](size_t) const;
+      const _Tp&          operator[](size_t) const _GLIBCXX_NOTHROW;
 
       // _lib.valarray.sub_ subset operations:
       /**
@@ -581,7 +581,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   template<typename _Tp>
     inline const _Tp&
-    valarray<_Tp>::operator[](size_t __i) const
+    valarray<_Tp>::operator[](size_t __i) const _GLIBCXX_NOTHROW
     {
       __glibcxx_requires_subscript(__i);
       return _M_data[__i];
@@ -589,7 +589,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   template<typename _Tp>
     inline _Tp&
-    valarray<_Tp>::operator[](size_t __i)
+    valarray<_Tp>::operator[](size_t __i) _GLIBCXX_NOTHROW
     {
       __glibcxx_requires_subscript(__i);
       return _M_data[__i];
@@ -618,7 +618,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   template<typename _Tp>
     inline
-    valarray<_Tp>::valarray() : _M_size(0), _M_data(0) {}
+    valarray<_Tp>::valarray() _GLIBCXX_NOTHROW : _M_size(0), _M_data(0) {}
 
   template<typename _Tp>
     inline
@@ -1210,9 +1210,10 @@ _DEFINE_BINARY_OPERATOR(>=, __greater_equal)
    *  @param  __va  valarray.
    */
   template<class _Tp>
+    [[__nodiscard__]]
     inline _Tp*
-    begin(valarray<_Tp>& __va)
-    { return std::__addressof(__va[0]); }
+    begin(valarray<_Tp>& __va) noexcept
+    { return __va.size() ? std::__addressof(__va[0]) : nullptr; }
 
   /**
    *  @brief  Return an iterator pointing to the first element of
@@ -1220,9 +1221,10 @@ _DEFINE_BINARY_OPERATOR(>=, __greater_equal)
    *  @param  __va  valarray.
    */
   template<class _Tp>
+    [[__nodiscard__]]
     inline const _Tp*
-    begin(const valarray<_Tp>& __va)
-    { return std::__addressof(__va[0]); }
+    begin(const valarray<_Tp>& __va) noexcept
+    { return __va.size() ? std::__addressof(__va[0]) : nullptr; }
 
   /**
    *  @brief  Return an iterator pointing to one past the last element of
@@ -1230,9 +1232,15 @@ _DEFINE_BINARY_OPERATOR(>=, __greater_equal)
    *  @param  __va  valarray.
    */
   template<class _Tp>
+    [[__nodiscard__]]
     inline _Tp*
-    end(valarray<_Tp>& __va)
-    { return std::__addressof(__va[0]) + __va.size(); }
+    end(valarray<_Tp>& __va) noexcept
+    {
+      if (auto __n = __va.size())
+	return std::__addressof(__va[0]) + __n;
+      else
+	return nullptr;
+    }
 
   /**
    *  @brief  Return an iterator pointing to one past the last element of
@@ -1240,9 +1248,15 @@ _DEFINE_BINARY_OPERATOR(>=, __greater_equal)
    *  @param  __va  valarray.
    */
   template<class _Tp>
+    [[__nodiscard__]]
     inline const _Tp*
-    end(const valarray<_Tp>& __va)
-    { return std::__addressof(__va[0]) + __va.size(); }
+    end(const valarray<_Tp>& __va) noexcept
+    {
+      if (auto __n = __va.size())
+	return std::__addressof(__va[0]) + __n;
+      else
+	return nullptr;
+    }
 #endif // C++11
 
   /// @} group numeric_arrays
diff --git a/libstdc++-v3/include/std/variant b/libstdc++-v3/include/std/variant
index 3da7dad..ab4503b 100644
--- a/libstdc++-v3/include/std/variant
+++ b/libstdc++-v3/include/std/variant
@@ -54,24 +54,6 @@ namespace std _GLIBCXX_VISIBILITY(default)
 {
 _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
-namespace __detail
-{
-namespace __variant
-{
-  template<size_t _Np, typename... _Types>
-    struct _Nth_type;
-
-  template<size_t _Np, typename _First, typename... _Rest>
-    struct _Nth_type<_Np, _First, _Rest...>
-    : _Nth_type<_Np-1, _Rest...> { };
-
-  template<typename _First, typename... _Rest>
-    struct _Nth_type<0, _First, _Rest...>
-    { using type = _First; };
-
-} // namespace __variant
-} // namespace __detail
-
 #if __cplusplus >= 202002L && __cpp_concepts
 // P2231R1 constexpr needs constexpr unions and constrained destructors.
 # define __cpp_lib_variant 202106L
@@ -102,16 +84,24 @@ namespace __variant
   template<typename _Variant>
     inline constexpr size_t variant_size_v = variant_size<_Variant>::value;
 
+  template<typename... _Types>
+    inline constexpr size_t
+    variant_size_v<variant<_Types...>> = sizeof...(_Types);
+
+  template<typename... _Types>
+    inline constexpr size_t
+    variant_size_v<const variant<_Types...>> = sizeof...(_Types);
+
   template<size_t _Np, typename _Variant>
     struct variant_alternative;
 
-  template<size_t _Np, typename _First, typename... _Rest>
-    struct variant_alternative<_Np, variant<_First, _Rest...>>
-    : variant_alternative<_Np-1, variant<_Rest...>> {};
+  template<size_t _Np, typename... _Types>
+    struct variant_alternative<_Np, variant<_Types...>>
+    {
+      static_assert(_Np < sizeof...(_Types));
 
-  template<typename _First, typename... _Rest>
-    struct variant_alternative<0, variant<_First, _Rest...>>
-    { using type = _First; };
+      using type = typename _Nth_type<_Np, _Types...>::type;
+    };
 
   template<size_t _Np, typename _Variant>
     using variant_alternative_t =
@@ -171,19 +161,6 @@ namespace __detail
 {
 namespace __variant
 {
-  // Returns the first appearance of _Tp in _Types.
-  // Returns sizeof...(_Types) if _Tp is not in _Types.
-  template<typename _Tp, typename... _Types>
-    struct __index_of : std::integral_constant<size_t, 0> {};
-
-  template<typename _Tp, typename... _Types>
-    inline constexpr size_t __index_of_v = __index_of<_Tp, _Types...>::value;
-
-  template<typename _Tp, typename _First, typename... _Rest>
-    struct __index_of<_Tp, _First, _Rest...> :
-      std::integral_constant<size_t, is_same_v<_Tp, _First>
-	? 0 : __index_of_v<_Tp, _Rest...> + 1> {};
-
   // used for raw visitation
   struct __variant_cookie {};
   // used for raw visitation with indices passed in
@@ -390,7 +367,13 @@ namespace __variant
 
   // Defines members and ctors.
   template<typename... _Types>
-    union _Variadic_union { };
+    union _Variadic_union
+    {
+      _Variadic_union() = default;
+
+      template<size_t _Np, typename... _Args>
+	_Variadic_union(in_place_index_t<_Np>, _Args&&...) = delete;
+    };
 
   template<typename _First, typename... _Rest>
     union _Variadic_union<_First, _Rest...>
@@ -467,10 +450,6 @@ namespace __variant
   template<typename... _Types>
     struct _Variant_storage<false, _Types...>
     {
-      template<typename _Tp>
-	static constexpr size_t __index_of
-	  = __detail::__variant::__index_of_v<_Tp, _Types...>;
-
       constexpr
       _Variant_storage()
       : _M_index(static_cast<__index_type>(variant_npos))
@@ -517,10 +496,6 @@ namespace __variant
   template<typename... _Types>
     struct _Variant_storage<true, _Types...>
     {
-      template<typename _Tp>
-	static constexpr size_t __index_of
-	  = __detail::__variant::__index_of_v<_Tp, _Types...>;
-
       constexpr
       _Variant_storage()
       : _M_index(static_cast<__index_type>(variant_npos))
@@ -556,38 +531,22 @@ namespace __variant
       __index_type _M_index;
     };
 
-  template<typename... _Types>
-    using _Variant_storage_alias =
-	_Variant_storage<_Traits<_Types...>::_S_trivial_dtor, _Types...>;
-
-  template<typename _Tp, typename _Up>
+  // Implementation of v.emplace<N>(args...).
+  template<size_t _Np, bool _Triv, typename... _Types, typename... _Args>
     _GLIBCXX20_CONSTEXPR
-    void
-    __variant_construct_single(_Tp&& __lhs, _Up&& __rhs_mem)
+    inline void
+    __emplace(_Variant_storage<_Triv, _Types...>& __v, _Args&&... __args)
     {
-      using _Type = __remove_cvref_t<_Up>;
-
-      if constexpr (!is_same_v<_Type, __variant_cookie>)
-	{
-	  using _Lhs = remove_reference_t<_Tp>;
-	  std::_Construct(std::__addressof(__lhs._M_u),
-			  in_place_index<_Lhs::template __index_of<_Type>>,
-			  std::forward<_Up>(__rhs_mem));
-	}
+      __v._M_reset();
+      auto* __addr = std::__addressof(__variant::__get_n<_Np>(__v._M_u));
+      std::_Construct(__addr, std::forward<_Args>(__args)...);
+      // Construction didn't throw, so can set the new index now:
+      __v._M_index = _Np;
     }
 
-  template<typename... _Types, typename _Tp, typename _Up>
-    _GLIBCXX20_CONSTEXPR
-    void
-    __variant_construct(_Tp&& __lhs, _Up&& __rhs)
-    {
-      __lhs._M_index = __rhs._M_index;
-      __variant::__raw_visit([&__lhs](auto&& __rhs_mem) mutable
-        {
-	  __variant_construct_single(std::forward<_Tp>(__lhs),
-	      std::forward<decltype(__rhs_mem)>(__rhs_mem));
-	}, __variant_cast<_Types...>(std::forward<_Up>(__rhs)));
-    }
+  template<typename... _Types>
+    using _Variant_storage_alias =
+	_Variant_storage<_Traits<_Types...>::_S_trivial_dtor, _Types...>;
 
   // The following are (Copy|Move) (ctor|assign) layers for forwarding
   // triviality and handling non-trivial SMF behaviors.
@@ -602,7 +561,15 @@ namespace __variant
       _Copy_ctor_base(const _Copy_ctor_base& __rhs)
 	  noexcept(_Traits<_Types...>::_S_nothrow_copy_ctor)
       {
-	__variant_construct<_Types...>(*this, __rhs);
+	__variant::__raw_idx_visit(
+	  [this](auto&& __rhs_mem, auto __rhs_index) mutable
+	  {
+	    constexpr size_t __j = __rhs_index;
+	    if constexpr (__j != variant_npos)
+	      std::_Construct(std::__addressof(this->_M_u),
+			      in_place_index<__j>, __rhs_mem);
+	  }, __variant_cast<_Types...>(__rhs));
+	this->_M_index = __rhs._M_index;
       }
 
       _Copy_ctor_base(_Copy_ctor_base&&) = default;
@@ -631,27 +598,18 @@ namespace __variant
       _Move_ctor_base(_Move_ctor_base&& __rhs)
 	  noexcept(_Traits<_Types...>::_S_nothrow_move_ctor)
       {
-	__variant_construct<_Types...>(*this, std::move(__rhs));
+	__variant::__raw_idx_visit(
+	  [this](auto&& __rhs_mem, auto __rhs_index) mutable
+	  {
+	    constexpr size_t __j = __rhs_index;
+	    if constexpr (__j != variant_npos)
+	      std::_Construct(std::__addressof(this->_M_u),
+			      in_place_index<__j>,
+			      std::forward<decltype(__rhs_mem)>(__rhs_mem));
+	  }, __variant_cast<_Types...>(std::move(__rhs)));
+	this->_M_index = __rhs._M_index;
       }
 
-      template<typename _Up>
-	_GLIBCXX20_CONSTEXPR
-        void _M_destructive_move(unsigned short __rhs_index, _Up&& __rhs)
-        {
-	  this->_M_reset();
-	  __variant_construct_single(*this, std::forward<_Up>(__rhs));
-	  this->_M_index = __rhs_index;
-	}
-
-      template<typename _Up>
-	_GLIBCXX20_CONSTEXPR
-        void _M_destructive_copy(unsigned short __rhs_index, const _Up& __rhs)
-        {
-	  this->_M_reset();
-	  __variant_construct_single(*this, __rhs);
-	  this->_M_index = __rhs_index;
-	}
-
       _Move_ctor_base(const _Move_ctor_base&) = default;
       _Move_ctor_base& operator=(const _Move_ctor_base&) = default;
       _Move_ctor_base& operator=(_Move_ctor_base&&) = default;
@@ -662,24 +620,6 @@ namespace __variant
     {
       using _Base = _Copy_ctor_alias<_Types...>;
       using _Base::_Base;
-
-      template<typename _Up>
-	_GLIBCXX20_CONSTEXPR
-        void _M_destructive_move(unsigned short __rhs_index, _Up&& __rhs)
-        {
-	  this->_M_reset();
-	  __variant_construct_single(*this, std::forward<_Up>(__rhs));
-	  this->_M_index = __rhs_index;
-	}
-
-      template<typename _Up>
-	_GLIBCXX20_CONSTEXPR
-        void _M_destructive_copy(unsigned short __rhs_index, const _Up& __rhs)
-        {
-	  this->_M_reset();
-	  __variant_construct_single(*this, __rhs);
-	  this->_M_index = __rhs_index;
-	}
     };
 
   template<typename... _Types>
@@ -700,31 +640,24 @@ namespace __variant
 	__variant::__raw_idx_visit(
 	  [this](auto&& __rhs_mem, auto __rhs_index) mutable
 	  {
-	    if constexpr (__rhs_index != variant_npos)
+	    constexpr size_t __j = __rhs_index;
+	    if constexpr (__j == variant_npos)
+	      this->_M_reset(); // Make *this valueless.
+	    else if (this->_M_index == __j)
+	      __variant::__get<__j>(*this) = __rhs_mem;
+	    else
 	      {
-		if (this->_M_index == __rhs_index)
-		  __variant::__get<__rhs_index>(*this) = __rhs_mem;
+		using _Tj = typename _Nth_type<__j, _Types...>::type;
+		if constexpr (is_nothrow_copy_constructible_v<_Tj>
+			      || !is_nothrow_move_constructible_v<_Tj>)
+		  __variant::__emplace<__j>(*this, __rhs_mem);
 		else
 		  {
-		    using __rhs_type = __remove_cvref_t<decltype(__rhs_mem)>;
-		    if constexpr (is_nothrow_copy_constructible_v<__rhs_type>
-			|| !is_nothrow_move_constructible_v<__rhs_type>)
-		      // The standard says this->emplace<__rhs_type>(__rhs_mem)
-		      // should be used here, but _M_destructive_copy is
-		      // equivalent in this case. Either copy construction
-		      // doesn't throw, so _M_destructive_copy gives strong
-		      // exception safety guarantee, or both copy construction
-		      // and move construction can throw, so emplace only gives
-		      // basic exception safety anyway.
-		      this->_M_destructive_copy(__rhs_index, __rhs_mem);
-		    else
-		      __variant_cast<_Types...>(*this)
-			= variant<_Types...>(std::in_place_index<__rhs_index>,
-					     __rhs_mem);
+		    using _Variant = variant<_Types...>;
+		    _Variant& __self = __variant_cast<_Types...>(*this);
+		    __self = _Variant(in_place_index<__j>, __rhs_mem);
 		  }
 	      }
-	    else
-	      this->_M_reset();
 	  }, __variant_cast<_Types...>(__rhs));
 	return *this;
       }
@@ -759,13 +692,23 @@ namespace __variant
 	__variant::__raw_idx_visit(
 	  [this](auto&& __rhs_mem, auto __rhs_index) mutable
 	  {
-	    if constexpr (__rhs_index != variant_npos)
+	    constexpr size_t __j = __rhs_index;
+	    if constexpr (__j != variant_npos)
 	      {
-		if (this->_M_index == __rhs_index)
-		  __variant::__get<__rhs_index>(*this) = std::move(__rhs_mem);
+		if (this->_M_index == __j)
+		  __variant::__get<__j>(*this) = std::move(__rhs_mem);
 		else
-		  __variant_cast<_Types...>(*this)
-		    .template emplace<__rhs_index>(std::move(__rhs_mem));
+		  {
+		    using _Tj = typename _Nth_type<__j, _Types...>::type;
+		    if constexpr (is_nothrow_move_constructible_v<_Tj>)
+		      __variant::__emplace<__j>(*this, std::move(__rhs_mem));
+		    else
+		      {
+			using _Variant = variant<_Types...>;
+			_Variant& __self = __variant_cast<_Types...>(*this);
+			__self.template emplace<__j>(std::move(__rhs_mem));
+		      }
+		  }
 	      }
 	    else
 	      this->_M_reset();
@@ -810,28 +753,9 @@ namespace __variant
       _Variant_base& operator=(_Variant_base&&) = default;
     };
 
-  // For how many times does _Tp appear in _Tuple?
-  template<typename _Tp, typename _Tuple>
-    struct __tuple_count;
-
-  template<typename _Tp, typename _Tuple>
-    inline constexpr size_t __tuple_count_v =
-      __tuple_count<_Tp, _Tuple>::value;
-
-  template<typename _Tp, typename... _Types>
-    struct __tuple_count<_Tp, tuple<_Types...>>
-    : integral_constant<size_t, 0> { };
-
-  template<typename _Tp, typename _First, typename... _Rest>
-    struct __tuple_count<_Tp, tuple<_First, _Rest...>>
-    : integral_constant<
-	size_t,
-	__tuple_count_v<_Tp, tuple<_Rest...>> + is_same_v<_Tp, _First>> { };
-
-  // TODO: Reuse this in <tuple> ?
   template<typename _Tp, typename... _Types>
-    inline constexpr bool __exactly_once =
-      __tuple_count_v<_Tp, tuple<_Types...>> == 1;
+    inline constexpr bool __exactly_once
+      = std::__find_uniq_type_in_pack<_Tp, _Types...>() < sizeof...(_Types);
 
   // Helper used to check for valid conversions that don't involve narrowing.
   template<typename _Ti> struct _Arr { _Ti _M_x[1]; };
@@ -1181,17 +1105,6 @@ namespace __variant
 	>;
     }
 
-  template<size_t _Np, typename _Variant, typename... _Args>
-    _GLIBCXX20_CONSTEXPR
-    inline void
-    __construct_by_index(_Variant& __v, _Args&&... __args)
-    {
-      std::_Construct(std::__addressof(__variant::__get<_Np>(__v)),
-		      std::forward<_Args>(__args)...);
-      // Construction didn't throw, so can set the new index now:
-      __v._M_index = _Np;
-    }
-
 } // namespace __variant
 } // namespace __detail
 
@@ -1201,45 +1114,51 @@ namespace __variant
     {
       static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
 		    "T must occur exactly once in alternatives");
-      return __v.index() == __detail::__variant::__index_of_v<_Tp, _Types...>;
+      return __v.index() == std::__find_uniq_type_in_pack<_Tp, _Types...>();
     }
 
   template<typename _Tp, typename... _Types>
-    constexpr _Tp& get(variant<_Types...>& __v)
+    constexpr _Tp&
+    get(variant<_Types...>& __v)
     {
       static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
 		    "T must occur exactly once in alternatives");
       static_assert(!is_void_v<_Tp>, "_Tp must not be void");
-      return std::get<__detail::__variant::__index_of_v<_Tp, _Types...>>(__v);
+      constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
+      return std::get<__n>(__v);
     }
 
   template<typename _Tp, typename... _Types>
-    constexpr _Tp&& get(variant<_Types...>&& __v)
+    constexpr _Tp&&
+    get(variant<_Types...>&& __v)
     {
       static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
 		    "T must occur exactly once in alternatives");
       static_assert(!is_void_v<_Tp>, "_Tp must not be void");
-      return std::get<__detail::__variant::__index_of_v<_Tp, _Types...>>(
-	std::move(__v));
+      constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
+      return std::get<__n>(std::move(__v));
     }
 
   template<typename _Tp, typename... _Types>
-    constexpr const _Tp& get(const variant<_Types...>& __v)
+    constexpr const _Tp&
+    get(const variant<_Types...>& __v)
     {
       static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
 		    "T must occur exactly once in alternatives");
       static_assert(!is_void_v<_Tp>, "_Tp must not be void");
-      return std::get<__detail::__variant::__index_of_v<_Tp, _Types...>>(__v);
+      constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
+      return std::get<__n>(__v);
     }
 
   template<typename _Tp, typename... _Types>
-    constexpr const _Tp&& get(const variant<_Types...>&& __v)
+    constexpr const _Tp&&
+    get(const variant<_Types...>&& __v)
     {
       static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
 		    "T must occur exactly once in alternatives");
       static_assert(!is_void_v<_Tp>, "_Tp must not be void");
-      return std::get<__detail::__variant::__index_of_v<_Tp, _Types...>>(
-	std::move(__v));
+      constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
+      return std::get<__n>(std::move(__v));
     }
 
   template<size_t _Np, typename... _Types>
@@ -1276,8 +1195,8 @@ namespace __variant
       static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
 		    "T must occur exactly once in alternatives");
       static_assert(!is_void_v<_Tp>, "_Tp must not be void");
-      return std::get_if<__detail::__variant::__index_of_v<_Tp, _Types...>>(
-	  __ptr);
+      constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
+      return std::get_if<__n>(__ptr);
     }
 
   template<typename _Tp, typename... _Types>
@@ -1287,8 +1206,8 @@ namespace __variant
       static_assert(__detail::__variant::__exactly_once<_Tp, _Types...>,
 		    "T must occur exactly once in alternatives");
       static_assert(!is_void_v<_Tp>, "_Tp must not be void");
-      return std::get_if<__detail::__variant::__index_of_v<_Tp, _Types...>>(
-	  __ptr);
+      constexpr size_t __n = std::__find_uniq_type_in_pack<_Tp, _Types...>();
+      return std::get_if<__n>(__ptr);
     }
 
   struct monostate { };
@@ -1432,11 +1351,6 @@ namespace __variant
 	friend _GLIBCXX20_CONSTEXPR decltype(auto)
 	__variant_cast(_Tp&&);
 
-      template<size_t _Np, typename _Variant, typename... _Args>
-	friend _GLIBCXX20_CONSTEXPR void
-	__detail::__variant::__construct_by_index(_Variant& __v,
-						  _Args&&... __args);
-
       static_assert(sizeof...(_Types) > 0,
 		    "variant must have at least one alternative");
       static_assert(!(std::is_reference_v<_Types> || ...),
@@ -1463,14 +1377,14 @@ namespace __variant
 	  = __detail::__variant::__accepted_index<_Tp, variant>::value;
 
       template<size_t _Np, typename = enable_if_t<(_Np < sizeof...(_Types))>>
-	using __to_type = variant_alternative_t<_Np, variant>;
+	using __to_type = typename _Nth_type<_Np, _Types...>::type;
 
       template<typename _Tp, typename = enable_if_t<__not_self<_Tp>>>
 	using __accepted_type = __to_type<__accepted_index<_Tp>>;
 
       template<typename _Tp>
-	static constexpr size_t __index_of =
-	  __detail::__variant::__index_of_v<_Tp, _Types...>;
+	static constexpr size_t __index_of
+	  = std::__find_uniq_type_in_pack<_Tp, _Types...>();
 
       using _Traits = __detail::__variant::_Traits<_Types...>;
 
@@ -1595,30 +1509,24 @@ namespace __variant
 
       template<size_t _Np, typename... _Args>
 	_GLIBCXX20_CONSTEXPR
-	enable_if_t<is_constructible_v<variant_alternative_t<_Np, variant>,
-				       _Args...>,
-		    variant_alternative_t<_Np, variant>&>
+	enable_if_t<is_constructible_v<__to_type<_Np>, _Args...>,
+		    __to_type<_Np>&>
 	emplace(_Args&&... __args)
 	{
-	  static_assert(_Np < sizeof...(_Types),
-			"The index must be in [0, number of alternatives)");
-	  using type = variant_alternative_t<_Np, variant>;
 	  namespace __variant = std::__detail::__variant;
+	  using type = typename _Nth_type<_Np, _Types...>::type;
 	  // Provide the strong exception-safety guarantee when possible,
 	  // to avoid becoming valueless.
 	  if constexpr (is_nothrow_constructible_v<type, _Args...>)
 	    {
-	      this->_M_reset();
-	      __variant::__construct_by_index<_Np>(*this,
-		  std::forward<_Args>(__args)...);
+	      __variant::__emplace<_Np>(*this, std::forward<_Args>(__args)...);
 	    }
 	  else if constexpr (is_scalar_v<type>)
 	    {
 	      // This might invoke a potentially-throwing conversion operator:
 	      const type __tmp(std::forward<_Args>(__args)...);
-	      // But these steps won't throw:
-	      this->_M_reset();
-	      __variant::__construct_by_index<_Np>(*this, __tmp);
+	      // But this won't throw:
+	      __variant::__emplace<_Np>(*this, __tmp);
 	    }
 	  else if constexpr (__variant::_Never_valueless_alt<type>()
 	      && _Traits::_S_move_assign)
@@ -1633,33 +1541,28 @@ namespace __variant
 	    {
 	      // This case only provides the basic exception-safety guarantee,
 	      // i.e. the variant can become valueless.
-	      this->_M_reset();
-	      __variant::__construct_by_index<_Np>(*this,
-		std::forward<_Args>(__args)...);
+	      __variant::__emplace<_Np>(*this, std::forward<_Args>(__args)...);
 	    }
 	  return std::get<_Np>(*this);
 	}
 
       template<size_t _Np, typename _Up, typename... _Args>
 	_GLIBCXX20_CONSTEXPR
-	enable_if_t<is_constructible_v<variant_alternative_t<_Np, variant>,
+	enable_if_t<is_constructible_v<__to_type<_Np>,
 				       initializer_list<_Up>&, _Args...>,
-		    variant_alternative_t<_Np, variant>&>
+		    __to_type<_Np>&>
 	emplace(initializer_list<_Up> __il, _Args&&... __args)
 	{
-	  static_assert(_Np < sizeof...(_Types),
-			"The index must be in [0, number of alternatives)");
-	  using type = variant_alternative_t<_Np, variant>;
 	  namespace __variant = std::__detail::__variant;
+	  using type = typename _Nth_type<_Np, _Types...>::type;
 	  // Provide the strong exception-safety guarantee when possible,
 	  // to avoid becoming valueless.
 	  if constexpr (is_nothrow_constructible_v<type,
 						   initializer_list<_Up>&,
 						   _Args...>)
 	    {
-	      this->_M_reset();
-	      __variant::__construct_by_index<_Np>(*this, __il,
-		  std::forward<_Args>(__args)...);
+	      __variant::__emplace<_Np>(*this, __il,
+					std::forward<_Args>(__args)...);
 	    }
 	  else if constexpr (__variant::_Never_valueless_alt<type>()
 	      && _Traits::_S_move_assign)
@@ -1674,13 +1577,18 @@ namespace __variant
 	    {
 	      // This case only provides the basic exception-safety guarantee,
 	      // i.e. the variant can become valueless.
-	      this->_M_reset();
-	      __variant::__construct_by_index<_Np>(*this, __il,
-		std::forward<_Args>(__args)...);
+	      __variant::__emplace<_Np>(*this, __il,
+					std::forward<_Args>(__args)...);
 	    }
 	  return std::get<_Np>(*this);
 	}
 
+      template<size_t _Np, typename... _Args>
+	enable_if_t<!(_Np < sizeof...(_Types))> emplace(_Args&&...) = delete;
+
+      template<typename _Tp, typename... _Args>
+	enable_if_t<!__exactly_once<_Tp>> emplace(_Args&&...) = delete;
+
       constexpr bool valueless_by_exception() const noexcept
       { return !this->_M_valid(); }
 
@@ -1701,54 +1609,57 @@ namespace __variant
       noexcept((__is_nothrow_swappable<_Types>::value && ...)
 	       && is_nothrow_move_constructible_v<variant>)
       {
-	__detail::__variant::__raw_idx_visit(
+	static_assert((is_move_constructible_v<_Types> && ...));
+
+	// Handle this here to simplify the visitation.
+	if (__rhs.valueless_by_exception()) [[__unlikely__]]
+	  {
+	    if (!this->valueless_by_exception()) [[__likely__]]
+	      __rhs.swap(*this);
+	    return;
+	  }
+
+	namespace __variant = __detail::__variant;
+
+	__variant::__raw_idx_visit(
 	  [this, &__rhs](auto&& __rhs_mem, auto __rhs_index) mutable
 	  {
-	    if constexpr (__rhs_index != variant_npos)
+	    constexpr size_t __j = __rhs_index;
+	    if constexpr (__j != variant_npos)
 	      {
-		if (this->index() == __rhs_index)
+		if (this->index() == __j)
 		  {
-		    auto& __this_mem =
-		      std::get<__rhs_index>(*this);
 		    using std::swap;
-		    swap(__this_mem, __rhs_mem);
+		    swap(std::get<__j>(*this), __rhs_mem);
 		  }
 		else
 		  {
-		    if (!this->valueless_by_exception()) [[__likely__]]
-		      {
-			auto __tmp(std::move(__rhs_mem));
-			__rhs = std::move(*this);
-			this->_M_destructive_move(__rhs_index,
-						  std::move(__tmp));
-		      }
+		    auto __tmp(std::move(__rhs_mem));
+
+		    if constexpr (_Traits::_S_trivial_move_assign)
+		      __rhs = std::move(*this);
 		    else
-		      {
-			this->_M_destructive_move(__rhs_index,
-						  std::move(__rhs_mem));
-			__rhs._M_reset();
-		      }
-		  }
-	      }
-	    else
-	      {
-		if (!this->valueless_by_exception()) [[__likely__]]
-		  {
-		    __rhs = std::move(*this);
-		    this->_M_reset();
+		      __variant::__raw_idx_visit(
+			[&__rhs](auto&& __this_mem, auto __this_index) mutable
+			{
+			  constexpr size_t __k = __this_index;
+			  if constexpr (__k != variant_npos)
+			    __variant::__emplace<__k>(__rhs,
+						      std::move(__this_mem));
+			}, *this);
+
+		    __variant::__emplace<__j>(*this, std::move(__tmp));
 		  }
 	      }
 	  }, __rhs);
       }
 
-    private:
-
 #if defined(__clang__) && __clang_major__ <= 7
     public:
       using _Base::_M_u; // See https://bugs.llvm.org/show_bug.cgi?id=31852
-    private:
 #endif
 
+    private:
       template<size_t _Np, typename _Vp>
 	friend constexpr decltype(auto)
 	__detail::__variant::__get(_Vp&& __v) noexcept;
@@ -1826,8 +1737,7 @@ namespace __variant
 	  constexpr size_t __max = 11; // "These go to eleven."
 
 	  // The type of the first variant in the pack.
-	  using _V0
-	    = typename __detail::__variant::_Nth_type<0, _Variants...>::type;
+	  using _V0 = typename _Nth_type<0, _Variants...>::type;
 	  // The number of alternatives in that first variant.
 	  constexpr auto __n = variant_size_v<remove_reference_t<_V0>>;
 
diff --git a/libstdc++-v3/include/std/vector b/libstdc++-v3/include/std/vector
index 096511c..890b0dd 100644
--- a/libstdc++-v3/include/std/vector
+++ b/libstdc++-v3/include/std/vector
@@ -102,6 +102,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #define __cpp_lib_erase_if 202002L
 
   template<typename _Tp, typename _Alloc, typename _Predicate>
+    _GLIBCXX20_CONSTEXPR
     inline typename vector<_Tp, _Alloc>::size_type
     erase_if(vector<_Tp, _Alloc>& __cont, _Predicate __pred)
     {
@@ -115,6 +116,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     }
 
   template<typename _Tp, typename _Alloc, typename _Up>
+    _GLIBCXX20_CONSTEXPR
     inline typename vector<_Tp, _Alloc>::size_type
     erase(vector<_Tp, _Alloc>& __cont, const _Up& __value)
     {
diff --git a/libstdc++-v3/include/std/version b/libstdc++-v3/include/std/version
index 0a7b28a..0930de8 100644
--- a/libstdc++-v3/include/std/version
+++ b/libstdc++-v3/include/std/version
@@ -296,6 +296,9 @@
 # define __cpp_lib_monadic_optional 202110L
 #endif
 #define __cpp_lib_move_only_function 202110L
+#if __cpp_lib_span
+# define __cpp_lib_spanstream 202106L
+#endif
 #define __cpp_lib_string_contains 202011L
 #if _GLIBCXX_USE_CXX11_ABI // Only supported with cxx11-abi
 # define __cpp_lib_string_resize_and_overwrite 202110L
diff --git a/libstdc++-v3/libsupc++/eh_personality.cc b/libstdc++-v3/libsupc++/eh_personality.cc
index 33c3cc3..edd5f6a 100644
--- a/libstdc++-v3/libsupc++/eh_personality.cc
+++ b/libstdc++-v3/libsupc++/eh_personality.cc
@@ -673,10 +673,13 @@ PERSONALITY_FUNCTION (int version,
 	std::terminate ();
       else if (handler_switch_value < 0)
 	{
-	  __try 
-	    { std::unexpected (); } 
-	  __catch(...) 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+	  __try
+	    { std::unexpected (); }
+	  __catch(...)
 	    { std::terminate (); }
+#pragma GCC diagnostic pop
 	}
     }
   else
diff --git a/libstdc++-v3/libsupc++/eh_ptr.cc b/libstdc++-v3/libsupc++/eh_ptr.cc
index 5c46856..9f47b666 100644
--- a/libstdc++-v3/libsupc++/eh_ptr.cc
+++ b/libstdc++-v3/libsupc++/eh_ptr.cc
@@ -198,7 +198,10 @@ std::rethrow_exception(std::exception_ptr ep)
   dep->primaryException = obj;
   __gnu_cxx::__eh_atomic_inc (&eh->referenceCount);
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
   dep->unexpectedHandler = get_unexpected ();
+#pragma GCC diagnostic pop
   dep->terminateHandler = get_terminate ();
   __GXX_INIT_DEPENDENT_EXCEPTION_CLASS(dep->unwindHeader.exception_class);
   dep->unwindHeader.exception_cleanup = __gxx_dependent_exception_cleanup;
diff --git a/libstdc++-v3/libsupc++/eh_terminate.cc b/libstdc++-v3/libsupc++/eh_terminate.cc
index af257b6..a94f173 100644
--- a/libstdc++-v3/libsupc++/eh_terminate.cc
+++ b/libstdc++-v3/libsupc++/eh_terminate.cc
@@ -58,6 +58,7 @@ std::terminate () throw()
   __cxxabiv1::__terminate (get_terminate ());
 }
 
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 void
 __cxxabiv1::__unexpected (std::unexpected_handler handler)
 {
diff --git a/libstdc++-v3/libsupc++/eh_throw.cc b/libstdc++-v3/libsupc++/eh_throw.cc
index 765a6c1..51cbbc9 100644
--- a/libstdc++-v3/libsupc++/eh_throw.cc
+++ b/libstdc++-v3/libsupc++/eh_throw.cc
@@ -63,7 +63,10 @@ _GLIBCXX_NOTHROW
   header->referenceCount = 0;
   header->exc.exceptionType = tinfo;
   header->exc.exceptionDestructor = dest;
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
   header->exc.unexpectedHandler = std::get_unexpected ();
+#pragma GCC diagnostic pop
   header->exc.terminateHandler = std::get_terminate ();
   __GXX_INIT_PRIMARY_EXCEPTION_CLASS(header->exc.unwindHeader.exception_class);
   header->exc.unwindHeader.exception_cleanup = __gxx_exception_cleanup;
diff --git a/libstdc++-v3/libsupc++/exception b/libstdc++-v3/libsupc++/exception
index a023e22..5088681 100644
--- a/libstdc++-v3/libsupc++/exception
+++ b/libstdc++-v3/libsupc++/exception
@@ -67,9 +67,6 @@ namespace std
   /// If you write a replacement %terminate handler, it must be of this type.
   typedef void (*terminate_handler) ();
 
-  /// If you write a replacement %unexpected handler, it must be of this type.
-  typedef void (*unexpected_handler) ();
-
   /// Takes a new handler function as an argument, returns the old function.
   terminate_handler set_terminate(terminate_handler) _GLIBCXX_USE_NOEXCEPT;
 
@@ -82,17 +79,35 @@ namespace std
    *  abandoned for any reason.  It can also be called by the user.  */
   void terminate() _GLIBCXX_USE_NOEXCEPT __attribute__ ((__noreturn__));
 
-  /// Takes a new handler function as an argument, returns the old function.
+#if __cplusplus < 201703L || _GLIBCXX_USE_DEPRECATED
+  /// If you write a replacement %unexpected handler, it must be of this type.
+  typedef void (*_GLIBCXX11_DEPRECATED unexpected_handler) ();
+
+  /** Takes a new handler function as an argument, returns the old function.
+   *
+   * @deprecated Removed from the C++ standard in C++17
+   */
+  _GLIBCXX11_DEPRECATED
   unexpected_handler set_unexpected(unexpected_handler) _GLIBCXX_USE_NOEXCEPT;
 
 #if __cplusplus >= 201103L
-  /// Return the current unexpected handler.
+  /** Return the current unexpected handler.
+   *
+   * @since C++11
+   * @deprecated Removed from the C++ standard in C++17
+   */
+  _GLIBCXX11_DEPRECATED
   unexpected_handler get_unexpected() noexcept;
 #endif
 
   /** The runtime will call this function if an %exception is thrown which
-   *  violates the function's %exception specification.  */
+   *  violates the function's %exception specification.
+   *
+   * @deprecated Removed from the C++ standard in C++17
+   */
+  _GLIBCXX11_DEPRECATED
   void unexpected() __attribute__ ((__noreturn__));
+#endif
 
   /** [18.6.4]/1:  'Returns true after completing evaluation of a
    *  throw-expression until either completing initialization of the
diff --git a/libstdc++-v3/libsupc++/unwind-cxx.h b/libstdc++-v3/libsupc++/unwind-cxx.h
index 65fd530..96d50ea 100644
--- a/libstdc++-v3/libsupc++/unwind-cxx.h
+++ b/libstdc++-v3/libsupc++/unwind-cxx.h
@@ -67,7 +67,7 @@ struct __cxa_exception
 
   // The C++ standard has entertaining rules wrt calling set_terminate
   // and set_unexpected in the middle of the exception cleanup process.
-  std::unexpected_handler unexpectedHandler;
+  std::terminate_handler unexpectedHandler;
   std::terminate_handler terminateHandler;
 
   // The caught exception stack threads through here.
@@ -121,7 +121,7 @@ struct __cxa_dependent_exception
 
   // The C++ standard has entertaining rules wrt calling set_terminate
   // and set_unexpected in the middle of the exception cleanup process.
-  std::unexpected_handler unexpectedHandler;
+  std::terminate_handler unexpectedHandler;
   std::terminate_handler terminateHandler;
 
   // The caught exception stack threads through here.
@@ -191,12 +191,12 @@ extern "C" void __cxa_tm_cleanup (void *, void *, unsigned int) throw();
 // so inconsiderate as to return.
 extern void __terminate(std::terminate_handler) throw () 
   __attribute__((__noreturn__));
-extern void __unexpected(std::unexpected_handler)
+extern void __unexpected(std::terminate_handler)
   __attribute__((__noreturn__));
 
 // The current installed user handlers.
 extern std::terminate_handler __terminate_handler;
-extern std::unexpected_handler __unexpected_handler;
+extern std::terminate_handler __unexpected_handler;
 
 // These are explicitly GNU C++ specific.
 
diff --git a/libstdc++-v3/python/libstdcxx/v6/printers.py b/libstdc++-v3/python/libstdcxx/v6/printers.py
index c7da407..c5072c5 100644
--- a/libstdc++-v3/python/libstdcxx/v6/printers.py
+++ b/libstdc++-v3/python/libstdcxx/v6/printers.py
@@ -240,32 +240,63 @@ class SharedPointerPrinter:
                 state = 'use count %d, weak count %d' % (usecount, weakcount - 1)
         return '%s<%s> (%s)' % (self.typename, str(self.val.type.template_argument(0)), state)
 
+def _tuple_impl_get(val):
+    "Return the tuple element stored in a _Tuple_impl<N, T> base class."
+    bases = val.type.fields()
+    if not bases[-1].is_base_class:
+        raise ValueError("Unsupported implementation for std::tuple: %s" % str(val.type))
+    # Get the _Head_base<N, T> base class:
+    head_base = val.cast(bases[-1].type)
+    fields = head_base.type.fields()
+    if len(fields) == 0:
+        raise ValueError("Unsupported implementation for std::tuple: %s" % str(val.type))
+    if fields[0].name == '_M_head_impl':
+        # The tuple element is the _Head_base::_M_head_impl data member.
+        return head_base['_M_head_impl']
+    elif fields[0].is_base_class:
+        # The tuple element is an empty base class of _Head_base.
+        # Cast to that empty base class.
+        return head_base.cast(fields[0].type)
+    else:
+        raise ValueError("Unsupported implementation for std::tuple: %s" % str(val.type))
+
+def tuple_get(n, val):
+    "Return the result of std::get<n>(val) on a std::tuple"
+    tuple_size = len(get_template_arg_list(val.type))
+    if n > tuple_size:
+        raise ValueError("Out of range index for std::get<N> on std::tuple")
+    # Get the first _Tuple_impl<0, T...> base class:
+    node = val.cast(val.type.fields()[0].type)
+    while n > 0:
+        # Descend through the base classes until the Nth one.
+        node = node.cast(node.type.fields()[0].type)
+        n -= 1
+    return _tuple_impl_get(node)
+
+def unique_ptr_get(val):
+    "Return the result of val.get() on a std::unique_ptr"
+    # std::unique_ptr<T, D> contains a std::tuple<D::pointer, D>,
+    # either as a direct data member _M_t (the old implementation)
+    # or within a data member of type __uniq_ptr_data.
+    impl_type = val.type.fields()[0].type.strip_typedefs()
+    # Check for new implementations first:
+    if is_specialization_of(impl_type, '__uniq_ptr_data') \
+        or is_specialization_of(impl_type, '__uniq_ptr_impl'):
+        tuple_member = val['_M_t']['_M_t']
+    elif is_specialization_of(impl_type, 'tuple'):
+        tuple_member = val['_M_t']
+    else:
+        raise ValueError("Unsupported implementation for unique_ptr: %s" % str(impl_type))
+    return tuple_get(0, tuple_member)
+
 class UniquePointerPrinter:
     "Print a unique_ptr"
 
     def __init__ (self, typename, val):
         self.val = val
-        impl_type = val.type.fields()[0].type.strip_typedefs()
-        # Check for new implementations first:
-        if is_specialization_of(impl_type, '__uniq_ptr_data') \
-            or is_specialization_of(impl_type, '__uniq_ptr_impl'):
-            tuple_member = val['_M_t']['_M_t']
-        elif is_specialization_of(impl_type, 'tuple'):
-            tuple_member = val['_M_t']
-        else:
-            raise ValueError("Unsupported implementation for unique_ptr: %s" % str(impl_type))
-        tuple_impl_type = tuple_member.type.fields()[0].type # _Tuple_impl
-        tuple_head_type = tuple_impl_type.fields()[1].type   # _Head_base
-        head_field = tuple_head_type.fields()[0]
-        if head_field.name == '_M_head_impl':
-            self.pointer = tuple_member['_M_head_impl']
-        elif head_field.is_base_class:
-            self.pointer = tuple_member.cast(head_field.type)
-        else:
-            raise ValueError("Unsupported implementation for tuple in unique_ptr: %s" % str(impl_type))
 
     def children (self):
-        return SmartPtrIterator(self.pointer)
+        return SmartPtrIterator(unique_ptr_get(self.val))
 
     def to_string (self):
         return ('std::unique_ptr<%s>' % (str(self.val.type.template_argument(0))))
@@ -1370,7 +1401,7 @@ class StdPathPrinter:
     def __init__ (self, typename, val):
         self.val = val
         self.typename = typename
-        impl = self.val['_M_cmpts']['_M_impl']['_M_t']['_M_t']['_M_head_impl']
+        impl = unique_ptr_get(self.val['_M_cmpts']['_M_impl'])
         self.type = impl.cast(gdb.lookup_type('uintptr_t')) & 3
         if self.type == 0:
             self.impl = impl
diff --git a/libstdc++-v3/python/libstdcxx/v6/xmethods.py b/libstdc++-v3/python/libstdcxx/v6/xmethods.py
index 77870e9..991d945 100644
--- a/libstdc++-v3/python/libstdcxx/v6/xmethods.py
+++ b/libstdc++-v3/python/libstdcxx/v6/xmethods.py
@@ -597,7 +597,7 @@ class UniquePtrGetWorker(gdb.xmethod.XMethodWorker):
         tuple_head_type = tuple_impl_type.fields()[1].type   # _Head_base
         head_field = tuple_head_type.fields()[0]
         if head_field.name == '_M_head_impl':
-            return tuple_member['_M_head_impl']
+            return tuple_member.cast(tuple_head_type)['_M_head_impl']
         elif head_field.is_base_class:
             return tuple_member.cast(head_field.type)
         else:
diff --git a/libstdc++-v3/scripts/run_doxygen b/libstdc++-v3/scripts/run_doxygen
index a367721..6236a07 100644
--- a/libstdc++-v3/scripts/run_doxygen
+++ b/libstdc++-v3/scripts/run_doxygen
@@ -269,6 +269,8 @@ find . -name "* *" -print0 | xargs -0r rm        # requires GNU tools
 
 # man pages are for functions/types/other entities, not source files
 # directly.  who the heck would type "man foo.h" anyhow?
+# FIXME: This also removes std.3 which is the only place that a lot of
+# functions are documented. Should we keep it?
 find . -name "[a-z]*" -a ! -name "std_*" -print | xargs rm
 rm -f *.h.3 *.hpp.3 *config* *.cc.3 *.tcc.3 *_t.3
 #rm ext_*.3 tr1_*.3 debug_*.3
@@ -278,18 +280,23 @@ rm -f *.h.3 *.hpp.3 *config* *.cc.3 *.tcc.3 *_t.3
 #find . -name "[a-z]*" -a ! -name "std_*" -print | xargs -i mv {} trash
 #mv *.h.3 *config* *.cc.3 *.tcc.3 *_t.3  trash
 
+gxx=$($builddir/scripts/testsuite_flags --build-cxx)
+cppflags=$($builddir/scripts/testsuite_flags --build-includes)
+cxxflags="-Og -g -std=gnu++23"
+
 # Standardize the displayed header names.  If anyone who knows perl cares
 # enough to rewrite all this, feel free.  This only gets run once a century,
 # and I'm off getting coffee then anyhow, so I didn't care enough to make
 # this super-fast.
-g++ ${srcdir}/doc/doxygen/stdheader.cc -o ./stdheader
-problematic=`egrep -l '#include <.*_.*>' [a-z]*.3`
+$gxx $cppflags $cxxflags ${srcdir}/doc/doxygen/stdheader.cc -o ./stdheader || exit 1
+# Doxygen outputs something like "\fC#include <unique_lock\&.h>\fP" and
+# we want that internal header to be replaced with something like <mutex>.
+problematic=`egrep -l '#include <.*h>' [a-z]*.3`
 for f in $problematic; do
     # this is also slow, but safe and easy to debug
     oldh=`sed -n '/fC#include </s/.*<\(.*\)>.*/\1/p' $f`
-    newh=`echo $oldh | ./stdheader`
-    sed 's=${oldh}=${newh}=' $f > TEMP
-    mv TEMP $f
+    newh=`echo $oldh | sed 's/\\\\&\\././g' | ./stdheader`
+    sed "s=${oldh/\\/.}=${newh}=" $f > TEMP && mv TEMP $f
 done
 rm stdheader
 
@@ -323,10 +330,14 @@ cp ${srcdir}/doc/doxygen/Intro.3 C++Intro.3
 # Why didn't I do this at the start?  Were rabid weasels eating my brain?
 # Who the fsck would "man std_vector" when the class isn't named that?
 
+# If no files match a glob, skip the for-loop:
+shopt -s nullglob
 # First, deal with nested namespaces.
-for f in *chrono_*; do
-    newname=`echo $f | sed 's/chrono_/chrono::/'`
-    mv $f $newname
+for ns in chrono filesystem ranges views literals; do
+  for f in std_${ns}_*; do
+      newname=`echo $f | sed "s/std_${ns}_/std::${ns}::/"`
+      mv $f $newname
+  done
 done
 for f in *__debug_*; do
     newname=`echo $f | sed 's/__debug_/__debug::/'`
@@ -413,11 +424,16 @@ for f in *_src_*; do
     rm $f
 done
 
+# Remove all internal implementation details?
+# rm std::_[A-Z]*.3 std::__detail*.3
+
+shopt -u nullglob
+
 
 # Also, for some reason, typedefs don't get their own man pages.  Sigh.
 for f in ios streambuf istream ostream iostream stringbuf \
 	 istringstream ostringstream stringstream filebuf ifstream \
-	 ofstream fstream string;
+	 ofstream fstream string
 do
     echo ".so man3/std::basic_${f}.3" > std::${f}.3
     echo ".so man3/std::basic_${f}.3" > std::w${f}.3
diff --git a/libstdc++-v3/src/c++11/debug.cc b/libstdc++-v3/src/c++11/debug.cc
index 0128535..77cb2a2 100644
--- a/libstdc++-v3/src/c++11/debug.cc
+++ b/libstdc++-v3/src/c++11/debug.cc
@@ -33,7 +33,8 @@
 #include <debug/vector>
 
 #include <cassert>
-#include <cstdio>
+#include <cstdio>	// for std::fprintf, stderr
+#include <cstdlib>	// for std::abort
 #include <cctype>	// for std::isspace.
 #include <cstring>	// for std::strstr.
 
@@ -43,6 +44,21 @@
 
 #include "mutex_pool.h"
 
+#ifdef _GLIBCXX_VERBOSE_ASSERT
+namespace std
+{
+  [[__noreturn__]]
+  void
+  __glibcxx_assert_fail(const char* file, int line,
+			const char* function, const char* condition) noexcept
+  {
+    fprintf(stderr, "%s:%d: %s: Assertion '%s' failed.\n",
+		      file, line, function, condition);
+    abort();
+  }
+}
+#endif
+
 using namespace std;
 
 namespace
diff --git a/libstdc++-v3/src/c++11/random.cc b/libstdc++-v3/src/c++11/random.cc
index 4b64bde..76857e1 100644
--- a/libstdc++-v3/src/c++11/random.cc
+++ b/libstdc++-v3/src/c++11/random.cc
@@ -37,6 +37,8 @@
 # ifdef _GLIBCXX_X86_RDSEED
 #  define USE_RDSEED 1
 # endif
+#elif defined __powerpc64__ && defined __BUILTIN_CPU_SUPPORTS__
+# define USE_DARN 1
 #endif
 
 #include <cerrno>
@@ -66,10 +68,15 @@
 # include <stdlib.h>
 #endif
 
-#if defined _GLIBCXX_USE_CRT_RAND_S || defined _GLIBCXX_USE_DEV_RANDOM
+#ifdef _GLIBCXX_HAVE_GETENTROPY
+# include <unistd.h>
+#endif
+
+#if defined _GLIBCXX_USE_CRT_RAND_S || defined _GLIBCXX_USE_DEV_RANDOM \
+  || _GLIBCXX_HAVE_GETENTROPY
 // The OS provides a source of randomness we can use.
 # pragma GCC poison _M_mt
-#elif defined USE_RDRAND || defined USE_RDSEED
+#elif defined USE_RDRAND || defined USE_RDSEED || defined USE_DARN
 // Hardware instructions might be available, but use cpuid checks at runtime.
 # pragma GCC poison _M_mt
 // If the runtime cpuid checks fail we'll use a linear congruential engine.
@@ -95,7 +102,7 @@ namespace std _GLIBCXX_VISIBILITY(default)
       unsigned int retries = 100;
       unsigned int val;
 
-      while (__builtin_ia32_rdrand32_step(&val) == 0)
+      while (__builtin_ia32_rdrand32_step(&val) == 0) [[__unlikely__]]
 	if (--retries == 0)
 	  std::__throw_runtime_error(__N("random_device: rdrand failed"));
 
@@ -111,7 +118,7 @@ namespace std _GLIBCXX_VISIBILITY(default)
       unsigned int retries = 100;
       unsigned int val;
 
-      while (__builtin_ia32_rdseed_si_step(&val) == 0)
+      while (__builtin_ia32_rdseed_si_step(&val) == 0) [[__unlikely__]]
 	{
 	  if (--retries == 0)
 	    {
@@ -135,6 +142,24 @@ namespace std _GLIBCXX_VISIBILITY(default)
 #endif
 #endif
 
+#ifdef USE_DARN
+    unsigned int
+    __attribute__((target("cpu=power9")))
+    __ppc_darn(void*)
+    {
+      const uint64_t failed = -1;
+      unsigned int retries = 10;
+      uint64_t val = __builtin_darn();
+      while (val == failed) [[__unlikely__]]
+	{
+	  if (--retries == 0)
+	    std::__throw_runtime_error(__N("random_device: darn failed"));
+	  val = __builtin_darn();
+	}
+      return (uint32_t)val;
+    }
+#endif
+
 #ifdef _GLIBCXX_USE_CRT_RAND_S
     unsigned int
     __winxp_rand_s(void*)
@@ -146,6 +171,25 @@ namespace std _GLIBCXX_VISIBILITY(default)
     }
 #endif
 
+#ifdef _GLIBCXX_HAVE_GETENTROPY
+    unsigned int
+    __libc_getentropy(void*)
+    {
+      unsigned int val;
+      if (::getentropy(&val, sizeof(val)) != 0)
+	std::__throw_runtime_error(__N("random_device: getentropy failed"));
+      return val;
+    }
+#endif
+
+#ifdef _GLIBCXX_HAVE_ARC4RANDOM
+    unsigned int
+    __libc_arc4random(void*)
+    {
+      return ::arc4random();
+    }
+#endif
+
 #ifdef USE_LCG
     // TODO: use this to seed std::mt19937 engine too.
     unsigned
@@ -193,11 +237,16 @@ namespace std _GLIBCXX_VISIBILITY(default)
     }
 #endif
 
-    enum Which {
-      rand_s = 1, rdseed = 2, rdrand = 4, device_file = 8, prng = 16,
+    enum Which : unsigned {
+      device_file = 1, prng = 2, rand_s = 4, getentropy = 8, arc4random = 16,
+      rdseed = 64, rdrand = 128, darn = 256,
       any = 0xffff
     };
 
+    constexpr Which
+    operator|(Which l, Which r) noexcept
+    { return Which(unsigned(l) | unsigned(r)); }
+
     inline Which
     which_source(random_device::result_type (*func [[maybe_unused]])(void*),
 		 void* file [[maybe_unused]])
@@ -221,11 +270,26 @@ namespace std _GLIBCXX_VISIBILITY(default)
 	return rdrand;
 #endif
 
+#ifdef USE_DARN
+      if (func == &__ppc_darn)
+	return darn;
+#endif
+
 #ifdef _GLIBCXX_USE_DEV_RANDOM
       if (file != nullptr)
 	return device_file;
 #endif
 
+#ifdef _GLIBCXX_HAVE_ARC4RANDOM
+      if (func == __libc_arc4random)
+	return arc4random;
+#endif
+
+#ifdef _GLIBCXX_HAVE_GETENTROPY
+      if (func == __libc_getentropy)
+	return getentropy;
+#endif
+
 #ifdef USE_LCG
       if (func == &__lcg)
 	return prng;
@@ -269,10 +333,26 @@ namespace std _GLIBCXX_VISIBILITY(default)
     else if (token == "rdrand" || token == "rdrnd")
       which = rdrand;
 #endif // USE_RDRAND
+#ifdef USE_DARN
+    else if (token == "darn")
+      which = darn;
+#endif
+#if defined USE_RDRAND || defined USE_RDSEED || defined USE_DARN
+    else if (token == "hw" || token == "hardware")
+      which = rdrand | rdseed | darn;
+#endif
 #ifdef _GLIBCXX_USE_CRT_RAND_S
     else if (token == "rand_s")
       which = rand_s;
 #endif // _GLIBCXX_USE_CRT_RAND_S
+#ifdef _GLIBCXX_HAVE_GETENTROPY
+    else if (token == "getentropy")
+      which = getentropy;
+#endif // _GLIBCXX_HAVE_GETENTROPY
+#ifdef _GLIBCXX_HAVE_ARC4RANDOM
+    else if (token == "arc4random")
+      which = arc4random;
+#endif // _GLIBCXX_HAVE_ARC4RANDOM
 #ifdef _GLIBCXX_USE_DEV_RANDOM
     else if (token == "/dev/urandom" || token == "/dev/random")
       {
@@ -346,6 +426,37 @@ namespace std _GLIBCXX_VISIBILITY(default)
     }
 #endif // USE_RDRAND
 
+#ifdef USE_DARN
+    if (which & darn)
+      {
+	if (__builtin_cpu_supports("darn"))
+	  {
+	    _M_func = &__ppc_darn;
+	    return;
+	  }
+      }
+#endif // USE_DARN
+
+#ifdef _GLIBCXX_HAVE_ARC4RANDOM
+    if (which & arc4random)
+      {
+	_M_func = &__libc_arc4random;
+	return;
+      }
+#endif // _GLIBCXX_HAVE_ARC4RANDOM
+
+#ifdef _GLIBCXX_HAVE_GETENTROPY
+    if (which & getentropy)
+      {
+	unsigned int i;
+	if (::getentropy(&i, sizeof(i)) == 0) // On linux the syscall can fail.
+	  {
+	    _M_func = &__libc_getentropy;
+	    return;
+	  }
+      }
+#endif // _GLIBCXX_HAVE_GETENTROPY
+
 #ifdef _GLIBCXX_USE_DEV_RANDOM
     if (which & device_file)
     {
@@ -497,6 +608,10 @@ namespace std _GLIBCXX_VISIBILITY(default)
     {
     case rdrand:
     case rdseed:
+    case darn:
+      return (double) max;
+    case arc4random:
+    case getentropy:
       return (double) max;
     case rand_s:
     case prng:
diff --git a/libstdc++-v3/src/c++17/memory_resource.cc b/libstdc++-v3/src/c++17/memory_resource.cc
index 1ba7990..9fc3bb7 100644
--- a/libstdc++-v3/src/c++17/memory_resource.cc
+++ b/libstdc++-v3/src/c++17/memory_resource.cc
@@ -603,7 +603,7 @@ namespace pmr
     void* pointer = nullptr;
     aligned_size<min> _M_size;
 
-    size_t size() const noexcept
+    constexpr size_t size() const noexcept
     {
       if (_M_size.value == size_t(-1)) [[unlikely]]
 	return size_t(-1);
diff --git a/libstdc++-v3/testsuite/17_intro/headers/c++1998/49745.cc b/libstdc++-v3/testsuite/17_intro/headers/c++1998/49745.cc
index 204975e..f98fa3b 100644
--- a/libstdc++-v3/testsuite/17_intro/headers/c++1998/49745.cc
+++ b/libstdc++-v3/testsuite/17_intro/headers/c++1998/49745.cc
@@ -1,4 +1,5 @@
 // { dg-do compile { target *-*-linux* *-*-gnu* } }
+// { dg-add-options no_pch }
 
 // Copyright (C) 2011-2021 Free Software Foundation, Inc.
 //
@@ -18,7 +19,117 @@
 // <http://www.gnu.org/licenses/>.
 
 // libstdc++/49745
-#include <bits/stdc++.h>
+// error: 'int truncate' redeclared as different kind of symbol
+
+// This tests that no libstdc++ headers include <unistd.h>.
+// However, as discussed in PR libstdc++/100117 we cannot guarantee that
+// no libc headers include it indirectly, so only include the C++ headers.
+// That means we can't use <bits/stdc++.h> because that includes <cstdio> etc.
+// so we list the C++ headers explicitly here. This list is unfortunately
+// doomed to get out of date as new headers are added to the library.
+
+#include <algorithm>
+#include <bitset>
+#include <complex>
+#include <deque>
+#include <exception>
+#include <fstream>
+#include <functional>
+#include <iomanip>
+#include <ios>
+#include <iosfwd>
+#include <iostream>
+#include <istream>
+#include <iterator>
+#include <limits>
+#include <list>
+#include <locale>
+#include <map>
+#include <memory>
+#include <new>
+#include <numeric>
+#include <ostream>
+#include <queue>
+#include <set>
+#include <sstream>
+#include <stack>
+#include <stdexcept>
+#include <streambuf>
+#include <string>
+#include <typeinfo>
+#include <utility>
+#include <valarray>
+#include <vector>
+
+#if __cplusplus >= 201103L
+#include <array>
+#include <atomic>
+#include <chrono>
+#include <codecvt>
+#include <condition_variable>
+#include <forward_list>
+#include <future>
+#include <initializer_list>
+#include <mutex>
+#include <random>
+#include <ratio>
+#include <regex>
+#include <scoped_allocator>
+#include <system_error>
+#include <thread>
+#include <tuple>
+#include <typeindex>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+#endif
+
+#if __cplusplus >= 201402L
+#include <shared_mutex>
+#endif
+
+#if __cplusplus >= 201703L
+#include <any>
+#include <charconv>
+// #include <execution>
+#include <filesystem>
+#include <optional>
+#include <memory_resource>
+#include <string_view>
+#include <variant>
+#endif
+
+#if __cplusplus >= 202002L
+#include <barrier>
+#include <bit>
+#include <compare>
+#include <concepts>
+#if __cpp_impl_coroutine
+# include <coroutine>
+#endif
+#if __has_include (<format>)
+# include <format>
+#endif
+#include <latch>
+#include <numbers>
+#include <ranges>
+#include <span>
+#include <stop_token>
+#include <semaphore>
+#include <source_location>
+#include <syncstream>
+#include <version>
+#endif
+
+#if __cplusplus > 202002L
+#if __has_include(<spanstream>)
+# include <spanstream>
+#endif
+#if __has_include(<stacktrace>)
+# include <stacktrace>
+#endif
+#endif
+
 int truncate = 0;
 
 // { dg-xfail-if "PR libstdc++/99995" { c++20 } }
diff --git a/libstdc++-v3/testsuite/17_intro/names.cc b/libstdc++-v3/testsuite/17_intro/names.cc
index b5e926f..2a908ea 100644
--- a/libstdc++-v3/testsuite/17_intro/names.cc
+++ b/libstdc++-v3/testsuite/17_intro/names.cc
@@ -251,6 +251,10 @@
 #undef p
 // See https://gcc.gnu.org/ml/libstdc++/2019-05/msg00175.html
 #undef ptr
+// <sys/timespec_util.h> uses this as parameter
+#undef r
+// <stdlib.h> uses this as member of drand48_data
+#undef x
 #endif
 
 #ifdef __VXWORKS__
diff --git a/libstdc++-v3/testsuite/18_support/exception_ptr/60612-unexpected.cc b/libstdc++-v3/testsuite/18_support/exception_ptr/60612-unexpected.cc
index 600ccb5..18bd1fe 100644
--- a/libstdc++-v3/testsuite/18_support/exception_ptr/60612-unexpected.cc
+++ b/libstdc++-v3/testsuite/18_support/exception_ptr/60612-unexpected.cc
@@ -1,3 +1,4 @@
+// { dg-options "-Wno-deprecated-declarations" }
 // { dg-do run { target { c++11_only || c++14_only } } }
 
 // Copyright (C) 2014-2021 Free Software Foundation, Inc.
diff --git a/libstdc++-v3/testsuite/18_support/headers/exception/synopsis.cc b/libstdc++-v3/testsuite/18_support/headers/exception/synopsis.cc
index 2f0bae4..de35ec4 100644
--- a/libstdc++-v3/testsuite/18_support/headers/exception/synopsis.cc
+++ b/libstdc++-v3/testsuite/18_support/headers/exception/synopsis.cc
@@ -24,7 +24,7 @@ namespace std {
   class bad_exception;
 
   typedef void (*unexpected_handler)();
-  unexpected_handler set_unexpected(unexpected_handler  f ) throw();
+  unexpected_handler set_unexpected(unexpected_handler  f ) throw(); // { dg-warning "deprecated" { target c++11 } }
   unexpected_handler get_unexpected() noexcept;
   void unexpected();
 
diff --git a/libstdc++-v3/testsuite/18_support/set_unexpected.cc b/libstdc++-v3/testsuite/18_support/set_unexpected.cc
index 3f38431..6cedc7d 100644
--- a/libstdc++-v3/testsuite/18_support/set_unexpected.cc
+++ b/libstdc++-v3/testsuite/18_support/set_unexpected.cc
@@ -15,7 +15,7 @@
 // with this library; see the file COPYING3.  If not see
 // <http://www.gnu.org/licenses/>.
 
-// { dg-options "-std=gnu++11" }
+// { dg-options "-std=gnu++11 -Wno-deprecated-declarations" }
 // { dg-do run { target { c++11_only || c++14_only } } }
 
 #include <exception>
diff --git a/libstdc++-v3/testsuite/18_support/unexpected_handler.cc b/libstdc++-v3/testsuite/18_support/unexpected_handler.cc
index 509bb5c..551b0e7 100644
--- a/libstdc++-v3/testsuite/18_support/unexpected_handler.cc
+++ b/libstdc++-v3/testsuite/18_support/unexpected_handler.cc
@@ -15,6 +15,7 @@
 // with this library; see the file COPYING3.  If not see
 // <http://www.gnu.org/licenses/>.
 
+// { dg-options "-Wno-deprecated-declarations" }
 // { dg-do run { target c++11 } }
 
 // D.11 Violating exception-specifications
diff --git a/libstdc++-v3/testsuite/20_util/any/102894.cc b/libstdc++-v3/testsuite/20_util/any/102894.cc
new file mode 100644
index 0000000..66ea9a0
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/any/102894.cc
@@ -0,0 +1,20 @@
+// { dg-do compile { target c++17 } }
+#include <any>
+
+template<typename T, typename = void>
+struct can_make_any
+: std::false_type
+{ };
+
+template<typename T>
+struct can_make_any<T, std::void_t<decltype(std::make_any<T>())>>
+: std::true_type
+{ };
+
+struct move_only
+{
+  move_only() = default;
+  move_only(move_only&&) = default;
+};
+
+static_assert( ! can_make_any<move_only>::value ); // PR libstdc++/102894
diff --git a/libstdc++-v3/testsuite/20_util/to_address/1_neg.cc b/libstdc++-v3/testsuite/20_util/to_address/1_neg.cc
index ff3a1fb..5030f89 100644
--- a/libstdc++-v3/testsuite/20_util/to_address/1_neg.cc
+++ b/libstdc++-v3/testsuite/20_util/to_address/1_neg.cc
@@ -32,5 +32,5 @@ struct P
 void test01()
 {
   P p;
-  std::to_address(p); // { dg-error "required from here" }
+  std::to_address(p); // { dg-error "" }
 }
diff --git a/libstdc++-v3/testsuite/20_util/tuple/element_access/get_neg.cc b/libstdc++-v3/testsuite/20_util/tuple/element_access/get_neg.cc
index 225bb62..113a7fd 100644
--- a/libstdc++-v3/testsuite/20_util/tuple/element_access/get_neg.cc
+++ b/libstdc++-v3/testsuite/20_util/tuple/element_access/get_neg.cc
@@ -60,3 +60,4 @@ test03()
 }
 
 // { dg-error "tuple index must be in range" "" { target *-*-* } 0 }
+// { dg-prune-output "no type named 'type' in .*_Nth_type" }
diff --git a/libstdc++-v3/testsuite/20_util/variant/102912.cc b/libstdc++-v3/testsuite/20_util/variant/102912.cc
new file mode 100644
index 0000000..0dea138
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/variant/102912.cc
@@ -0,0 +1,71 @@
+// { dg-do compile { target c++17 } }
+#include <variant>
+
+void
+test01()
+{
+  struct X {
+    ~X() { } // non-trivial
+  };
+
+  std::variant<const int, const X> v;
+  auto vv = v;
+}
+
+#if __cpp_lib_variant >= 202106L // P2231R1 constexpr destruction in variant
+constexpr bool
+test02()
+{
+  struct Y {
+    constexpr ~Y() { } // non-trivial
+  };
+  using V = std::variant<int, const int, const Y, Y>;
+
+  V v1(std::in_place_index<1>, 1);
+  V vv1 = v1;
+  if (vv1.index() != v1.index())
+    return false;
+
+  V v2(std::in_place_index<2>);
+  V vv2 = v2;
+  if (vv2.index() != v2.index())
+    return false;
+
+  return true;
+}
+static_assert( test02() );
+
+constexpr bool
+test03()
+{
+  struct Y {
+    constexpr ~Y() { } // non-trivial
+  };
+  using V = std::variant<int, int, Y, Y>;
+
+  V v1(std::in_place_index<1>, 1);
+  V vv1 = v1;
+  if (vv1.index() != v1.index())
+    return false;
+  vv1 = v1;
+  if (vv1.index() != v1.index())
+    return false;
+  vv1 = std::move(v1);
+  if (vv1.index() != v1.index())
+    return false;
+
+  V v2(std::in_place_index<2>);
+  V vv2 = v2;
+  if (vv2.index() != v2.index())
+    return false;
+  vv2 = v2;
+  if (vv2.index() != v2.index())
+    return false;
+  vv2 = std::move(v2);
+  if (vv2.index() != v2.index())
+    return false;
+
+  return true;
+}
+static_assert( test03() );
+#endif
diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/allocator/71964.cc b/libstdc++-v3/testsuite/21_strings/basic_string/allocator/71964.cc
index c57cb96..4196b33 100644
--- a/libstdc++-v3/testsuite/21_strings/basic_string/allocator/71964.cc
+++ b/libstdc++-v3/testsuite/21_strings/basic_string/allocator/71964.cc
@@ -40,7 +40,7 @@ template<typename T>
       a.moved_from = true;
     }
 
-    T* allocate(unsigned n) { return std::allocator<T>{}.allcoate(n); }
+    T* allocate(unsigned n) { return std::allocator<T>{}.allocate(n); }
     void deallocate(T* p, unsigned n) { std::allocator<T>{}.deallocate(p, n); }
 
     bool moved_to;
diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/1.cc b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/1.cc
index a2320f2..b3990f6 100644
--- a/libstdc++-v3/testsuite/21_strings/basic_string/capacity/1.cc
+++ b/libstdc++-v3/testsuite/21_strings/basic_string/capacity/1.cc
@@ -17,7 +17,7 @@
 // with this library; see the file COPYING3.  If not see
 // <http://www.gnu.org/licenses/>.
 
-// { dg-options "-Wno-stringop-overflow" }
+// { dg-options "-Wno-stringop-overflow -Wno-stringop-overread" }
 
 // 21.3.3 string capacity
 
diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/append/wchar_t/1.cc b/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/append/wchar_t/1.cc
index bb2d682..684209f 100644
--- a/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/append/wchar_t/1.cc
+++ b/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/append/wchar_t/1.cc
@@ -117,7 +117,7 @@ void test01(void)
   VERIFY( str06 == L"corpus, corpus" );
 
   str06 = str02;
-  str06.append(L"corpus, ", 12);
+  str06.append(L"corpus, ", 9); // n=9 includes null terminator
   VERIFY( str06 != L"corpus, corpus, " );
 
 
diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/operations/compare/wchar_t/1.cc b/libstdc++-v3/testsuite/21_strings/basic_string/operations/compare/wchar_t/1.cc
index 27836f8..6f2113f 100644
--- a/libstdc++-v3/testsuite/21_strings/basic_string/operations/compare/wchar_t/1.cc
+++ b/libstdc++-v3/testsuite/21_strings/basic_string/operations/compare/wchar_t/1.cc
@@ -81,8 +81,8 @@ test01()
   test_value(wcsncmp(str_1.data(), str_0.data(), 6), z);
   test_value(wcsncmp(str_1.data(), str_0.data(), 14), lt);
   test_value(wmemcmp(str_1.data(), str_0.data(), 6), z);
-  test_value(wmemcmp(str_1.data(), str_0.data(), 14), lt);
-  test_value(wmemcmp(L"costa marbella", L"costa rica", 14), lt);
+  test_value(wmemcmp(str_1.data(), str_0.data(), 10), lt);
+  test_value(wmemcmp(L"costa marbella", L"costa rica", 10), lt);
 
   // int compare(const basic_string& str) const;
   test_value(str_0.compare(str_1), gt); //because r>m
diff --git a/libstdc++-v3/testsuite/23_containers/headers/vector/synopsis.cc b/libstdc++-v3/testsuite/23_containers/headers/vector/synopsis.cc
index 0fc46fe..f02659f 100644
--- a/libstdc++-v3/testsuite/23_containers/headers/vector/synopsis.cc
+++ b/libstdc++-v3/testsuite/23_containers/headers/vector/synopsis.cc
@@ -27,13 +27,21 @@
 # define NOTHROW(X)
 #endif
 
+#if __cplusplus >= 202002L
+# define CONSTEXPR constexpr
+#else
+# define CONSTEXPR
+#endif
+
 namespace std {
   template <class T, class Allocator> class vector;
 
   template <class T, class Allocator>
+    CONSTEXPR
     bool operator==(const vector<T,Allocator>& x,
                     const vector<T,Allocator>& y);
 
+#if __cplusplus < 202002L
   template <class T, class Allocator>
     bool operator< (const vector<T,Allocator>& x,
                     const vector<T,Allocator>& y);
@@ -53,17 +61,21 @@ namespace std {
   template <class T, class Allocator>
     bool operator<=(const vector<T,Allocator>& x,
                     const vector<T,Allocator>& y);
+#endif
 
   template <class T, class Allocator>
+    CONSTEXPR
     void swap(vector<T,Allocator>& x, vector<T,Allocator>& y)
       NOTHROW(noexcept(x.swap(y)));
 
   template <class Allocator> class vector<bool,Allocator>;
 
   template <class Allocator>
+    CONSTEXPR
     bool operator==(const vector<bool,Allocator>& x,
                     const vector<bool,Allocator>& y);
 
+#if __cplusplus < 202002L
   template <class Allocator>
     bool operator< (const vector<bool,Allocator>& x,
                     const vector<bool,Allocator>& y);
@@ -83,7 +95,9 @@ namespace std {
   template <class Allocator>
     bool operator<=(const vector<bool,Allocator>& x,
                     const vector<bool,Allocator>& y);
+#endif
 
   template <class Allocator>
+    CONSTEXPR
     void swap(vector<bool,Allocator>& x, vector<bool,Allocator>& y);
 }
diff --git a/libstdc++-v3/testsuite/23_containers/list/operations/66742.cc b/libstdc++-v3/testsuite/23_containers/list/operations/66742.cc
index 24bda39..94a37a3 100644
--- a/libstdc++-v3/testsuite/23_containers/list/operations/66742.cc
+++ b/libstdc++-v3/testsuite/23_containers/list/operations/66742.cc
@@ -48,8 +48,31 @@ test01()
   VERIFY( is_sorted(l, std::greater<int>()) );
 }
 
+void
+test02()
+{
+  // The standard doesn't require comparisons to be const-correct.
+  // The initial fix for PR 66742 caused a regression here.
+
+  struct X
+  {
+    bool operator<(X&) /* non-const */ { return false; }
+  };
+
+  struct Cmp
+  {
+    bool operator()(X&, X&) /* non-const */ { return false; }
+  };
+
+  std::list<X> l;
+  l.sort();
+  Cmp c;
+  l.sort(c);
+}
+
 int
 main()
 {
   test01();
+  test02();
 }
diff --git a/libstdc++-v3/testsuite/23_containers/set/allocator/71964.cc b/libstdc++-v3/testsuite/23_containers/set/allocator/71964.cc
index 34a02d8..a2c166a 100644
--- a/libstdc++-v3/testsuite/23_containers/set/allocator/71964.cc
+++ b/libstdc++-v3/testsuite/23_containers/set/allocator/71964.cc
@@ -40,7 +40,7 @@ template<typename T>
       a.moved_from = true;
     }
 
-    T* allocate(unsigned n) { return std::allocator<T>{}.allcoate(n); }
+    T* allocate(unsigned n) { return std::allocator<T>{}.allocate(n); }
     void deallocate(T* p, unsigned n) { std::allocator<T>{}.deallocate(p, n); }
 
     bool moved_to;
diff --git a/libstdc++-v3/testsuite/23_containers/stack/deduction.cc b/libstdc++-v3/testsuite/23_containers/stack/deduction.cc
index dea7ba0..0ac3737 100644
--- a/libstdc++-v3/testsuite/23_containers/stack/deduction.cc
+++ b/libstdc++-v3/testsuite/23_containers/stack/deduction.cc
@@ -98,6 +98,6 @@ test03()
   check_type<std::stack<long>>(s1);
 
   std::stack s2(l.begin(), l.end(), std::allocator<long>());
-  check_type<std::stack<long>>(s1);
+  check_type<std::stack<long>>(s2);
 }
 #endif
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge1_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge1_neg.cc
new file mode 100644
index 0000000..6d00794
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge1_neg.cc
@@ -0,0 +1,31 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_map>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_map<int, double>;
+
+void
+test01()
+{
+  test_type c0{ { 1, 3.5 }, { 2, 5.5 }, { 3, 7.5 }, { 5, 11.5 }, { 6, 13.5 } };
+  test_type c1{ { 1, 3.5 }, { 2, 5.5 }, { 3, 7.5 }, { 4, 9.5 } };
+
+  auto it2 = c1.find(2);
+  auto it4 = c1.find(4);
+  VERIFY( it2->second == 5.5 );
+  VERIFY( it4->second == 9.5 );
+
+  c0.merge(c1);
+
+  VERIFY( it2->second == 5.5 );
+  VERIFY( it4 != it2 ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge2_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge2_neg.cc
new file mode 100644
index 0000000..543cd96
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge2_neg.cc
@@ -0,0 +1,32 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_map>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_map<int, int>;
+
+void
+test01()
+{
+  test_type c0{ { 1, 1 }, { 2, 2 }, { 3, 3 }, { 5, 5 }, { 6, 6 } };
+  test_type c1{ { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 } };
+
+  auto it2 = c1.find(2);
+  auto it4 = c1.find(4);
+  VERIFY( it2->second == 2 );
+  VERIFY( it4->second == 4 );
+
+  c0.merge(std::move(c1));
+
+  VERIFY( it2->second == 2 );
+  VERIFY( it2 != it4 ); // Invalid iterator.
+}
+
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge3_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge3_neg.cc
new file mode 100644
index 0000000..8e23479
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge3_neg.cc
@@ -0,0 +1,42 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_map>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_map<int, int>;
+
+void
+test01()
+{
+  test_type c0
+    {
+     { 1, 1 }, { 2, 2 }, { 3, 3 },
+     { 5, 5 }, { 6, 6 }, { 7, 7 }
+    };
+  std::unordered_multimap<int, int> c1
+    {
+     { 1, 1 }, { 1, 1 }, { 2, 2 }, { 2, 2 },
+     { 3, 3 }, { 3, 3 }, { 4, 4 }, { 4, 4 },
+     { 5, 5 }
+    };
+
+  auto it1 = c1.find(1);
+  auto it41 = c1.find(4);
+  auto it42 = it41;
+  ++it42;
+  VERIFY( it42->second == 4 );
+
+  c0.merge(c1);
+
+  VERIFY( it1->second == 1 );
+  VERIFY( c1.count(4) == 1 );
+  VERIFY( it41 != it42 ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge4_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge4_neg.cc
new file mode 100644
index 0000000..3c9c826
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_map/debug/merge4_neg.cc
@@ -0,0 +1,42 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_map>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_map<int, int>;
+
+void
+test01()
+{
+  test_type c0
+    {
+     { 1, 1 }, { 2, 2 }, { 3, 3 },
+     { 5, 5 }, { 6, 6 }, { 7, 7 }
+    };
+  std::unordered_multimap<int, int> c1
+    {
+     { 1, 1 }, { 1, 1 }, { 2, 2 }, { 2, 2 },
+     { 3, 3 }, { 3, 3 }, { 4, 4 }, { 4, 4 },
+     { 5, 5 }
+    };
+
+  auto it1 = c1.find(1);
+  auto it41 = c1.find(4);
+  auto it42 = it41;
+  ++it42;
+  VERIFY( it42->second == 4 );
+
+  c0.merge(std::move(c1));
+
+  VERIFY( it1->second == 1 );
+  VERIFY( c1.count(4) == 1 );
+  VERIFY( it41 != it42 ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge1_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge1_neg.cc
new file mode 100644
index 0000000..25b3b9e
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge1_neg.cc
@@ -0,0 +1,32 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_map>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_multimap<int, int>;
+
+void
+test01()
+{
+  test_type c0
+    {
+     { 1, 1 }, { 1, 1 }, { 2, 2 },
+     { 2, 2 }, { 3, 3 }, { 3, 3 }
+    };
+  test_type c1 = c0;
+
+  auto it = c1.find(2);
+  VERIFY( it->second == 2 );
+
+  c0.merge(c1);
+
+  VERIFY( it != c1.end() ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge2_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge2_neg.cc
new file mode 100644
index 0000000..8d28d83
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge2_neg.cc
@@ -0,0 +1,32 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_map>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_multimap<int, int>;
+
+void
+test01()
+{
+  test_type c0
+    {
+     { 1, 1 }, { 1, 1 }, { 2, 2 },
+     { 2, 2 }, { 3, 3 }, { 3, 3 }
+    };
+  test_type c1 = c0;
+
+  auto it = c1.find(2);
+  VERIFY( it->second == 2 );
+
+  c0.merge(std::move(c1));
+
+  VERIFY( it != c1.end() ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge3_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge3_neg.cc
new file mode 100644
index 0000000..5db91a2
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge3_neg.cc
@@ -0,0 +1,32 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_map>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_multimap<int, int>;
+
+void
+test01()
+{
+  test_type c0
+    {
+     { 1, 1 }, { 1, 1 }, { 2, 2 },
+     { 2, 2 }, { 3, 3 }, { 3, 3 }
+    };
+  std::unordered_map<int, int> c1{ { 1, 1 }, { 2, 2 }, { 3, 3 } };
+
+  auto it = c1.find(2);
+  VERIFY( it->second == 2 );
+
+  c0.merge(c1);
+
+  VERIFY( it != c1.end() ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge4_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge4_neg.cc
new file mode 100644
index 0000000..a163670
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multimap/debug/merge4_neg.cc
@@ -0,0 +1,32 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_map>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_multimap<int, int>;
+
+void
+test01()
+{
+  test_type c0
+    {
+     { 1, 1 }, { 1, 1 }, { 2, 2 },
+     { 2, 2 }, { 3, 3 }, { 3, 3 }
+    };
+  std::unordered_map<int, int> c1{ { 1, 1 }, { 2, 2 }, { 3, 3 } };
+
+  auto it = c1.find(2);
+  VERIFY( it->second == 2 );
+
+  c0.merge(std::move(c1));
+
+  VERIFY( it != c1.end() ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge1_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge1_neg.cc
new file mode 100644
index 0000000..bce8da7
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge1_neg.cc
@@ -0,0 +1,28 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_set>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_multiset<int>;
+
+void
+test01()
+{
+  test_type c0{ 1, 1, 2, 2, 3, 3 };
+  test_type c1 = c0;
+
+  auto it = c1.find(2);
+  VERIFY( *it == 2 );
+
+  c0.merge(c1);
+
+  VERIFY( it != c1.end() ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge2_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge2_neg.cc
new file mode 100644
index 0000000..72317a3
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge2_neg.cc
@@ -0,0 +1,28 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_set>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_multiset<int>;
+
+void
+test01()
+{
+  test_type c0{ 1, 1, 2, 2, 3, 3 };
+  test_type c1 = c0;
+
+  auto it = c1.find(2);
+  VERIFY( *it == 2 );
+
+  c0.merge(std::move(c1));
+
+  VERIFY( it != c1.end() ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge3_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge3_neg.cc
new file mode 100644
index 0000000..1b1f487
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge3_neg.cc
@@ -0,0 +1,28 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_set>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_multiset<int>;
+
+void
+test01()
+{
+  test_type c0{ 1, 1, 2, 2, 3, 3 };
+  std::unordered_set<int> c1{ 1, 2, 3 };
+
+  auto it = c1.find(2);
+  VERIFY( *it == 2 );
+
+  c0.merge(c1);
+
+  VERIFY( it != c1.end() ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge4_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge4_neg.cc
new file mode 100644
index 0000000..5005cf8
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multiset/debug/merge4_neg.cc
@@ -0,0 +1,28 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_set>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_multiset<int>;
+
+void
+test01()
+{
+  test_type c0{ 1, 1, 2, 2, 3, 3 };
+  std::unordered_set<int> c1{ 1, 2, 3 };
+
+  auto it = c1.find(2);
+  VERIFY( *it == 2 );
+
+  c0.merge(std::move(c1));
+
+  VERIFY( it != c1.end() ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multiset/modifiers/merge.cc b/libstdc++-v3/testsuite/23_containers/unordered_multiset/modifiers/merge.cc
index 1ed2ce2..07b8a34 100644
--- a/libstdc++-v3/testsuite/23_containers/unordered_multiset/modifiers/merge.cc
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multiset/modifiers/merge.cc
@@ -17,6 +17,7 @@
 
 // { dg-do run { target c++17 } }
 
+#include <string>
 #include <unordered_set>
 #include <algorithm>
 #include <testsuite_hooks.h>
@@ -105,6 +106,26 @@ test04()
   VERIFY( c2.empty() );
 }
 
+void
+test05()
+{
+  const std::unordered_multiset<std::string> c0{ "abcd", "abcd", "efgh", "efgh", "ijkl", "ijkl" };
+  std::unordered_multiset<std::string> c1 = c0;
+  std::unordered_set<std::string> c2( c0.begin(), c0.end() );
+
+  c1.merge(c2);
+  VERIFY( c1.size() == (1.5 * c0.size()) );
+  for (auto& i : c1)
+    VERIFY( c1.count(i) == (1.5 * c0.count(i)) );
+  VERIFY( c2.empty() );
+
+  c1.clear();
+  c2.insert( c0.begin(), c0.end() );
+  c1.merge(std::move(c2));
+  VERIFY( c1.size() == (0.5 * c0.size()) );
+  VERIFY( c2.empty() );
+}
+
 int
 main()
 {
@@ -112,4 +133,5 @@ main()
   test02();
   test03();
   test04();
+  test05();
 }
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge1_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge1_neg.cc
new file mode 100644
index 0000000..8a2bc6e
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge1_neg.cc
@@ -0,0 +1,31 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_set>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_set<int>;
+
+void
+test01()
+{
+  test_type c0{ 1, 2, 3, 5, 6 };
+  test_type c1{ 1, 2, 3, 4 };
+
+  auto it2 = c1.find(2);
+  auto it4 = c1.find(4);
+  VERIFY( *it2 == 2 );
+  VERIFY( *it4 == 4 );
+
+  c0.merge(c1);
+
+  VERIFY( *it2 == 2 );
+  VERIFY( it2 != it4 ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge2_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge2_neg.cc
new file mode 100644
index 0000000..3ac9654
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge2_neg.cc
@@ -0,0 +1,31 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_set>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_set<int>;
+
+void
+test01()
+{
+  test_type c0{ 1, 2, 3, 5, 6 };
+  test_type c1{ 1, 2, 3, 4 };
+
+  auto it2 = c1.find(2);
+  auto it4 = c1.find(4);
+  VERIFY( *it2 == 2 );
+  VERIFY( *it4 == 4 );
+
+  c0.merge(std::move(c1));
+
+  VERIFY( *it2 == 2 );
+  VERIFY( it2 != it4 ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge3_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge3_neg.cc
new file mode 100644
index 0000000..7e93b55
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge3_neg.cc
@@ -0,0 +1,33 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_set>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_set<int>;
+
+void
+test01()
+{
+  test_type c0{ 1, 2, 3, 5, 6, 7 };
+  std::unordered_multiset<int> c1{ 1, 1, 2, 2, 3, 3, 4, 4, 5 };
+
+  auto it1 = c1.find(1);
+  auto it41 = c1.find(4);
+  auto it42 = it41;
+  ++it42;
+  VERIFY( *it42 == 4 );
+
+  c0.merge(c1);
+
+  VERIFY( *it1 == 1 );
+  VERIFY( c1.count(4) == 1 );
+  VERIFY( it41 != it42 ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge4_neg.cc b/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge4_neg.cc
new file mode 100644
index 0000000..14c8ff6
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_set/debug/merge4_neg.cc
@@ -0,0 +1,33 @@
+// { dg-do run { target c++17 xfail *-*-* } }
+// { dg-require-debug-mode "" }
+
+#include <unordered_set>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+using test_type = std::unordered_set<int>;
+
+void
+test01()
+{
+  test_type c0{ 1, 2, 3, 5, 6, 7 };
+  std::unordered_multiset<int> c1{ 1, 1, 2, 2, 3, 3, 4, 4, 5 };
+
+  auto it1 = c1.find(1);
+  auto it41 = c1.find(4);
+  auto it42 = it41;
+  ++it42;
+  VERIFY( *it42 == 4 );
+
+  c0.merge(std::move(c1));
+
+  VERIFY( *it1 == 1 );
+  VERIFY( c1.count(4) == 1 );
+  VERIFY( it41 != it42 ); // Invalid iterator.
+}
+
+int
+main()
+{
+  test01();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_set/modifiers/merge.cc b/libstdc++-v3/testsuite/23_containers/unordered_set/modifiers/merge.cc
index c9c8a60..0e184b1 100644
--- a/libstdc++-v3/testsuite/23_containers/unordered_set/modifiers/merge.cc
+++ b/libstdc++-v3/testsuite/23_containers/unordered_set/modifiers/merge.cc
@@ -17,6 +17,7 @@
 
 // { dg-do run { target c++17 } }
 
+#include <string>
 #include <unordered_set>
 #include <algorithm>
 #include <testsuite_hooks.h>
@@ -125,10 +126,52 @@ test03()
   VERIFY( c2.empty() );
 }
 
+void
+test04()
+{
+  const std::unordered_set<std::string> c0{ "abcd", "efgh", "ijkl", };
+  std::unordered_set<std::string> c1 = c0;
+  std::unordered_multiset<std::string> c2( c0.begin(), c0.end() );
+  c1.merge(c2);
+  VERIFY( c1 == c0 );
+  VERIFY( equal_elements(c2, c0) );
+
+  c1.clear();
+  c1.merge(c2);
+  VERIFY( c1 == c0 );
+  VERIFY( c2.empty() );
+
+  c2.merge(c1);
+  VERIFY( c1.empty() );
+  VERIFY( equal_elements(c2, c0) );
+
+  c1 = c0;
+  c2.merge(c1);
+  VERIFY( c1.empty() );
+  VERIFY( c2.size() == (2 * c0.size()) );
+  VERIFY( c2.count("abcd") == 2 );
+  VERIFY( c2.count("efgh") == 2 );
+  VERIFY( c2.count("ijkl") == 2 );
+
+  c1.merge(c2);
+  VERIFY( c1 == c0 );
+  VERIFY( equal_elements(c2, c0) );
+
+  c1.merge(std::move(c2));
+  VERIFY( c1 == c0 );
+  VERIFY( equal_elements(c2, c0) );
+
+  c1.clear();
+  c1.merge(std::move(c2));
+  VERIFY( c1 == c0 );
+  VERIFY( c2.empty() );
+}
+
 int
 main()
 {
   test01();
   test02();
   test03();
+  test04();
 }
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/capacity/29134.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/capacity/29134.cc
index 43f3343..da5e6b4 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/bool/capacity/29134.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/capacity/29134.cc
@@ -26,12 +26,7 @@ void test01()
 {
   using std::vector;
   using std::numeric_limits;
-
-#ifdef _GLIBCXX_DEBUG
-  using std::_GLIBCXX_STD_C::_S_word_bit;
-#else
   using std::_S_word_bit;
-#endif
 
   // Actually, vector<bool> is special, see libstdc++/31370.
   vector<bool> vb;
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/capacity/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/capacity/constexpr.cc
new file mode 100644
index 0000000..52b10b6
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/capacity/constexpr.cc
@@ -0,0 +1,115 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <testsuite_hooks.h>
+
+constexpr std::size_t
+capacity_for(std::size_t n)
+{
+  std::size_t N = std::vector<bool>(1).capacity();
+  if (auto r = n % N)
+    return n - r + N;
+  return n;
+}
+
+constexpr bool
+test_empty()
+{
+  std::vector<bool> v;
+  VERIFY( v.empty() );
+  v = {1};
+  VERIFY( !v.empty() );
+
+  return true;
+}
+
+static_assert( test_empty() );
+
+constexpr bool
+test_size()
+{
+  std::vector<bool> v;
+  VERIFY( v.size() == 0 );
+  v = {1};
+  VERIFY( v.size() == 1 );
+
+  VERIFY( v.max_size() != 0 );
+
+  return true;
+}
+
+static_assert( test_size() );
+
+constexpr bool
+test_capacity()
+{
+  std::vector<bool> v;
+  VERIFY( v.size() == 0 );
+  VERIFY( v.capacity() == v.size() );
+  v = {false, false, false};
+  VERIFY( v.size() == 3 );
+  VERIFY( v.capacity() >= v.size() );
+
+  return true;
+}
+
+static_assert( test_capacity() );
+
+constexpr bool
+test_resize()
+{
+  std::vector<bool> v;
+  v.reserve(9);
+  VERIFY( v.size() == 0 );
+  VERIFY( v.capacity() == capacity_for(9) );
+  v.resize(5);
+  VERIFY( v.size() == 5 );
+  VERIFY( v.capacity() == capacity_for(9) );
+  v.resize(900, true);
+  VERIFY( v.size() == 900 );
+  VERIFY( v.capacity() == capacity_for(900) );
+  VERIFY( v[10] == true );
+
+  return true;
+}
+
+static_assert( test_resize() );
+
+constexpr bool
+test_reserve()
+{
+  std::vector<bool> v;
+  v.reserve(9);
+  VERIFY( v.size() == 0 );
+  VERIFY( v.capacity() == capacity_for(9) );
+  v.resize(2);
+  VERIFY( v.size() == 2 );
+  VERIFY( v.capacity() == capacity_for(9) );
+  v.resize(300);
+  v.resize(100);
+  VERIFY( v.size() == 100 );
+  VERIFY( v.capacity() == capacity_for(300) );
+
+  return true;
+}
+
+static_assert( test_reserve() );
+
+constexpr bool
+test_shrink_to_fit()
+{
+  std::vector<bool> v;
+  v.reserve(9);
+  v.shrink_to_fit();
+  VERIFY( v.capacity() == 0 );
+  v.reserve(9);
+  v.resize(5);
+  v.shrink_to_fit();
+  VERIFY( v.capacity() == capacity_for(v.size()) );
+
+  return true;
+}
+
+static_assert( test_shrink_to_fit() );
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/cmp_c++20.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/cmp_c++20.cc
index 7a730df..ae2bd03 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/bool/cmp_c++20.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/cmp_c++20.cc
@@ -16,12 +16,13 @@
 // <http://www.gnu.org/licenses/>.
 
 // { dg-options "-std=gnu++2a" }
-// { dg-do run { target c++2a } }
+// { dg-do compile { target c++2a } }
+// { dg-xfail-if "not supported" { debug-mode } }
 
 #include <vector>
 #include <testsuite_hooks.h>
 
-void
+constexpr bool
 test01()
 {
   std::vector<bool> c1{ 1, 0, 1 }, c2{ 1, 0, 1, 0 }, c3{ 1, 1, 1 };
@@ -38,9 +39,11 @@ test01()
 
   static_assert( std::three_way_comparable<std::vector<bool>,
 					   std::strong_ordering> );
+
+  return true;
 }
 
-void
+constexpr bool
 test05()
 {
   // vector<bool> iterators are random access, so should support <=>
@@ -63,11 +66,9 @@ test05()
 
   static_assert( std::same_as<decltype(c.begin() <=> c.begin()),
 			      std::strong_ordering> );
-}
 
-int
-main()
-{
-  test01();
-  test05();
+  return true;
 }
+
+static_assert( test01() );
+static_assert( test05() );
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/cons/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/cons/constexpr.cc
new file mode 100644
index 0000000..edd5060
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/cons/constexpr.cc
@@ -0,0 +1,280 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
+
+template<typename T>
+struct Alloc : std::allocator<T>
+{
+  using std::allocator<T>::allocator;
+
+  constexpr explicit Alloc(int p) : personality(p) { }
+
+  template<typename U>
+    constexpr Alloc(const Alloc<U>& a) : personality(a.personality) { }
+
+  int personality = 0;
+
+  constexpr Alloc select_on_container_copy_construction() const
+  { return Alloc(-1); }
+
+  constexpr bool operator==(const Alloc& a) const noexcept
+  { return personality == a.personality; }
+};
+
+namespace default_constructor_global_scope
+{
+  constexpr std::vector<bool> v1;
+  static_assert(v1.size() == 0);
+  static_assert(v1.capacity() == 0);
+
+  constexpr std::allocator<bool> a;
+  constexpr std::vector<bool> v2(a);
+  static_assert(v2.size() == 0);
+  static_assert(v2.capacity() == 0);
+
+  constexpr Alloc<bool> aa(10);
+  constexpr std::vector<bool, Alloc<bool>> v3(aa);
+  static_assert(v3.size() == 0);
+  static_assert(v3.capacity() == 0);
+  static_assert(v3.get_allocator() == aa);
+}
+
+constexpr bool
+default_constructor_function_scope()
+{
+  // vector()
+
+  std::vector<bool> v1;
+  VERIFY(v1.size() == 0);
+  VERIFY(v1.capacity() == 0);
+
+  // vector(const Allocator&)
+
+  const std::allocator<bool> a;
+  std::vector<bool> v2(a);
+  VERIFY(v2.size() == 0);
+  VERIFY(v2.capacity() == 0);
+
+  const Alloc<bool> aa(10);
+  std::vector<bool, Alloc<bool>> v3(aa);
+  VERIFY(v3.size() == 0);
+  VERIFY(v3.capacity() == 0);
+  VERIFY(v3.get_allocator() == aa);
+
+  return true;
+}
+
+static_assert( default_constructor_function_scope() );
+
+constexpr bool
+sequence_constructors()
+{
+  // vector(size_type, const Allocator& = Allocator())
+
+  std::vector<bool> v0(0);
+  VERIFY(v0.size() == 0);
+
+  std::vector<bool> v1(1);
+  VERIFY(v1.size() == 1);
+
+  std::vector<bool> v2(2);
+  VERIFY(v2.size() == 2);
+
+  std::vector<bool> v50(50);
+  VERIFY(v50.size() == 50);
+
+  const std::allocator<bool> a;
+  std::vector<bool> a0(0, a);
+  VERIFY(a0.size() == 0);
+
+  std::vector<bool> a1(1, a);
+  VERIFY(a1.size() == 1);
+
+  std::vector<bool> a2(2, a);
+  VERIFY(a2.size() == 2);
+
+  std::vector<bool> a50(50, a);
+  VERIFY(a50.size() == 50);
+
+  const Alloc<bool> la(10);
+  std::vector<bool, Alloc<bool>> l0(0, la);
+  VERIFY(l0.size() == 0);
+  VERIFY(l0.get_allocator() == la);
+
+  std::vector<bool, Alloc<bool>> l1(1, la);
+  VERIFY(l1.size() == 1);
+  VERIFY(l1.get_allocator() == la);
+
+  std::vector<bool, Alloc<bool>> l2(2, la);
+  VERIFY(l2.size() == 2);
+  VERIFY(l2.get_allocator() == la);
+
+  std::vector<bool, Alloc<bool>> l50(50, la);
+  VERIFY(l50.size() == 50);
+  VERIFY(l50.get_allocator() == la);
+
+  // vector(size_type, const T&, const Allocator& = Allocator())
+
+  std::vector<bool> v3(3, true);
+  VERIFY(v3.size() == 3);
+  VERIFY(v3[0] == true && v3[2] == true);
+
+  std::vector<bool> a3(3, false, a);
+  VERIFY(a3.size() == 3);
+  VERIFY(a3[0] == false && a3[2] == false);
+
+  std::vector<bool, Alloc<bool>> l3(3, true, la);
+  VERIFY(l3.size() == 3);
+  VERIFY(l3[0] == true && l3[2] == true);
+  VERIFY(l3.get_allocator() == la);
+
+  return true;
+}
+
+static_assert(sequence_constructors());
+
+constexpr bool
+iterator_range_constructor()
+{
+  // vector(InputIterator, InputIterator, const Allocator& = Allocator())
+
+  short range[3] = { true, false, true };
+
+  std::vector<bool> v0(std::begin(range), std::end(range));
+  VERIFY(v0.size() == std::size(range));
+  VERIFY(v0[0] == true && v0[1] == false && v0[2] == true);
+
+  const Alloc<bool> a(5);
+  std::vector<bool, Alloc<bool>> l0(std::begin(range), std::end(range), a);
+  VERIFY(l0.size() == std::size(range));
+  VERIFY(l0.get_allocator() == a);
+  VERIFY(l0[0] == true && l0[1] == false && l0[2] == true);
+
+  struct input_iterator
+  {
+    using iterator_category = std::input_iterator_tag;
+    using value_type = bool;
+    using pointer = const bool*;
+    using reference = bool;
+    using difference_type = int;
+
+    constexpr input_iterator() : val(0) { }
+    constexpr input_iterator(int i) : val(i) { }
+
+    constexpr input_iterator& operator++() { --val; return *this; }
+    constexpr input_iterator operator++(int) { return {val--}; }
+
+    constexpr bool operator*() const { return val % 2; }
+    constexpr const bool* operator->() const { return nullptr; }
+
+    constexpr bool operator==(const input_iterator&) const = default;
+
+    int val;
+  };
+
+  std::vector<bool> v1(input_iterator(3), input_iterator());
+  VERIFY(v1.size() == 3);
+  VERIFY(v1[0] == true && v1[1] == false && v1[2] == true);
+
+  std::vector<bool, Alloc<bool>> l1(input_iterator(2), input_iterator(), a);
+  VERIFY(l1.size() == 2);
+  VERIFY(l1.get_allocator() == a);
+  VERIFY(l1[0] == false && l1[1] == true);
+
+  return true;
+}
+
+static_assert(iterator_range_constructor());
+
+constexpr bool
+initializer_list_constructor()
+{
+  // vector(initializer_list<T>, const Allocator& = Allocator())
+
+  std::vector<bool> v0({ false, true, false });
+  VERIFY(v0.size() == 3);
+  VERIFY(v0[0] == false && v0[1] == true && v0[2] == false);
+
+  const Alloc<bool> a(5);
+  std::vector<bool, Alloc<bool>> l0({ true, false, false }, a);
+  VERIFY(l0.size() == 3);
+  VERIFY(l0.get_allocator() == a);
+  VERIFY(l0[0] == true && l0[1] == false && l0[2] == false);
+
+  return true;
+}
+
+static_assert(initializer_list_constructor());
+
+constexpr bool
+copy_constructor()
+{
+  const std::vector<bool> v0({ 1, 0, 0, 1, 0, 1, 1, 0 });
+  const std::vector<bool, Alloc<bool>> l0({ 0, 0, 1, 1, 0, 1, 0, 1, 1, 1 });
+
+  // vector(const vector&)
+
+  std::vector<bool> v1(v0);
+  VERIFY( v1.size() == v0.size() );
+  VERIFY( v1[0] == v0[0] && v1[1] == v0[1] && v1[2] == v0[2] );
+  VERIFY( v1.get_allocator() == v0.get_allocator() );
+
+  const Alloc<bool> as(6);
+  std::vector<bool, Alloc<bool>> s1(3, true, as);
+  std::vector<bool, Alloc<bool>> s2(s1);
+  VERIFY( s2.size() == s1.size() );
+  VERIFY( s2.get_allocator().personality == -1 );
+
+  // vector(const vector&, const Allocator&)
+
+  const Alloc<bool> a(6);
+  std::vector<bool, Alloc<bool>> l1(l0, a);
+  VERIFY( l1.size() == l0.size() );
+  VERIFY( l1[0] == l0[0] && l1[1] == l0[1] && l1[2] == l0[2] );
+  VERIFY( l1.get_allocator() == a );
+  VERIFY( l1.get_allocator() != l0.get_allocator() );
+
+  return true;
+}
+
+static_assert(copy_constructor());
+
+constexpr bool
+move_constructor()
+{
+  const std::vector<bool> v0({ 1, 0, 0, 1, 0, 1, 1, 0 });
+  const std::vector<bool, Alloc<bool>> l0({ 0, 0, 1, 1, 0, 1, 0, 1, 1, 1 });
+
+  // vector(const vector&)
+
+  std::vector<bool> v1(v0);
+  std::vector<bool> v2(std::move(v1));
+  VERIFY( v2.size() == v0.size() );
+  VERIFY( v1.empty() );
+  VERIFY( v2[0] == v0[0] && v2[1] == v0[1] && v2[2] == v0[2] );
+  VERIFY( v2.get_allocator() == v0.get_allocator() );
+
+  // vector(const vector&, const Allocator&)
+
+  const Alloc<bool> a(6);
+  std::vector<bool, Alloc<bool>> l1(l0);
+  std::vector<bool, Alloc<bool>> l2(std::move(l1), a);
+  VERIFY( l2.size() == l0.size() );
+  VERIFY( l2[0] == l0[0] && l2[1] == l0[1] && l2[2] == l0[2] );
+  VERIFY( l2.get_allocator() == a );
+  VERIFY( l2.get_allocator() != l0.get_allocator() );
+
+  std::vector<bool, Alloc<bool>> l3(std::move(l2), a);
+  VERIFY( l3.size() == l0.size() );
+  VERIFY( l3[0] == l0[0] && l3[1] == l0[1] && l3[2] == l0[2] );
+  VERIFY( l3.get_allocator() == a );
+  VERIFY( l3.get_allocator() == l2.get_allocator() );
+
+  return true;
+}
+
+static_assert(move_constructor());
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/element_access/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/element_access/constexpr.cc
new file mode 100644
index 0000000..6fa8758
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/element_access/constexpr.cc
@@ -0,0 +1,102 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <testsuite_hooks.h>
+
+constexpr bool
+test_iterators()
+{
+  std::vector<bool> v;
+  VERIFY( v.begin() == v.end() );
+  v.reserve(1);
+  VERIFY( v.begin() == v.end() );
+  v.resize(2);
+  VERIFY( v.begin() != v.end() );
+  VERIFY( v.cbegin() == v.begin() );
+  VERIFY( v.crbegin() == v.rbegin() );
+  VERIFY( v.cend() == v.end() );
+  VERIFY( v.crend() == v.rend() );
+
+  auto it = v.begin();
+  VERIFY( *it == v.front() );
+  VERIFY( it++ == v.begin() );
+  VERIFY( ++it == v.end() );
+  VERIFY( (it - 2) == v.begin() );
+  it -= 2;
+  it += 1;
+  VERIFY( (it + 1) == v.end() );
+
+  auto rit = v.rbegin();
+  VERIFY( *rit == v.back() );
+  VERIFY( rit++ == v.rbegin() );
+  VERIFY( ++rit == v.rend() );
+  VERIFY( (rit - 2) == v.rbegin() );
+  rit -= 2;
+  rit += 1;
+  VERIFY( (rit + 1) == v.rend() );
+
+  return true;
+}
+
+static_assert(test_iterators());
+
+constexpr bool
+test_access()
+{
+  std::vector<bool> v{1, 1, 0, 0, 1, 0, 1, 0, 1};
+  std::vector<bool>::reference r1 = v.at(1);
+  VERIFY( r1 );
+  std::vector<bool>::reference r2 = v[2];
+  VERIFY( ! r2 );
+  r1 = r2;
+  VERIFY( ! r1 );
+  VERIFY( ! v[1] );
+  r2 = true;
+  VERIFY( r2 );
+  VERIFY( v[2] );
+
+  const auto& vc = v;
+  VERIFY( vc.at(1) == false );
+  VERIFY( vc.at(1) == v.at(1) );
+  VERIFY( vc.at(1) == vc[1] );
+  VERIFY( vc.front() == vc[0] );
+  VERIFY( vc.back() == vc[2] );
+
+  return true;
+}
+
+static_assert(test_access());
+
+template<typename T = bool>
+  constexpr std::false_type
+  access_empty() { return {}; }
+
+template<typename T = bool>
+  requires (std::bool_constant<(std::vector<T>().at(0), true)>::value)
+  constexpr std::true_type
+  access_empty() { return {}; }
+
+template<typename T = bool>
+  requires (std::bool_constant<(std::vector<T>().back(), true)>::value)
+  constexpr std::true_type
+  access_empty() { return {}; }
+
+static_assert( ! access_empty() );
+
+template<typename T = bool>
+  constexpr std::false_type
+  access_empty_front() { return {}; }
+
+template<typename T = bool>
+  requires (std::bool_constant<(std::vector<T>()[0], true)>::value)
+  constexpr std::true_type
+  access_empty_front() { return {}; }
+
+template<typename T = bool>
+  requires (std::bool_constant<(std::vector<T>().front(), true)>::value)
+  constexpr std::true_type
+  access_empty_front() { return {}; }
+
+static_assert( ! access_empty_front() ); // { dg-error "ambiguous" "PR 103191" { target { ! debug-mode } } }
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/assign/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/assign/constexpr.cc
new file mode 100644
index 0000000..02faa02
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/assign/constexpr.cc
@@ -0,0 +1,216 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <testsuite_hooks.h>
+
+template<typename T>
+struct Alloc : std::allocator<T>
+{
+  using std::allocator<T>::allocator;
+
+  constexpr explicit Alloc(int p) : personality(p) { }
+
+  template<typename U>
+    constexpr Alloc(const Alloc<U>& a) : personality(a.personality) { }
+
+  int personality = 0;
+
+  using propagate_on_container_move_assignment = std::false_type;
+
+  constexpr bool operator==(const Alloc& a) const noexcept
+  { return personality == a.personality; }
+};
+
+constexpr std::size_t
+capacity_for(std::size_t n)
+{
+  std::size_t N = std::vector<bool>(1).capacity();
+  if (auto r = n % N)
+    return n - r + N;
+  return n;
+}
+
+constexpr bool
+copy_assign()
+{
+  // vector::operator=(const vector&)
+
+  std::vector<bool> v1, v2;
+  v1 = v1;
+  v2 = v1;
+  VERIFY(v2.size() == 0);
+  VERIFY(v2.capacity() == 0);
+
+  for (int i = 0; i < 10; ++i)
+    v1.push_back(i);
+  v2 = v1;
+  VERIFY(v2.size() == 10);
+  v2.reserve(50);
+  v1.push_back(1);
+  v2 = v1;
+  VERIFY(v2.size() == 11);
+  VERIFY(v2.capacity() == capacity_for(50));
+
+  std::vector<bool, Alloc<bool>> va1(Alloc<bool>(1)), va2(Alloc<bool>(2));
+  va2 = va1;
+  VERIFY( va2.get_allocator().personality == 2 );
+  va1.push_back(999);
+  va2 = va1;
+  VERIFY( va2.get_allocator().personality == 2 );
+
+  return true;
+}
+
+static_assert( copy_assign() );
+
+constexpr bool
+move_assign()
+{
+  // vector::operator=(const vector&)
+
+  std::vector<bool> v1, v2;
+  v1 = std::move(v1);
+  v2 = std::move(v1);
+  VERIFY(v2.size() == 0);
+  VERIFY(v2.capacity() == 0);
+
+  for (int i = 0; i < 10; ++i)
+    v1.push_back(i);
+  v2 = std::move(v1);
+  VERIFY(v2.size() == 10);
+  v2.reserve(50);
+  v1.push_back(1);
+  v2 = std::move(v1);
+  VERIFY(v2.size() == 1);
+  VERIFY(v1.capacity() == 0);
+  VERIFY(v2.capacity() == capacity_for(1));
+
+  std::vector<bool, Alloc<bool>> va1(Alloc<bool>(1)), va2(Alloc<bool>(2));
+  va2 = std::move(va1);
+  VERIFY( va2.get_allocator().personality == 2 );
+  va1.push_back(9);
+  va1.push_back(99);
+  va1.push_back(999);
+  va1.push_back(9999);
+  va2 = std::move(va1);
+  va2 = std::move(va1);
+  VERIFY( va2.get_allocator().personality == 2 );
+
+  return true;
+}
+
+static_assert( move_assign() );
+
+constexpr bool
+initializer_list_assign()
+{
+  std::vector<bool> v1;
+  v1 = {1, 0, 1};
+  VERIFY( v1.size() == 3 );
+  VERIFY( v1.capacity() == capacity_for(3) );
+  v1 = {1, 0};
+  VERIFY( v1.size() == 2 );
+  VERIFY( v1.capacity() == capacity_for(3) );
+  v1 = {1, 1, 0, 1, 1, 1, 0, 1, 1};
+  VERIFY( v1.size() == 9 );
+  VERIFY( v1.capacity() == capacity_for(9) );
+
+  std::vector<bool, Alloc<bool>> va1(Alloc<bool>(111));
+  va1 = {1, 0, 0};
+  VERIFY( va1.get_allocator().personality == 111 );
+
+  return true;
+}
+
+static_assert( initializer_list_assign() );
+
+constexpr bool
+assign_iterator_range()
+{
+  std::vector<bool> v;
+
+  int arr[] = { 1, 2, 3, 4 };
+  v.assign(arr, arr+3);
+  VERIFY( v.size() == 3 );
+  v.reserve(5);
+  v.assign(arr, arr+3);
+  VERIFY( v.capacity() == capacity_for(5) );
+
+  struct input_iterator
+  {
+    using iterator_category = std::input_iterator_tag;
+    using value_type = bool;
+    using pointer = const bool*;
+    using reference = bool;
+    using difference_type = int;
+
+    constexpr input_iterator() : val(0) { }
+    constexpr input_iterator(int i) : val(i) { }
+
+    constexpr input_iterator& operator++() { --val; return *this; }
+    constexpr input_iterator operator++(int) { return {val--}; }
+
+    constexpr bool operator*() const { return val % 2; }
+    constexpr const bool* operator->() const { return nullptr; }
+
+    constexpr bool operator==(const input_iterator&) const = default;
+
+    int val;
+  };
+
+  v.assign(input_iterator(9), input_iterator());
+  VERIFY( v.size() == 9 );
+
+  return true;
+}
+
+static_assert( assign_iterator_range() );
+
+constexpr bool
+assign_value()
+{
+  std::vector<bool> v1;
+  v1.assign(3, 8);
+  VERIFY( v1.size() == 3 );
+  VERIFY( v1.capacity() == capacity_for(3) );
+  v1.assign(2, 9);
+  VERIFY( v1.size() == 2 );
+  VERIFY( v1.capacity() == capacity_for(3) );
+  v1.assign(9, 10);
+  VERIFY( v1.size() == 9 );
+  VERIFY( v1.capacity() == capacity_for(9) );
+
+  std::vector<bool, Alloc<bool>> va1(Alloc<bool>(111));
+  va1.assign(2, 9);
+  VERIFY( va1.size() == 2 );
+  VERIFY( va1.get_allocator().personality == 111 );
+
+  return true;
+}
+
+static_assert( assign_value() );
+
+constexpr bool
+assign_initializer_list()
+{
+  std::vector<bool> v1;
+  v1.assign({0, 1, 0});
+  VERIFY( v1.size() == 3 );
+  VERIFY( v1.capacity() == capacity_for(3) );
+  v1.assign({1, 0});
+  VERIFY( v1.size() == 2 );
+  VERIFY( v1.capacity() == capacity_for(3) );
+  v1.assign({1, 0, 0, 1, 1, 0, 0, 1, 1});
+  VERIFY( v1.size() == 9 );
+  VERIFY( v1.capacity() == capacity_for(9) );
+
+  std::vector<bool, Alloc<bool>> va1(Alloc<bool>(111));
+  va1.assign({1, 1, 1});
+  VERIFY( va1.get_allocator().personality == 111 );
+
+  return true;
+}
+
+static_assert( assign_initializer_list() );
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/constexpr.cc
new file mode 100644
index 0000000..b27df2b
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/constexpr.cc
@@ -0,0 +1,248 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <testsuite_hooks.h>
+
+template<typename T>
+struct Alloc : std::allocator<T>
+{
+  using std::allocator<T>::allocator;
+
+  constexpr explicit Alloc(int p) : personality(p) { }
+
+  template<typename U>
+    constexpr Alloc(const Alloc<U>& a) : personality(a.personality) { }
+
+  int personality = 0;
+
+  constexpr bool operator==(const Alloc& a) const noexcept
+  { return personality == a.personality; }
+};
+
+constexpr bool
+test_push_back()
+{
+  std::vector<bool> v;
+  std::vector<bool>::reference r = v.emplace_back("");
+  VERIFY( r == true );
+  v.emplace_back(r);
+  VERIFY( v.back() == true );
+  v.emplace_back(v.front());
+  VERIFY( v.back() == true );
+  v.resize(64);
+  v.emplace_back(v.back());
+  VERIFY( v.back() == false );
+  VERIFY( v.size() == 65 );
+  v.emplace_back(8);
+  VERIFY( v.size() == 66 );
+  VERIFY( v.back() == true );
+  v.emplace_back();
+  VERIFY( v.size() == 67 );
+  VERIFY( v.back() == false );
+
+  v.pop_back();
+  VERIFY( v.size() == 66 );
+  VERIFY( v.back() == true );
+  for (int i = 0, n = v.size(); i < n; ++i)
+    v.pop_back();
+  VERIFY( v.empty() );
+
+  v.push_back(true);
+  for (std::size_t i = 0, c = v.capacity(); i <= c; ++i)
+    v.push_back(v.front());
+  VERIFY( v.capacity() > v.size() );
+
+  std::vector<bool, Alloc<bool>> va;
+  va.push_back(true);
+  va.push_back(va.front());
+  VERIFY( va.size() == 2 );
+
+  return true;
+}
+
+static_assert( test_push_back() );
+
+template<typename T = bool>
+constexpr std::false_type
+pop_back_empty() { return {}; }
+
+template<typename T = bool>
+requires (std::bool_constant<(std::vector<T>().pop_back(), true)>::value)
+constexpr std::true_type
+pop_back_empty() { return {}; }
+
+static_assert( ! pop_back_empty() );
+
+constexpr bool
+test_insert_erase()
+{
+  std::vector<bool> v;
+
+  // vector::emplace(const_iterator, Args&&...)
+  auto p = v.emplace(v.begin());
+  VERIFY( p == v.begin() );
+  p = v.emplace(v.end(), '7');
+  VERIFY( p == --v.end() );
+
+  // vector::insert(const_iterator, const T&)
+  p = v.insert(v.begin(), *p);
+  VERIFY( p == v.begin() );
+  VERIFY( *p );
+  // vector::insert(const_iterator, T&&)
+  p = v.insert(v.end(), 1);
+  VERIFY( p == --v.end() );
+  v.insert(p, v.front());
+  v.insert(v.end(), v.back());
+  VERIFY( v.size() == 6 );
+  v.insert(v.end(), true);
+  VERIFY( v.size() == 7 );
+  VERIFY( v.back() == true );
+
+  // vector::insert(const_iterator, size_type, const T&)
+  v.insert(v.begin(), 2, v.front());
+  v.insert(v.end(), 3, 99);
+  VERIFY( v.size() == 12 );
+
+  struct input_iterator
+  {
+    using iterator_category = std::input_iterator_tag;
+    using value_type = bool;
+    using pointer = const bool*;
+    using reference = bool;
+    using difference_type = int;
+
+    constexpr input_iterator() : val(0) { }
+    constexpr input_iterator(int i) : val(i) { }
+
+    constexpr input_iterator& operator++() { --val; return *this; }
+    constexpr input_iterator operator++(int) { return {val--}; }
+
+    constexpr bool operator*() const { return val % 2; }
+    constexpr const bool* operator->() const { return nullptr; }
+
+    constexpr bool operator==(const input_iterator&) const = default;
+
+    int val;
+  };
+
+  // vector::insert(const_iterator, Iter, Iter);
+  v.insert(v.begin() + 2, input_iterator(), input_iterator());
+  VERIFY( v.size() == 12 );
+  v.insert(v.end() - 9, input_iterator(18), input_iterator());
+  VERIFY( v.size() == 30 );
+  short a[] = { false, true };
+  v.insert(v.end(), a, a + 2);
+  VERIFY( v.size() == 32 );
+
+  // vector::insert(const_iterator, initializer_list<T>)
+  v.insert(v.begin(), {1,1,1});
+  VERIFY( v.size() == 35 );
+
+  // vector::erase(const_iterator)
+  v.erase(v.end() - 1);
+  VERIFY( v.size() == 34 );
+  VERIFY( v.back() == false );
+  v.erase(v.begin());
+  v.erase(v.begin() + 1);
+  v.erase(v.end() - 1);
+  VERIFY( v.size() == 31 );
+
+  // vector::erase(const_iterator, const_iterator)
+  v.erase(v.begin(), v.begin());
+  v.erase(v.end(), v.end());
+  v.erase(v.begin(), v.begin() + 1);
+  VERIFY( v.size() == 30 );
+  v.erase(v.begin() + 2, v.end() - 2);
+  VERIFY( v.size() == 4 );
+  v.erase(v.begin(), v.end());
+  VERIFY( v.empty() );
+  v.erase( v.begin(), v.begin() );
+  VERIFY( v.empty() );
+
+  v.insert(v.end(), 99);
+  for (std::size_t i = 0, c = v.capacity(); i <= c; ++i)
+    v.insert(v.end() - 1, v.front());
+  VERIFY( v.capacity() > v.size() );
+  v.insert(v.end(), 999);
+  for (std::size_t i = 0, c = v.capacity(); i <= c; ++i)
+    v.insert(v.begin(), v.front());
+
+  std::vector<bool, Alloc<bool>> va;
+  va.insert(va.begin(), 99);
+  va.insert(va.begin(), va.front());
+  VERIFY( va.size() == 2 );
+  va.erase(va.begin());
+
+  return true;
+}
+
+static_assert( test_insert_erase() );
+
+constexpr std::size_t
+capacity_for(std::size_t n)
+{
+  std::size_t N = std::vector<bool>(1).capacity();
+  if (auto r = n % N)
+    return n - r + N;
+  return n;
+}
+
+constexpr bool
+test_clear()
+{
+  std::vector<bool> v0;
+  v0.clear();
+  VERIFY( v0.size() == 0 );
+  VERIFY( v0.capacity() == 0 );
+
+  std::vector<bool> v{1, 0, 0};
+  v.clear();
+  VERIFY( v.size() == 0 );
+  VERIFY( v.capacity() == capacity_for(3) );
+
+  std::vector<bool, Alloc<bool>> va;
+  va.clear();
+  va.push_back(1);
+  va.clear();
+  va.clear();
+
+  return true;
+}
+
+static_assert( test_clear() );
+
+constexpr bool
+test_flip()
+{
+  std::vector<bool> v{1, 0, 0, 1, 0, 1, 1, 0};
+  v.flip();
+  VERIFY( !v[0] && v[1] && v[2] && !v[3] && v[4] && !v[5] && !v[6] && v[7] );
+  v[2].flip();
+  VERIFY( !v[2] );
+  v[2].flip();
+  VERIFY( v[2] );
+
+  return true;
+}
+
+static_assert( test_flip() );
+
+constexpr bool
+test_erasure()
+{
+  std::vector<bool> e{true,true,true,false,true,false};
+
+  auto n = std::erase(e, false);
+  VERIFY( n == 2 );
+  VERIFY( e.size() == 4 );
+  e[2] = false;
+  n = std::erase_if(e, [](std::vector<bool>::reference val) { return !val; });
+  VERIFY( n == 1 );
+  VERIFY( e.size() == 3 );
+
+  return true;
+}
+
+static_assert( test_erasure() );
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/31370.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/31370.cc
index a2b8c1a..c679446 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/31370.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/31370.cc
@@ -24,11 +24,7 @@
 #include <stdexcept>
 #include <testsuite_hooks.h>
 
-#ifdef _GLIBCXX_DEBUG
-  using std::_GLIBCXX_STD_C::_S_word_bit;
-#else
-  using std::_S_word_bit;
-#endif
+using std::_S_word_bit;
 
 inline void
 check_cap_ge_size(const std::vector<bool>& x)
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/swap/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/swap/constexpr.cc
new file mode 100644
index 0000000..ba444d0
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/swap/constexpr.cc
@@ -0,0 +1,66 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <utility>
+#include <testsuite_hooks.h>
+
+constexpr bool
+test_std_swap()
+{
+  std::vector<bool> v1 = {true, false, true};
+  std::vector<bool> v2 = {false, true};
+
+  std::swap(v1, v2);
+
+  VERIFY(v1.size() == 2);
+  VERIFY(v1.at(0) == false);
+  VERIFY(v1.at(1) == true);
+
+  VERIFY(v2.size() == 3);
+  VERIFY(v2[0]);
+  VERIFY(!v2[1]);
+  VERIFY(v2[2]);
+
+  return true;
+}
+
+static_assert(test_std_swap());
+
+constexpr bool
+test_member_swap()
+{
+  std::vector<bool> v1 = {true, false, true};
+  std::vector<bool> v2 = {false, true};
+
+  v1.swap(v2);
+
+  VERIFY(v1.size() == 2);
+  VERIFY(v1.at(0) == false);
+  VERIFY(v1.at(1) == true);
+
+  VERIFY(v2.size() == 3);
+  VERIFY(v2[0]);
+  VERIFY(!v2[1]);
+  VERIFY(v2[2]);
+
+  return true;
+}
+
+static_assert(test_member_swap());
+
+constexpr bool
+test_reference_swap()
+{
+  std::vector<bool> v1 = {true, false, true};
+  std::vector<bool>::swap(v1[0], v1[1]);
+
+  VERIFY(v1[0] == false);
+  VERIFY(v1[1] == true);
+  VERIFY(v1[2] == true);
+
+  return true;
+}
+
+static_assert(test_reference_swap());
diff --git a/libstdc++-v3/testsuite/23_containers/vector/capacity/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/capacity/constexpr.cc
new file mode 100644
index 0000000..de9d0b0
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/capacity/constexpr.cc
@@ -0,0 +1,101 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <testsuite_hooks.h>
+
+constexpr bool
+test_empty()
+{
+  std::vector<int> v;
+  VERIFY( v.empty() );
+  v = {1};
+  VERIFY( !v.empty() );
+
+  return true;
+}
+
+static_assert( test_empty() );
+
+constexpr bool
+test_size()
+{
+  std::vector<int> v;
+  VERIFY( v.size() == 0 );
+  v = {1};
+  VERIFY( v.size() == 1 );
+
+  VERIFY( v.max_size() != 0 );
+
+  return true;
+}
+
+static_assert( test_size() );
+
+constexpr bool
+test_capacity()
+{
+  std::vector<int> v;
+  VERIFY( v.size() == 0 );
+  VERIFY( v.capacity() == v.size() );
+  v = {1, 2, 3};
+  VERIFY( v.size() == 3 );
+  VERIFY( v.capacity() == v.size() );
+
+  return true;
+}
+
+static_assert( test_capacity() );
+
+constexpr bool
+test_resize()
+{
+  std::vector<int> v;
+  v.reserve(9);
+  VERIFY( v.size() == 0 );
+  VERIFY( v.capacity() == 9 );
+  v.resize(5);
+  VERIFY( v.size() == 5 );
+  VERIFY( v.capacity() == 9 );
+  v.resize(15, 6);
+  VERIFY( v.size() == 15 );
+  VERIFY( v[10] == 6 );
+
+  return true;
+}
+
+static_assert( test_resize() );
+
+constexpr bool
+test_reserve()
+{
+  std::vector<int> v;
+  v.reserve(9);
+  VERIFY( v.size() == 0 );
+  VERIFY( v.capacity() == 9 );
+  v.resize(2);
+  VERIFY( v.size() == 2 );
+  VERIFY( v.capacity() == 9 );
+
+  return true;
+}
+
+static_assert( test_reserve() );
+
+constexpr bool
+test_shrink_to_fit()
+{
+  std::vector<int> v;
+  v.reserve(9);
+  v.shrink_to_fit();
+  VERIFY( v.capacity() == 0 );
+  v.reserve(9);
+  v.resize(5);
+  v.shrink_to_fit();
+  VERIFY( v.capacity() == v.size() );
+
+  return true;
+}
+
+static_assert( test_shrink_to_fit() );
diff --git a/libstdc++-v3/testsuite/23_containers/vector/cmp_c++20.cc b/libstdc++-v3/testsuite/23_containers/vector/cmp_c++20.cc
index 0ffacb9..8f4a264 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/cmp_c++20.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/cmp_c++20.cc
@@ -16,12 +16,13 @@
 // <http://www.gnu.org/licenses/>.
 
 // { dg-options "-std=gnu++2a" }
-// { dg-do run { target c++2a } }
+// { dg-do compile { target c++2a } }
+// { dg-xfail-if "not supported" { debug-mode } }
 
 #include <vector>
 #include <testsuite_hooks.h>
 
-void
+constexpr bool
 test01()
 {
   std::vector<int> c1{ 1, 2, 3 }, c2{ 1, 2, 3, 4 }, c3{ 1, 2, 4 };
@@ -47,24 +48,26 @@ test01()
 
   struct E
   {
-    bool operator==(E) { return true; }
+    constexpr bool operator==(E) const { return true; }
   };
   static_assert( ! std::totally_ordered<std::vector<E>> );
   static_assert( ! std::three_way_comparable<E> );
   static_assert( ! std::three_way_comparable<std::vector<E>> );
+
+  return true;
 }
 
-void
+constexpr bool
 test02()
 {
   struct W
   {
     int value = 0;
 
-    bool operator==(W rhs) const noexcept
+    constexpr bool operator==(W rhs) const noexcept
     { return (value | 1) == (rhs.value | 1); }
 
-    std::weak_ordering
+    constexpr std::weak_ordering
     operator<=>(W rhs) const noexcept
     { return (value | 1) <=> (rhs.value | 1); }
   };
@@ -75,23 +78,25 @@ test02()
   static_assert( std::same_as<decltype(c1 <=> c1), std::weak_ordering> );
   VERIFY( c1 == c2 );
   VERIFY( std::is_eq(c1 <=> c2) );
+
+  return true;
 }
 
-void
+constexpr bool
 test03()
 {
   struct P
   {
     int value = 0;
 
-    bool operator==(P rhs) const noexcept
+    constexpr bool operator==(P rhs) const noexcept
     {
       if (value < 0 || rhs.value < 0)
 	return false;
       return value == rhs.value;
     }
 
-    std::partial_ordering
+    constexpr std::partial_ordering
     operator<=>(P rhs) const noexcept
     {
       if (value < 0 || rhs.value < 0)
@@ -106,16 +111,18 @@ test03()
   static_assert( std::three_way_comparable<P> );
   static_assert( std::same_as<decltype(c <=> c), std::partial_ordering> );
   VERIFY( (c <=> c) == std::partial_ordering::unordered );
+
+  return true;
 }
 
-void
+constexpr bool
 test04()
 {
   struct L
   {
     int value = 0;
 
-    bool operator<(L rhs) const noexcept { return value < rhs.value; }
+    constexpr bool operator<(L rhs) const noexcept { return value < rhs.value; }
   };
 
   static_assert( std::totally_ordered<std::vector<L>> );
@@ -123,9 +130,11 @@ test04()
   std::vector<L> c{ {1}, {2}, {3} }, d{ {1}, {2}, {3}, {4} };
   static_assert( std::same_as<decltype(c <=> c), std::weak_ordering> );
   VERIFY( std::is_lt(c <=> d) );
+
+  return true;
 }
 
-void
+constexpr bool
 test05()
 {
   // vector iterators are random access, so should support <=>
@@ -148,14 +157,12 @@ test05()
 
   static_assert( std::same_as<decltype(c.begin() <=> c.begin()),
 			      std::strong_ordering> );
-}
 
-int
-main()
-{
-  test01();
-  test02();
-  test03();
-  test04();
-  test05();
+  return true;
 }
+
+static_assert( test01() );
+static_assert( test02() );
+static_assert( test03() );
+static_assert( test04() );
+static_assert( test05() );
diff --git a/libstdc++-v3/testsuite/23_containers/vector/cons/89164.cc b/libstdc++-v3/testsuite/23_containers/vector/cons/89164.cc
index 302af9c..4f8396f 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/cons/89164.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/cons/89164.cc
@@ -31,20 +31,18 @@ void test01()
 {
   X x[1];
   // Should not be able to create vector using uninitialized_copy:
-  std::vector<X> v1{x, x+1};	// { dg-error "here" }
-
-  // Should not be able to create vector using uninitialized_fill_n:
-  std::vector<X> v2{2u, X{}};	// { dg-error "here" }
+  std::vector<X> v1{x, x+1};	// { dg-error "here" "" { target c++17_down } }
+  // { dg-error "deleted function 'X::X" "" { target c++20 } 0 }
 }
 
 void test02()
 {
-#if __cplusplus >= 201703L
   struct Y : X { };
-  // Can create initializer_list<Y> with C++17 guaranteed copy elision,
-  // but shouldn't be able to copy from it with uninitialized_copy:
-  std::vector<Y> v3{Y{}, Y{}, Y{}};   // { dg-error "here" "" { target c++17 } }
-#endif
+
+  // Should not be able to create vector using uninitialized_fill_n:
+  std::vector<Y> v2{2u, Y{}};	// { dg-error "here" "" { target c++17_down } }
+  // { dg-error "deleted function .*Y::Y" "" { target c++20 } 0 }
 }
 
 // { dg-error "must be constructible from input type" "" { target *-*-* } 0 }
+// { dg-prune-output "construct_at" }
diff --git a/libstdc++-v3/testsuite/23_containers/vector/cons/89164_c++17.cc b/libstdc++-v3/testsuite/23_containers/vector/cons/89164_c++17.cc
new file mode 100644
index 0000000..24ade4e
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/cons/89164_c++17.cc
@@ -0,0 +1,39 @@
+// Copyright (C) 2019-2021 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-do compile { target c++17 } }
+
+#include <vector>
+
+// PR libstdc++/89164
+
+struct X
+{
+  X() = default;
+  X(const X&) = delete;
+};
+
+void test03()
+{
+  // Can create initializer_list<Y> with C++17 guaranteed copy elision,
+  // but shouldn't be able to copy from it with uninitialized_copy:
+  std::vector<X> v3{X{}, X{}, X{}};   // { dg-error "here" "" { target c++17_only } }
+  // { dg-error "deleted function .*X::X" "" { target c++20 } 0 }
+}
+
+// { dg-error "must be constructible from input type" "" { target *-*-* } 0 }
+// { dg-prune-output "construct_at" }
diff --git a/libstdc++-v3/testsuite/23_containers/vector/cons/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/cons/constexpr.cc
new file mode 100644
index 0000000..f02b39a
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/cons/constexpr.cc
@@ -0,0 +1,279 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
+
+template<typename T>
+struct Alloc : std::allocator<T>
+{
+  using std::allocator<T>::allocator;
+
+  constexpr explicit Alloc(int p) : personality(p) { }
+
+  template<typename U>
+    constexpr Alloc(const Alloc<U>& a) : personality(a.personality) { }
+
+  int personality = 0;
+
+  constexpr Alloc select_on_container_copy_construction() const
+  { return Alloc(-1); }
+
+  constexpr bool operator==(const Alloc& a) const noexcept
+  { return personality == a.personality; }
+};
+
+namespace default_constructor_global_scope
+{
+  constexpr std::vector<int> v1;
+  static_assert(v1.size() == 0);
+  static_assert(v1.capacity() == 0);
+
+  constexpr std::allocator<int> a;
+  constexpr std::vector<int> v2(a);
+  static_assert(v2.size() == 0);
+  static_assert(v2.capacity() == 0);
+
+  constexpr Alloc<int> aa(10);
+  constexpr std::vector<int, Alloc<int>> v3(aa);
+  static_assert(v3.size() == 0);
+  static_assert(v3.capacity() == 0);
+  static_assert(v3.get_allocator() == aa);
+}
+
+constexpr bool
+default_constructor_function_scope()
+{
+  // vector()
+
+  std::vector<int> v1;
+  VERIFY(v1.size() == 0);
+  VERIFY(v1.capacity() == 0);
+
+  // vector(const Allocator&)
+
+  const std::allocator<int> a;
+  std::vector<int> v2(a);
+  VERIFY(v2.size() == 0);
+  VERIFY(v2.capacity() == 0);
+
+  const Alloc<long> aa(10);
+  std::vector<long, Alloc<long>> v3(aa);
+  VERIFY(v3.size() == 0);
+  VERIFY(v3.capacity() == 0);
+  VERIFY(v3.get_allocator() == aa);
+
+  return true;
+}
+
+static_assert( default_constructor_function_scope() );
+
+constexpr bool
+sequence_constructors()
+{
+  // vector(size_type, const Allocator& = Allocator())
+
+  std::vector<int> v0(0);
+  VERIFY(v0.size() == 0);
+
+  std::vector<int> v1(1);
+  VERIFY(v1.size() == 1);
+
+  std::vector<int> v2(2);
+  VERIFY(v2.size() == 2);
+
+  std::vector<int> v50(50);
+  VERIFY(v50.size() == 50);
+
+  const std::allocator<int> a;
+  std::vector<int> a0(0, a);
+  VERIFY(a0.size() == 0);
+
+  std::vector<int> a1(1, a);
+  VERIFY(a1.size() == 1);
+
+  std::vector<int> a2(2, a);
+  VERIFY(a2.size() == 2);
+
+  std::vector<int> a50(50, a);
+  VERIFY(a50.size() == 50);
+
+  const Alloc<long> la(10);
+  std::vector<long, Alloc<long>> l0(0, la);
+  VERIFY(l0.size() == 0);
+  VERIFY(l0.get_allocator() == la);
+
+  std::vector<long, Alloc<long>> l1(1, la);
+  VERIFY(l1.size() == 1);
+  VERIFY(l1.get_allocator() == la);
+
+  std::vector<long, Alloc<long>> l2(2, la);
+  VERIFY(l2.size() == 2);
+  VERIFY(l2.get_allocator() == la);
+
+  std::vector<long, Alloc<long>> l50(50, la);
+  VERIFY(l50.size() == 50);
+  VERIFY(l50.get_allocator() == la);
+
+  // vector(size_type, const T&, const Allocator& = Allocator())
+
+  std::vector<int> v3(3, 4);
+  VERIFY(v3.size() == 3);
+  VERIFY(v3[0] == 4 && v3[2] == 4);
+
+  std::vector<int> a3(3, 5, a);
+  VERIFY(a3.size() == 3);
+  VERIFY(a3[0] == 5 && a3[2] == 5);
+
+  std::vector<long, Alloc<long>> l3(3, 6, la);
+  VERIFY(l3.size() == 3);
+  VERIFY(l3[0] == 6 && l3[2] == 6);
+  VERIFY(l3.get_allocator() == la);
+
+  return true;
+}
+
+static_assert(sequence_constructors());
+
+constexpr bool
+iterator_range_constructor()
+{
+  // vector(InputIterator, InputIterator, const Allocator& = Allocator())
+
+  short range[3] = { 1, 2, 3 };
+
+  std::vector<int> v0(std::begin(range), std::end(range));
+  VERIFY(v0.size() == std::size(range));
+  VERIFY(v0[0] == 1 && v0[1] == 2 && v0[2] == 3);
+
+  const Alloc<long> a(5);
+  std::vector<long, Alloc<long>> l0(std::begin(range), std::end(range), a);
+  VERIFY(l0.size() == std::size(range));
+  VERIFY(l0.get_allocator() == a);
+  VERIFY(l0[0] == 1 && l0[1] == 2 && l0[2] == 3);
+
+  struct input_iterator
+  {
+    using iterator_category = std::input_iterator_tag;
+    using value_type = int;
+    using pointer = const int*;
+    using reference = int;
+    using difference_type = int;
+
+    constexpr input_iterator() : val(0) { }
+    constexpr input_iterator(int i) : val(i) { }
+
+    constexpr input_iterator& operator++() { --val; return *this; }
+    constexpr input_iterator operator++(int) { return {val--}; }
+
+    constexpr int operator*() const { return val; }
+    constexpr const int* operator->() const { return &val; }
+
+    constexpr bool operator==(const input_iterator&) const = default;
+
+    int val;
+  };
+
+  std::vector<int> v1(input_iterator(3), input_iterator());
+  VERIFY(v1.size() == 3);
+  VERIFY(v1[0] == 3 && v1[1] == 2 && v1[2] == 1);
+
+  std::vector<long, Alloc<long>> l1(input_iterator(2), input_iterator(), a);
+  VERIFY(l1.size() == 2);
+  VERIFY(l1.get_allocator() == a);
+  VERIFY(l1[0] == 2 && l1[1] == 1);
+
+  return true;
+}
+
+static_assert(iterator_range_constructor());
+
+constexpr bool
+initializer_list_constructor()
+{
+  // vector(initializer_list<T>, const Allocator& = Allocator())
+
+  std::vector<int> v0({ 1, 2, 3 });
+  VERIFY(v0.size() == 3);
+  VERIFY(v0[0] == 1 && v0[1] == 2 && v0[2] == 3);
+
+  const Alloc<long> a(5);
+  std::vector<long, Alloc<long>> l0({ 1, 2, 3 }, a);
+  VERIFY(l0.size() == 3);
+  VERIFY(l0.get_allocator() == a);
+
+  return true;
+}
+
+static_assert(initializer_list_constructor());
+
+constexpr bool
+copy_constructor()
+{
+  const std::vector<int> v0({ 1, 2, 3 });
+  const std::vector<long, Alloc<long>> l0({ 4, 5, 6 });
+
+  // vector(const vector&)
+
+  std::vector<int> v1(v0);
+  VERIFY( v1.size() == v0.size() );
+  VERIFY( v1[0] == v0[0] && v1[1] == v0[1] && v1[2] == v0[2] );
+  VERIFY( v1.get_allocator() == v0.get_allocator() );
+
+  const Alloc<short> as(6);
+  std::vector<short, Alloc<short>> s1(3, short(2), as);
+  std::vector<short, Alloc<short>> s2(s1);
+  VERIFY( s2.size() == s1.size() );
+  VERIFY( s2.get_allocator().personality == -1 );
+
+  // vector(const vector&, const Allocator&)
+
+  const Alloc<long> a(6);
+  std::vector<long, Alloc<long>> l1(l0, a);
+  VERIFY( l1.size() == l0.size() );
+  VERIFY( l1[0] == l0[0] && l1[1] == l0[1] && l1[2] == l0[2] );
+  VERIFY( l1.get_allocator() == a );
+  VERIFY( l1.get_allocator() != l0.get_allocator() );
+
+  return true;
+}
+
+static_assert(copy_constructor());
+
+constexpr bool
+move_constructor()
+{
+  const std::vector<int> v0({ 1, 2, 3 });
+  const std::vector<long, Alloc<long>> l0({ 4, 5, 6 });
+
+  // vector(const vector&)
+
+  std::vector<int> v1(v0);
+  std::vector<int> v2(std::move(v1));
+  VERIFY( v2.size() == v0.size() );
+  VERIFY( v1.empty() );
+  VERIFY( v2[0] == v0[0] && v2[1] == v0[1] && v2[2] == v0[2] );
+  VERIFY( v2.get_allocator() == v0.get_allocator() );
+
+  // vector(const vector&, const Allocator&)
+
+  const Alloc<long> a(6);
+  std::vector<long, Alloc<long>> l1(l0);
+  std::vector<long, Alloc<long>> l2(std::move(l1), a);
+  VERIFY( l2.size() == l0.size() );
+  VERIFY( l2[0] == l0[0] && l2[1] == l0[1] && l2[2] == l0[2] );
+  VERIFY( l2.get_allocator() == a );
+  VERIFY( l2.get_allocator() != l0.get_allocator() );
+
+  std::vector<long, Alloc<long>> l3(std::move(l2), a);
+  VERIFY( l3.size() == l0.size() );
+  VERIFY( l3[0] == l0[0] && l3[1] == l0[1] && l3[2] == l0[2] );
+  VERIFY( l3.get_allocator() == a );
+  VERIFY( l3.get_allocator() == l2.get_allocator() );
+
+  return true;
+}
+
+static_assert(move_constructor());
diff --git a/libstdc++-v3/testsuite/23_containers/vector/data_access/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/data_access/constexpr.cc
new file mode 100644
index 0000000..45b3167
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/data_access/constexpr.cc
@@ -0,0 +1,26 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <testsuite_hooks.h>
+
+constexpr bool
+test_data()
+{
+  std::vector<int> v;
+  VERIFY( v.data() == nullptr );
+  v.reserve(1);
+  VERIFY( v.data() != nullptr );
+  const std::vector<int> v2{1,3,5,9};
+  VERIFY( v.data() != v2.data() );
+  VERIFY( v2.data()[2] == 5 );
+
+  v = v2;
+  VERIFY( v.data() != v2.data() );
+  VERIFY( v.data()[1] == 3 );
+
+  return true;
+}
+
+static_assert(test_data());
diff --git a/libstdc++-v3/testsuite/23_containers/vector/element_access/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/element_access/constexpr.cc
new file mode 100644
index 0000000..9b3b7f63
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/element_access/constexpr.cc
@@ -0,0 +1,91 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <testsuite_hooks.h>
+
+constexpr bool
+test_iterators()
+{
+  std::vector<int> v;
+  VERIFY( v.begin() == v.end() );
+  v.reserve(1);
+  VERIFY( v.begin() == v.end() );
+  v.resize(2);
+  VERIFY( v.begin() != v.end() );
+  VERIFY( v.cbegin() == v.begin() );
+  VERIFY( v.crbegin() == v.rbegin() );
+  VERIFY( v.cend() == v.end() );
+  VERIFY( v.crend() == v.rend() );
+
+  auto it = v.begin();
+  VERIFY( &*it == &v.front() );
+  VERIFY( it++ == v.begin() );
+  VERIFY( ++it == v.end() );
+  VERIFY( (it - 2) == v.begin() );
+  it -= 2;
+  it += 1;
+  VERIFY( (it + 1) == v.end() );
+
+  auto rit = v.rbegin();
+  VERIFY( &*rit == &v.back() );
+  VERIFY( rit++ == v.rbegin() );
+  VERIFY( ++rit == v.rend() );
+  VERIFY( (rit - 2) == v.rbegin() );
+  rit -= 2;
+  rit += 1;
+  VERIFY( (rit + 1) == v.rend() );
+
+  return true;
+}
+
+static_assert(test_iterators());
+
+constexpr bool
+test_access()
+{
+  std::vector<int> v{1, 2, 3};
+  VERIFY( v.at(1) == 2 );
+  VERIFY( v[2] == 3 );
+  VERIFY( &v[2] == &v.at(2) );
+  VERIFY( &v.front() == &v[0] );
+  VERIFY( &v.back() == &v[2] );
+
+  const auto& vc = v;
+  VERIFY( vc.at(1) == 2 );
+  VERIFY( &vc.at(1) == &v.at(1) );
+  VERIFY( &vc.at(1) == &vc[1] );
+  VERIFY( &vc.front() == &vc[0] );
+  VERIFY( &vc.back() == &vc[2] );
+
+  return true;
+}
+
+static_assert(test_access());
+
+template<typename T = int>
+  constexpr std::false_type
+  access_empty() { return {}; }
+
+template<typename T = int>
+  requires (std::bool_constant<(std::vector<T>().at(0), true)>::value)
+  constexpr std::true_type
+  access_empty() { return {}; }
+
+template<typename T = int>
+  requires (std::bool_constant<(std::vector<T>()[0], true)>::value)
+  constexpr std::true_type
+  access_empty() { return {}; }
+
+template<typename T = int>
+  requires (std::bool_constant<(std::vector<T>().front(), true)>::value)
+  constexpr std::true_type
+  access_empty() { return {}; }
+
+template<typename T = int>
+  requires (std::bool_constant<(std::vector<T>().back(), true)>::value)
+  constexpr std::true_type
+  access_empty() { return {}; }
+
+static_assert( ! access_empty() );
diff --git a/libstdc++-v3/testsuite/23_containers/vector/modifiers/assign/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/modifiers/assign/constexpr.cc
new file mode 100644
index 0000000..638cffe
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/modifiers/assign/constexpr.cc
@@ -0,0 +1,207 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <testsuite_hooks.h>
+
+template<typename T>
+struct Alloc : std::allocator<T>
+{
+  using std::allocator<T>::allocator;
+
+  constexpr explicit Alloc(int p) : personality(p) { }
+
+  template<typename U>
+    constexpr Alloc(const Alloc<U>& a) : personality(a.personality) { }
+
+  int personality = 0;
+
+  using propagate_on_container_move_assignment = std::false_type;
+
+  constexpr bool operator==(const Alloc& a) const noexcept
+  { return personality == a.personality; }
+};
+
+constexpr bool
+copy_assign()
+{
+  // vector::operator=(const vector&)
+
+  std::vector<int> v1, v2;
+  v1 = v1;
+  v2 = v1;
+  VERIFY(v2.size() == 0);
+  VERIFY(v2.capacity() == 0);
+
+  for (int i = 0; i < 10; ++i)
+    v1.push_back(i);
+  v2 = v1;
+  VERIFY(v2.size() == 10);
+  v2.reserve(50);
+  v1.push_back(1);
+  v2 = v1;
+  VERIFY(v2.size() == 11);
+  VERIFY(v2.capacity() == 50);
+
+  std::vector<int, Alloc<int>> va1(Alloc<int>(1)), va2(Alloc<int>(2));
+  va2 = va1;
+  VERIFY( va2.get_allocator().personality == 2 );
+  va1.push_back(999);
+  va2 = va1;
+  VERIFY( va2.get_allocator().personality == 2 );
+
+  return true;
+}
+
+static_assert( copy_assign() );
+
+constexpr bool
+move_assign()
+{
+  // vector::operator=(const vector&)
+
+  std::vector<int> v1, v2;
+  v1 = std::move(v1);
+  v2 = std::move(v1);
+  VERIFY(v2.size() == 0);
+  VERIFY(v2.capacity() == 0);
+
+  for (int i = 0; i < 10; ++i)
+    v1.push_back(i);
+  v2 = std::move(v1);
+  VERIFY(v2.size() == 10);
+  v2.reserve(50);
+  v1.push_back(1);
+  v2 = std::move(v1);
+  VERIFY(v2.size() == 1);
+  VERIFY(v1.capacity() == 0);
+  VERIFY(v2.capacity() == 1);
+
+  std::vector<int, Alloc<int>> va1(Alloc<int>(1)), va2(Alloc<int>(2));
+  va2 = std::move(va1);
+  VERIFY( va2.get_allocator().personality == 2 );
+  va1.push_back(9);
+  va1.push_back(99);
+  va1.push_back(999);
+  va1.push_back(9999);
+  va2 = std::move(va1);
+  va2 = std::move(va1);
+  VERIFY( va2.get_allocator().personality == 2 );
+
+  return true;
+}
+
+static_assert( move_assign() );
+
+constexpr bool
+initializer_list_assign()
+{
+  std::vector<int> v1;
+  v1 = {1, 2, 3};
+  VERIFY( v1.size() == 3 );
+  VERIFY( v1.capacity() == 3 );
+  v1 = {1, 2};
+  VERIFY( v1.size() == 2 );
+  VERIFY( v1.capacity() == 3 );
+  v1 = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+  VERIFY( v1.size() == 9 );
+  VERIFY( v1.capacity() == 9 );
+
+  std::vector<int, Alloc<int>> va1(Alloc<int>(111));
+  va1 = {1, 2, 3};
+  VERIFY( va1.get_allocator().personality == 111 );
+
+  return true;
+}
+
+static_assert( initializer_list_assign() );
+
+constexpr bool
+assign_iterator_range()
+{
+  std::vector<int> v;
+
+  int arr[] = { 1, 2, 3, 4 };
+  v.assign(arr, arr+3);
+  VERIFY( v.size() == 3 );
+  v.reserve(5);
+  v.assign(arr, arr+3);
+  VERIFY( v.capacity() == 5 );
+
+  struct input_iterator
+  {
+    using iterator_category = std::input_iterator_tag;
+    using value_type = int;
+    using pointer = const int*;
+    using reference = int;
+    using difference_type = int;
+
+    constexpr input_iterator() : val(0) { }
+    constexpr input_iterator(int i) : val(i) { }
+
+    constexpr input_iterator& operator++() { --val; return *this; }
+    constexpr input_iterator operator++(int) { return {val--}; }
+
+    constexpr int operator*() const { return val; }
+    constexpr const int* operator->() const { return &val; }
+
+    constexpr bool operator==(const input_iterator&) const = default;
+
+    int val;
+  };
+
+  v.assign(input_iterator(9), input_iterator());
+  VERIFY( v.size() == 9 );
+
+  return true;
+}
+
+static_assert( assign_iterator_range() );
+
+constexpr bool
+assign_value()
+{
+  std::vector<int> v1;
+  v1.assign(3, 8);
+  VERIFY( v1.size() == 3 );
+  VERIFY( v1.capacity() == 3 );
+  v1.assign(2, 9);
+  VERIFY( v1.size() == 2 );
+  VERIFY( v1.capacity() == 3 );
+  v1.assign(9, 10);
+  VERIFY( v1.size() == 9 );
+  VERIFY( v1.capacity() == 9 );
+
+  std::vector<int, Alloc<int>> va1(Alloc<int>(111));
+  va1.assign(2, 9);
+  VERIFY( va1.size() == 2 );
+  VERIFY( va1.get_allocator().personality == 111 );
+
+  return true;
+}
+
+static_assert( assign_value() );
+
+constexpr bool
+assign_initializer_list()
+{
+  std::vector<int> v1;
+  v1.assign({1, 2, 3});
+  VERIFY( v1.size() == 3 );
+  VERIFY( v1.capacity() == 3 );
+  v1.assign({1, 2});
+  VERIFY( v1.size() == 2 );
+  VERIFY( v1.capacity() == 3 );
+  v1.assign({1, 2, 3, 4, 5, 6, 7, 8, 9});
+  VERIFY( v1.size() == 9 );
+  VERIFY( v1.capacity() == 9 );
+
+  std::vector<int, Alloc<int>> va1(Alloc<int>(111));
+  va1.assign({1, 2, 3});
+  VERIFY( va1.get_allocator().personality == 111 );
+
+  return true;
+}
+
+static_assert( assign_initializer_list() );
diff --git a/libstdc++-v3/testsuite/23_containers/vector/modifiers/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/modifiers/constexpr.cc
new file mode 100644
index 0000000..36efecf
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/modifiers/constexpr.cc
@@ -0,0 +1,226 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <testsuite_hooks.h>
+
+template<typename T>
+struct Alloc : std::allocator<T>
+{
+  using std::allocator<T>::allocator;
+
+  constexpr explicit Alloc(int p) : personality(p) { }
+
+  template<typename U>
+    constexpr Alloc(const Alloc<U>& a) : personality(a.personality) { }
+
+  int personality = 0;
+
+  constexpr bool operator==(const Alloc& a) const noexcept
+  { return personality == a.personality; }
+};
+
+constexpr bool
+test_push_back()
+{
+  std::vector<int> v;
+  int& r = v.emplace_back(7);
+  VERIFY( r == 7 );
+  VERIFY( &r == &v.front() );
+  v.emplace_back(r);
+  v.emplace_back(v.front());
+  v.emplace_back(v.back());
+  VERIFY( v.size() == 4 );
+  v.emplace_back(8);
+  VERIFY( v.size() == 5 );
+  VERIFY( v.back() == 8 );
+
+  v.pop_back();
+  VERIFY( v.size() == 4 );
+  VERIFY( v.back() == 7 );
+  v.pop_back();
+  v.pop_back();
+  v.pop_back();
+  v.pop_back();
+  VERIFY( v.empty() );
+
+  v.push_back(99);
+  for (std::size_t i = 0, c = v.capacity(); i <= c; ++i)
+    v.push_back(v.front());
+  VERIFY( v.capacity() > v.size() );
+
+  std::vector<int, Alloc<int>> va;
+  va.push_back(99);
+  va.push_back(va.front());
+  VERIFY( va.size() == 2 );
+
+  return true;
+}
+
+static_assert( test_push_back() );
+
+template<typename T = int>
+constexpr std::false_type
+pop_back_empty() { return {}; }
+
+template<typename T = int>
+requires (std::bool_constant<(std::vector<T>().pop_back(), true)>::value)
+constexpr std::true_type
+pop_back_empty() { return {}; }
+
+static_assert( ! pop_back_empty() );
+
+constexpr bool
+test_insert_erase()
+{
+  std::vector<int> v;
+
+  // vector::emplace(const_iterator, Args&&...)
+  auto p = v.emplace(v.begin());
+  VERIFY( p == v.begin() );
+  p = v.emplace(v.end(), 7);
+  VERIFY( p == --v.end() );
+
+  // vector::insert(const_iterator, const T&)
+  p = v.insert(v.begin(), *p);
+  VERIFY( p == v.begin() );
+  VERIFY( *p == 7 );
+  VERIFY( &*p == &v.front() );
+  // vector::insert(const_iterator, T&&)
+  p = v.insert(v.end(), 1);
+  VERIFY( p == --v.end() );
+  v.insert(p, v.front());
+  v.insert(v.end(), v.back());
+  VERIFY( v.size() == 6 );
+  v.insert(v.end(), 8);
+  VERIFY( v.size() == 7 );
+  VERIFY( v.back() == 8 );
+
+  // vector::insert(const_iterator, size_type, const T&)
+  v.insert(v.begin(), 2, v.front());
+  v.insert(v.end(), 3, 99);
+  VERIFY( v.size() == 12 );
+
+  struct input_iterator
+  {
+    using iterator_category = std::input_iterator_tag;
+    using value_type = int;
+    using pointer = const int*;
+    using reference = int;
+    using difference_type = int;
+
+    constexpr input_iterator() : val(0) { }
+    constexpr input_iterator(int i) : val(i) { }
+
+    constexpr input_iterator& operator++() { --val; return *this; }
+    constexpr input_iterator operator++(int) { return {val--}; }
+
+    constexpr int operator*() const { return val; }
+    constexpr const int* operator->() const { return &val; }
+
+    constexpr bool operator==(const input_iterator&) const = default;
+
+    int val;
+  };
+
+  // vector::insert(const_iterator, Iter, Iter);
+  v.insert(v.begin() + 2, input_iterator(), input_iterator());
+  VERIFY( v.size() == 12 );
+  v.reserve(13);
+  auto n = v.capacity() - v.size();
+  v.insert(v.end() - 9, input_iterator(n), input_iterator()); // no reallocation
+  VERIFY( v.size() == (12 + n) );
+  short a[] = { 84, 85 };
+  v.insert(v.end() - 1, a, a + 2); // reallocation needed
+  VERIFY( v.size() == (12 + n + 2) );
+  v.resize(32);
+
+  // vector::insert(const_iterator, initializer_list<T>)
+  v.insert(v.begin(), {1,2,3});
+  VERIFY( v.size() == 35 );
+
+  v.rbegin()[0] = 999;
+  v.rbegin()[1] = 888;
+
+  // vector::erase(const_iterator)
+  v.erase(v.end() - 1);
+  VERIFY( v.size() == 34 );
+  VERIFY( v.back() == 888 );
+  v.erase(v.begin());
+  v.erase(v.begin() + 1);
+  v.erase(v.end() - 1);
+  VERIFY( v.size() == 31 );
+
+  // vector::erase(const_iterator, const_iterator)
+  v.erase(v.begin(), v.begin());
+  v.erase(v.end(), v.end());
+  v.erase(v.begin(), v.begin() + 1);
+  VERIFY( v.size() == 30 );
+  v.erase(v.begin() + 2, v.end() - 2);
+  VERIFY( v.size() == 4 );
+  v.erase(v.begin(), v.end());
+  VERIFY( v.empty() );
+  v.erase( v.begin(), v.begin() );
+  VERIFY( v.empty() );
+
+  v.insert(v.end(), 99);
+  for (std::size_t i = 0, c = v.capacity(); i <= c; ++i)
+    v.insert(v.end() - 1, v.front());
+  VERIFY( v.capacity() > v.size() );
+  v.insert(v.end(), 999);
+  for (std::size_t i = 0, c = v.capacity(); i <= c; ++i)
+    v.insert(v.begin(), v.front());
+
+  std::vector<int, Alloc<int>> va;
+  va.insert(va.begin(), 99);
+  va.insert(va.begin(), va.front());
+  VERIFY( va.size() == 2 );
+  va.erase(va.begin());
+
+  return true;
+}
+
+static_assert( test_insert_erase() );
+
+constexpr bool
+test_clear()
+{
+  std::vector<int> v0;
+  v0.clear();
+  VERIFY( v0.size() == 0 );
+  VERIFY( v0.capacity() == 0 );
+
+  std::vector<int> v{1, 10, 100};
+  v.clear();
+  VERIFY( v.size() == 0 );
+  VERIFY( v.capacity() == 3 );
+
+  std::vector<int, Alloc<int>> va;
+  va.clear();
+  va.push_back(1);
+  va.clear();
+  va.clear();
+
+  return true;
+}
+
+static_assert( test_clear() );
+
+constexpr bool
+test_erasure()
+{
+  const char* names[] = { "Vince", "Clarke", "Andy", "Bell" };
+  std::vector<const char*> e(std::begin(names), std::end(names));
+
+  auto n = std::erase(e, names[0]);
+  VERIFY( n == 1 );
+  VERIFY( e.size() == 3 );
+  n = std::erase_if(e, [](auto name) { return name[4] == '\0'; });
+  VERIFY( n == 2 );
+  VERIFY( e.size() == 1 );
+
+  return true;
+}
+
+static_assert( test_erasure() );
diff --git a/libstdc++-v3/testsuite/23_containers/vector/modifiers/swap/constexpr.cc b/libstdc++-v3/testsuite/23_containers/vector/modifiers/swap/constexpr.cc
new file mode 100644
index 0000000..4d06558
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/vector/modifiers/swap/constexpr.cc
@@ -0,0 +1,51 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+// { dg-xfail-if "not supported" { debug-mode } }
+
+#include <vector>
+#include <utility>
+#include <testsuite_hooks.h>
+
+constexpr bool
+test_std_swap()
+{
+  std::vector<int> v1 = {1, 2, 3};
+  std::vector<int> v2 = {0, 1};
+
+  std::swap(v1, v2);
+
+  VERIFY(v1.size() == 2);
+  VERIFY(v1.at(0) == 0);
+  VERIFY(v1.at(1) == 1);
+
+  VERIFY(v2.size() == 3);
+  VERIFY(v2[0] == 1);
+  VERIFY(v2[1] == 2);
+  VERIFY(v2[2] == 3);
+
+  return true;
+}
+
+static_assert(test_std_swap());
+
+constexpr bool
+test_member_swap()
+{
+  std::vector<int> v1 = {1, 2, 3};
+  std::vector<int> v2 = {0, 1};
+
+  v1.swap(v2);
+
+  VERIFY(v1.size() == 2);
+  VERIFY(v1.at(0) == 0);
+  VERIFY(v1.at(1) == 1);
+
+  VERIFY(v2.size() == 3);
+  VERIFY(v2[0] == 1);
+  VERIFY(v2[1] == 2);
+  VERIFY(v2[2] == 3);
+
+  return true;
+}
+
+static_assert(test_member_swap());
diff --git a/libstdc++-v3/testsuite/25_algorithms/move/constexpr.cc b/libstdc++-v3/testsuite/25_algorithms/move/constexpr.cc
new file mode 100644
index 0000000..eb1f3b1
--- /dev/null
+++ b/libstdc++-v3/testsuite/25_algorithms/move/constexpr.cc
@@ -0,0 +1,30 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do compile { target c++20 } }
+
+#include <algorithm>
+#include <span>
+
+constexpr bool
+test01()
+{
+  // PR libstdc++/102358
+  int x[2] = {1,2}, y[2];
+  std::span in(x), out(y);
+  std::move(std::move_iterator(in.begin()), std::move_iterator(in.end()),
+	    out.begin());
+  return std::equal(std::move_iterator(in.begin()), std::move_iterator(in.end()),
+		    std::move_iterator(out.begin()));
+}
+
+static_assert(test01());
+
+constexpr bool
+test02()
+{
+  int x[2] = {1,2}, y[2];
+  std::span in(x), out(y);
+  std::move(in.rbegin(), in.rend(), out.rbegin());
+  return std::equal(in.rbegin(), in.rend(), out.rbegin());
+}
+
+static_assert(test02());
diff --git a/libstdc++-v3/testsuite/26_numerics/random/concept.cc b/libstdc++-v3/testsuite/26_numerics/random/concept.cc
index 1adc0c1..0a0cbbf 100644
--- a/libstdc++-v3/testsuite/26_numerics/random/concept.cc
+++ b/libstdc++-v3/testsuite/26_numerics/random/concept.cc
@@ -227,7 +227,11 @@ struct N12
   static constexpr unsigned max() { return 1; }
 };
 
+#if __cpp_implicit_constexpr
+static_assert( std::uniform_random_bit_generator<N12> ); // LWG 3150
+#else
 static_assert( ! std::uniform_random_bit_generator<N12> ); // LWG 3150
+#endif
 
 struct N13
 {
@@ -236,7 +240,11 @@ struct N13
   static unsigned max() { return 1; } // not constexpr
 };
 
+#if __cpp_implicit_constexpr
+static_assert( std::uniform_random_bit_generator<N13> ); // LWG 3150
+#else
 static_assert( ! std::uniform_random_bit_generator<N13> ); // LWG 3150
+#endif
 
 struct N14
 {
diff --git a/libstdc++-v3/testsuite/26_numerics/random/random_device/cons/token.cc b/libstdc++-v3/testsuite/26_numerics/random/random_device/cons/token.cc
index aeb7403..813db73 100644
--- a/libstdc++-v3/testsuite/26_numerics/random/random_device/cons/token.cc
+++ b/libstdc++-v3/testsuite/26_numerics/random/random_device/cons/token.cc
@@ -24,6 +24,7 @@
 
 #include <random>
 #include <stdexcept>
+#include <cstdio>
 #include <testsuite_hooks.h>
 #include <testsuite_random.h>
 
@@ -42,7 +43,7 @@ test02()
 #ifdef _GLIBCXX_USE_DEV_RANDOM
   std::random_device x1("/dev/urandom");
   std::random_device x2("/dev/random");
-  VERIFY( x1() != x2() );
+  VERIFY( x1() != x2() || x1() != x2() );
 #endif
 }
 
@@ -51,14 +52,22 @@ test03()
 {
   // At least one of these tokens should be valid.
   const std::string tokens[] = {
-    "rdseed", "rdrand", "rand_s", "/dev/urandom", "/dev/random", "mt19937",
-    "prng"
+    "rdseed", "rdrand", "darn",
+    "rand_s", "/dev/urandom", "/dev/random",
+    "getentropy", "arc4random",
+    "mt19937", "prng"
   };
   int count = 0;
   for (const std::string& token : tokens)
   {
+    std::printf("checking std::random_device(\"%s\"):\t", token.c_str());
     if (__gnu_test::random_device_available(token))
+    {
+      std::puts("yes");
       ++count;
+    }
+    else
+      std::puts("no");
   }
   VERIFY( count != 0 );
 }
diff --git a/libstdc++-v3/testsuite/26_numerics/random/random_device/entropy.cc b/libstdc++-v3/testsuite/26_numerics/random/random_device/entropy.cc
index 9ef1538..63b7043 100644
--- a/libstdc++-v3/testsuite/26_numerics/random/random_device/entropy.cc
+++ b/libstdc++-v3/testsuite/26_numerics/random/random_device/entropy.cc
@@ -22,7 +22,14 @@ test01()
       VERIFY( entropy <= max );
     }
 
-  for (auto token : { "rdrand", "rdseed" })
+  for (auto token : { "rdrand", "rdseed", "darn", "hw" })
+    if (__gnu_test::random_device_available(token))
+    {
+      const double entropy = std::random_device(token).entropy();
+      VERIFY( entropy == max );
+    }
+
+    for (auto token : { "getentropy", "arc4random" })
     if (__gnu_test::random_device_available(token))
     {
       const double entropy = std::random_device(token).entropy();
diff --git a/libstdc++-v3/testsuite/26_numerics/valarray/103022.cc b/libstdc++-v3/testsuite/26_numerics/valarray/103022.cc
new file mode 100644
index 0000000..d2e3467
--- /dev/null
+++ b/libstdc++-v3/testsuite/26_numerics/valarray/103022.cc
@@ -0,0 +1,15 @@
+// { dg-options "-D_GLIBCXX_DEBUG" }
+// { dg-do compile { target c++11 } }
+
+#include <valarray>
+
+int main()
+{
+  // PR libstdc++/103022
+  std::valarray<double> va;
+  (void) std::begin(va);
+  (void) std::end(va);
+  const auto& cva = va;
+  (void) std::begin(cva);
+  (void) std::end(cva);
+}
diff --git a/libstdc++-v3/testsuite/26_numerics/valarray/range_access.cc b/libstdc++-v3/testsuite/26_numerics/valarray/range_access.cc
index c015d18..c49c2c5 100644
--- a/libstdc++-v3/testsuite/26_numerics/valarray/range_access.cc
+++ b/libstdc++-v3/testsuite/26_numerics/valarray/range_access.cc
@@ -1,4 +1,4 @@
-// { dg-do compile { target c++11 } }
+// { dg-do run { target c++11 } }
 
 // Copyright (C) 2010-2021 Free Software Foundation, Inc.
 //
@@ -17,7 +17,7 @@
 // with this library; see the file COPYING3.  If not see
 // <http://www.gnu.org/licenses/>.
 
-// 26.6.10 valarray range access: [valarray.range]
+// C++11 26.6.10 valarray range access: [valarray.range]
 
 #include <valarray>
 
@@ -25,9 +25,46 @@ void
 test01()
 {
   std::valarray<double> va{1.0, 2.0, 3.0};
-  std::begin(va);
-  std::end(va);
+  (void) std::begin(va);
+  (void) std::end(va);
   const auto& cva = va;
-  std::begin(cva);
-  std::end(cva);
+  (void) std::begin(cva);
+  (void) std::end(cva);
+
+  using Iter = decltype(std::begin(va));
+  using IterTraits = std::iterator_traits<Iter>;
+  static_assert( std::is_same<Iter, decltype(std::end(va))>::value, "" );
+  static_assert( std::is_same<IterTraits::iterator_category,
+			      std::random_access_iterator_tag>::value, "" );
+  static_assert( std::is_same<IterTraits::value_type, double>::value, "" );
+  static_assert( std::is_same<IterTraits::reference, double&>::value, "" );
+  using CIter = decltype(std::begin(cva));
+  using CIterTraits = std::iterator_traits<CIter>;
+  static_assert( std::is_same<CIter, decltype(std::end(cva))>::value, "" );
+  static_assert( std::is_same<CIterTraits::iterator_category,
+			      std::random_access_iterator_tag>::value, "" );
+  static_assert( std::is_same<CIterTraits::value_type, double>::value, "" );
+  static_assert( std::is_same<CIterTraits::reference, const double&>::value, "" );
+#if __cplusplus >= 202002L
+  static_assert( std::contiguous_iterator<Iter> );
+  static_assert( std::contiguous_iterator<CIter> );
+#endif
+}
+
+// PR libstdc++/103022
+void
+test02()
+{
+  std::valarray<double> va;
+  (void) std::begin(va);
+  (void) std::end(va);
+  const auto& cva = va;
+  (void) std::begin(cva);
+  (void) std::end(cva);
+}
+
+int main()
+{
+  test01();
+  test02();
 }
diff --git a/libstdc++-v3/testsuite/26_numerics/valarray/range_access2.cc b/libstdc++-v3/testsuite/26_numerics/valarray/range_access2.cc
index cf5418a..4217367 100644
--- a/libstdc++-v3/testsuite/26_numerics/valarray/range_access2.cc
+++ b/libstdc++-v3/testsuite/26_numerics/valarray/range_access2.cc
@@ -1,4 +1,4 @@
-// { dg-do compile { target c++14 } }
+// { dg-do run { target c++14 } }
 
 // Copyright (C) 2015-2021 Free Software Foundation, Inc.
 //
@@ -17,7 +17,7 @@
 // with this library; see the file COPYING3.  If not see
 // <http://www.gnu.org/licenses/>.
 
-// 26.6.10 valarray range access: [valarray.range]
+// C++11 26.6.10 valarray range access: [valarray.range]
 
 #include <iterator>
 #include <valarray>
@@ -33,3 +33,21 @@ test01()
   (void) std::cbegin(cva);
   (void) std::cend(cva);
 }
+
+// PR libstdc++/103022
+void
+test02()
+{
+  std::valarray<double> va;
+  (void) std::cbegin(va);
+  (void) std::cend(va);
+  const auto& cva = va;
+  (void) std::cbegin(cva);
+  (void) std::cend(cva);
+}
+
+int main()
+{
+  test01();
+  test02();
+}
diff --git a/libstdc++-v3/testsuite/27_io/filesystem/path/factory/u8path-char8_t.cc b/libstdc++-v3/testsuite/27_io/filesystem/path/factory/u8path-char8_t.cc
index f52900d..1f6b7a3 100644
--- a/libstdc++-v3/testsuite/27_io/filesystem/path/factory/u8path-char8_t.cc
+++ b/libstdc++-v3/testsuite/27_io/filesystem/path/factory/u8path-char8_t.cc
@@ -15,7 +15,7 @@
 // with this library; see the file COPYING3.  If not see
 // <http://www.gnu.org/licenses/>.
 
-// { dg-options "-fchar8_t" }
+// { dg-options "-fchar8_t -Wno-stringop-overread" }
 // { dg-do run { target c++17 } }
 
 #include <filesystem>
@@ -36,6 +36,7 @@ test01()
   p = fs::u8path(u8"\xf0\x9d\x84\x9e");
   VERIFY( p.u8string() == u8"\U0001D11E" );
 
+  // The following triggers -Wstringop-overread.  See PR 102958.
   std::u8string s1 = u8"filename2";
   p = fs::u8path(s1);
   VERIFY( p.u8string() == u8"filename2" );
diff --git a/libstdc++-v3/testsuite/27_io/spanstream/1.cc b/libstdc++-v3/testsuite/27_io/spanstream/1.cc
new file mode 100644
index 0000000..b66ee60
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/spanstream/1.cc
@@ -0,0 +1,53 @@
+// { dg-options "-std=gnu++23" }
+// { dg-do run { target c++23 } }
+
+#include <spanstream>
+
+#ifndef __cpp_lib_spanstream
+# error "Feature-test macro for spanstream missing in <spanstream>"
+#elif __cpp_lib_spanstream != 202106L
+# error "Feature-test macro for spanstream has wrong value in <spanstream>"
+#endif
+
+#include <testsuite_hooks.h>
+
+using std::ispanstream;
+using std::ospanstream;
+using std::span;
+
+void
+test_input()
+{
+  // reading input from a fixed pre-arranged character buffer
+  char input[] = "10 20 30";
+  ispanstream is{span<char>{input}};
+  int i;
+  is >> i;
+  VERIFY(10 == i);
+  is >> i;
+  VERIFY(20 == i);
+  is >> i;
+  VERIFY(30 == i);
+  is >>i;
+  VERIFY(!is);
+}
+
+void
+test_output()
+{
+  // writing to a fixed pre-arranged character buffer
+  char output[30]{}; // zero-initialize array
+  ospanstream os{span<char>{output}};
+  os << 10 << 20 << 30;
+  auto const sp = os.span();
+  VERIFY(6 == sp.size());
+  VERIFY("102030" == std::string(sp.data(), sp.size()));
+  VERIFY(static_cast<void*>(output) == sp.data()); // no copying of underlying data!
+  VERIFY("102030" == std::string(output)); // initialization guaranteed NUL termination
+}
+
+int main()
+{
+  test_input();
+  test_output();
+}
diff --git a/libstdc++-v3/testsuite/27_io/spanstream/version.cc b/libstdc++-v3/testsuite/27_io/spanstream/version.cc
new file mode 100644
index 0000000..6261755
--- /dev/null
+++ b/libstdc++-v3/testsuite/27_io/spanstream/version.cc
@@ -0,0 +1,10 @@
+// { dg-options "-std=gnu++23" }
+// { dg-do compile { target c++23 } }
+
+#include <version>
+
+#ifndef __cpp_lib_spanstream
+# error "Feature-test macro for spanstream missing in <version>"
+#elif __cpp_lib_spanstream != 202106L
+# error "Feature-test macro for spanstream has wrong value in <version>"
+#endif
diff --git a/libstdc++-v3/testsuite/28_regex/basic_regex/84110.cc b/libstdc++-v3/testsuite/28_regex/basic_regex/84110.cc
index b9971dc..16f928b 100644
--- a/libstdc++-v3/testsuite/28_regex/basic_regex/84110.cc
+++ b/libstdc++-v3/testsuite/28_regex/basic_regex/84110.cc
@@ -11,6 +11,9 @@ void test01()
 
 #if __cpp_exceptions
   using namespace std::regex_constants;
+  // See https://gcc.gnu.org/pipermail/gcc-patches/2021-October/582486.html
+  using std::regex_constants::extended;
+
   for (auto syn : {basic, extended, awk, grep, egrep})
   {
     try
diff --git a/libstdc++-v3/testsuite/experimental/string_view/operations/compare/wchar_t/1.cc b/libstdc++-v3/testsuite/experimental/string_view/operations/compare/wchar_t/1.cc
index db523e6..20bb030 100644
--- a/libstdc++-v3/testsuite/experimental/string_view/operations/compare/wchar_t/1.cc
+++ b/libstdc++-v3/testsuite/experimental/string_view/operations/compare/wchar_t/1.cc
@@ -81,8 +81,8 @@ test01()
   test_value(wcsncmp(str_1.data(), str_0.data(), 6), z);
   test_value(wcsncmp(str_1.data(), str_0.data(), 14), lt);
   test_value(wmemcmp(str_1.data(), str_0.data(), 6), z);
-  test_value(wmemcmp(str_1.data(), str_0.data(), 14), lt);
-  test_value(wmemcmp(L"costa marbella", L"costa rica", 14), lt);
+  test_value(wmemcmp(str_1.data(), str_0.data(), 10), lt);
+  test_value(wmemcmp(L"costa marbella", L"costa rica", 10), lt);
 
   // int compare(const basic_string_view& str) const;
   test_value(str_0.compare(str_1), gt); //because r>m
diff --git a/libstdc++-v3/testsuite/lib/gdb-test.exp b/libstdc++-v3/testsuite/lib/gdb-test.exp
index f993355..db0fc2f 100644
--- a/libstdc++-v3/testsuite/lib/gdb-test.exp
+++ b/libstdc++-v3/testsuite/lib/gdb-test.exp
@@ -50,40 +50,48 @@ proc get_line_number {filename marker} {
     return $gdb_markers($filename,$marker)
 }
 
-# Make note of a gdb test.  A test consists of a variable name and an
-# expected result.
-proc note-test {var result} {
+proc register_gdb_test {var result kind rexp selector} {
     global gdb_tests
 
-    lappend gdb_tests $var $result print 0
+    set xfail 0
+    if {[string length $selector] > 0} {
+	switch [dg-process-target $selector] {
+	    "N" { return }
+	    "S" { }
+	    "P" { }
+	    "F" { set xfail 1 }
+	}
+    }
+
+    lappend gdb_tests $var $result $kind $rexp $xfail
+}
+
+# Make note of a gdb test.  A test consists of a variable name and an
+# expected result, and an optional target selector.
+proc note-test {var result {selector {}}} {
+    register_gdb_test $var $result print 0 $selector
 }
 
 # A test that uses a regular expression.  This is like note-test, but
 # the result is a regular expression that is matched against the
 # output.
-proc regexp-test {var result} {
-    global gdb_tests
-
-    lappend gdb_tests $var $result print 1
+proc regexp-test {var result {selector {}}} {
+    register_gdb_test $var $result print 1 $selector
 }
 
 # A test of 'whatis'.  This tests a type rather than a variable.
-proc whatis-test {var result} {
-    global gdb_tests
-
-    lappend gdb_tests $var $result whatis 0
+proc whatis-test {var result {selector {}}} {
+    register_gdb_test $var $result whatis 0 $selector
 }
 
 # A test of 'whatis' that uses a regular expression. This tests a type rather
 # than a variable.
-proc whatis-regexp-test {var result} {
-    global gdb_tests
-
-    lappend gdb_tests $var $result whatis 1
+proc whatis-regexp-test {var result {selector {}}} {
+    register_gdb_test $var $result whatis 1 $selector
 }
 
 # Utility for testing variable values using gdb, invoked via dg-final.
-# Tests all tests indicated by note-test and regexp-test.
+# Tests all tests indicated by note-test, whatis-test, and the regexp versions.
 #
 # Argument 0 is the marker on which to put a breakpoint
 # Argument 2 handles expected failures and the like
@@ -144,7 +152,7 @@ proc gdb-test { marker {selector {}} {load_xmethods 0} } {
     puts $fd "info share"
 
     set count 0
-    foreach {var result kind rexp} $gdb_tests {
+    foreach {var result kind rexp xfail} $gdb_tests {
 	incr count
 	set gdb_var($count) $var
 	set gdb_expected($count) $result
@@ -152,6 +160,7 @@ proc gdb-test { marker {selector {}} {load_xmethods 0} } {
 	    if {$do_whatis_tests} {
 		set gdb_is_type($count) 1
 		set gdb_is_regexp($count) $rexp
+		set gdb_is_xfail($count) $xfail
 		set gdb_command($count) "whatis $var"
 	    } else {
 	        unsupported "$testname"
@@ -161,6 +170,7 @@ proc gdb-test { marker {selector {}} {load_xmethods 0} } {
 	} else {
 	    set gdb_is_type($count) 0
 	    set gdb_is_regexp($count) $rexp
+	    set gdb_is_xfail($count) $xfail
 	    set gdb_command($count) "print $var"
 	}
 	puts $fd $gdb_command($count)
@@ -198,11 +208,20 @@ proc gdb-test { marker {selector {}} {load_xmethods 0} } {
 	    }
 
 	    if {$match} {
-		pass "$testname $gdb_command($test_counter)"
+		if {$gdb_is_xfail($test_counter)} {
+		    xpass "$testname $gdb_command($test_counter)"
+		    verbose " matched =>$first<="
+		} else {
+		    pass "$testname $gdb_command($test_counter)"
+		}
 	    } else {
-		fail "$testname $gdb_command($test_counter)"
-		verbose "     got =>$first<="
-		verbose "expected =>$gdb_expected($test_counter)<="
+		if {$gdb_is_xfail($test_counter)} {
+		    xfail "$testname $gdb_command($test_counter)"
+		} else {
+		    fail "$testname $gdb_command($test_counter)"
+		    verbose "     got =>$first<="
+		    verbose "expected =>$gdb_expected($test_counter)<="
+		}
 	    }
 
 	    if {$test_counter == $count} {
diff --git a/libstdc++-v3/testsuite/libstdc++-prettyprinters/80276.cc b/libstdc++-v3/testsuite/libstdc++-prettyprinters/80276.cc
index 1443f2a..00255c9 100644
--- a/libstdc++-v3/testsuite/libstdc++-prettyprinters/80276.cc
+++ b/libstdc++-v3/testsuite/libstdc++-prettyprinters/80276.cc
@@ -46,7 +46,7 @@ main()
   // { dg-final { whatis-regexp-test p1 "std::unique_ptr<std::(__debug::)?vector<std::unique_ptr<std::(__debug::)?vector<int>\\*>>>" } }
   // { dg-final { whatis-regexp-test p2 "std::unique_ptr<std::(__debug::)?vector<std::unique_ptr<std::(__debug::)?set<int>\\*>>\\\[\\\]>" } }
   // { dg-final { whatis-regexp-test p3 "std::unique_ptr<std::(__debug::)?set<std::unique_ptr<std::(__debug::)?vector<int>\\*>>\\\[10\\\]>" } }
-  // { dg-final { whatis-regexp-test p4 "std::unique_ptr<std::(__debug::)?vector<std::unique_ptr<std::(__debug::)?list<std::string>\\\[\\\]>>\\\[99\\\]>" } }
+  // { dg-final { whatis-regexp-test p4 "std::unique_ptr<std::(__debug::)?vector<std::unique_ptr<std::(__debug::)?list<std::string>\\\[\\\]>>\\\[99\\\]>" { xfail c++20 } } }
 
   placeholder(&p1);		// Mark SPOT
   placeholder(&p2);
diff --git a/libstdc++-v3/testsuite/libstdc++-prettyprinters/libfundts.cc b/libstdc++-v3/testsuite/libstdc++-prettyprinters/libfundts.cc
index 67d177b..af948e0 100644
--- a/libstdc++-v3/testsuite/libstdc++-prettyprinters/libfundts.cc
+++ b/libstdc++-v3/testsuite/libstdc++-prettyprinters/libfundts.cc
@@ -50,7 +50,7 @@ main()
   om = std::map<int, double>{ {1, 2.}, {3, 4.}, {5, 6.} };
 // { dg-final { regexp-test om {std::experimental::optional<std::(__debug::)?map<int, double>> containing std::(__debug::)?map with 3 elements = {\[1\] = 2, \[3\] = 4, \[5\] = 6}} } }
   optional<std::string> os{ "stringy" };
-// { dg-final { note-test os {std::experimental::optional<std::string> = {[contained value] = "stringy"}} } }
+// { dg-final { note-test os {std::experimental::optional<std::string> = {[contained value] = "stringy"}} { xfail c++20 } } }
 
   any a;
 // { dg-final { note-test a {std::experimental::any [no contained value]} } }
@@ -61,7 +61,7 @@ main()
   any ap = (void*)nullptr;
 // { dg-final { note-test ap {std::experimental::any containing void * = {[contained value] = 0x0}} } }
   any as = *os;
-// { dg-final { note-test as {std::experimental::any containing std::string = {[contained value] = "stringy"}} } }
+// { dg-final { note-test as {std::experimental::any containing std::string = {[contained value] = "stringy"}} { xfail c++20 } } }
   any as2("stringiest");
 // { dg-final { regexp-test as2 {std::experimental::any containing const char \* = {\[contained value\] = 0x[[:xdigit:]]+ "stringiest"}} } }
   any am = *om;
diff --git a/libstdc++-v3/testsuite/libstdc++-prettyprinters/prettyprinters.exp b/libstdc++-v3/testsuite/libstdc++-prettyprinters/prettyprinters.exp
index 9cd3a26..96686b2 100644
--- a/libstdc++-v3/testsuite/libstdc++-prettyprinters/prettyprinters.exp
+++ b/libstdc++-v3/testsuite/libstdc++-prettyprinters/prettyprinters.exp
@@ -49,7 +49,7 @@ gdb-dg-runtest [lsort [glob $srcdir/$subdir/*.cc]] \
 
 if { [check_effective_target_lto] } {
   append cxxflags " -flto"
-  # work around sourceware.org 20882
+  # Work around linker bug https://sourceware.org/PR20882
   regsub {^(.*)-Wl,--gc-sections(.*)$} $cxxldflags {\1\2} cxxldflags
   gdb-dg-runtest [lsort [glob $srcdir/$subdir/*.cc]] \
     "" "$DEFAULT_CXXFLAGS -flto $PCH_CXXFLAGS"
diff --git a/libstdc++-v3/testsuite/std/ranges/adaptors/p1739.cc b/libstdc++-v3/testsuite/std/ranges/adaptors/p1739.cc
new file mode 100644
index 0000000..3e51aa0
--- /dev/null
+++ b/libstdc++-v3/testsuite/std/ranges/adaptors/p1739.cc
@@ -0,0 +1,88 @@
+// { dg-options "-std=gnu++20" }
+// { dg-do run { target c++20 } }
+
+#include <algorithm>
+#include <ranges>
+#include <testsuite_hooks.h>
+
+// Verify P1739R4 (and LWG 3407) changes to views::take/drop/counted.
+
+namespace ranges = std::ranges;
+namespace views = ranges::views;
+
+void
+test01()
+{
+  ranges::empty_view<int> v;
+  using ty = decltype(v);
+  using ty = decltype(views::take(v, 42));
+  using ty = decltype(views::drop(v, 42));
+}
+
+void
+test02()
+{
+  int x[] = {1,2,3,4,5};
+  std::span r(x, x+5);
+  using ty = decltype(r);
+
+  std::same_as<ty> auto v1 = views::take(r, 2);
+  VERIFY( ranges::equal(v1, (int[]){1,2}) );
+
+  std::same_as<ty> auto v2 = views::drop(r, 3);
+  VERIFY( ranges::equal(v2, (int[]){4,5}) );
+
+  std::same_as<ty> auto v3 = views::counted(r.begin(), 4);
+  VERIFY( ranges::equal(v3, (int[]){1,2,3,4}) );
+}
+
+void
+test03()
+{
+  using namespace std::literals;
+  std::string_view r = "hello world"sv;
+  using ty = decltype(r);
+
+  std::same_as<ty> auto v1 = views::take(r, 5);
+  VERIFY( ranges::equal(v1, "hello"sv) );
+
+  std::same_as<ty> auto v2 = views::drop(r, 6);
+  VERIFY( ranges::equal(v2, "world"sv) );
+}
+
+void
+test04()
+{
+  int x[] = {1,2,3,4,5};
+  ranges::subrange r(x, x+5);
+  using ty = decltype(r);
+
+  std::same_as<ty> auto v1 = views::take(r, 2);
+  VERIFY( ranges::equal(v1, (int[]){1,2}) );
+
+  std::same_as<ty> auto v2 = views::drop(r, 1);
+  VERIFY( ranges::equal(v2, (int[]){2,3,4,5}) );
+}
+
+void
+test05()
+{
+  auto r1 = views::iota(1, 6);
+  using ty1 = decltype(r1);
+
+  std::same_as<ty1> auto v1 = views::take(r1, 2);
+  VERIFY( ranges::equal(v1, (int[]){1,2}) );
+
+  std::same_as<ty1> auto v2 = views::drop(r1, 3);
+  VERIFY( ranges::equal(v2, (int[]){4,5}) );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  test04();
+  test05();
+}
diff --git a/libstdc++-v3/testsuite/std/ranges/istream_view.cc b/libstdc++-v3/testsuite/std/ranges/istream_view.cc
index f5c0c2a..ea7c5a3 100644
--- a/libstdc++-v3/testsuite/std/ranges/istream_view.cc
+++ b/libstdc++-v3/testsuite/std/ranges/istream_view.cc
@@ -103,6 +103,18 @@ test06()
   static_assert( std::is_same_v<V, W> );
 }
 
+void
+test07()
+{
+  // P2432R1, views::istream
+  auto nums = std::istringstream("0 1 2 3 4");
+  ranges::istream_view<int> v(nums);
+  int sum = 0;
+  for (int val : views::istream<int>(nums))
+    sum += val;
+  VERIFY( sum == 10 );
+}
+
 int
 main()
 {
@@ -112,4 +124,5 @@ main()
   test04();
   test05();
   test06();
+  test07();
 }
diff --git a/libstdc++-v3/testsuite/util/testsuite_abi.cc b/libstdc++-v3/testsuite/util/testsuite_abi.cc
index 3af5dc5..1ca7da4 100644
--- a/libstdc++-v3/testsuite/util/testsuite_abi.cc
+++ b/libstdc++-v3/testsuite/util/testsuite_abi.cc
@@ -210,6 +210,7 @@ check_version(symbol& test, bool added)
       known_versions.push_back("GLIBCXX_3.4.27");
       known_versions.push_back("GLIBCXX_3.4.28");
       known_versions.push_back("GLIBCXX_3.4.29");
+      known_versions.push_back("GLIBCXX_3.4.30");
       known_versions.push_back("GLIBCXX_LDBL_3.4.29");
       known_versions.push_back("GLIBCXX_IEEE128_3.4.29");
       known_versions.push_back("CXXABI_1.3");
@@ -245,7 +246,7 @@ check_version(symbol& test, bool added)
 	test.version_status = symbol::incompatible;
 
       // Check that added symbols are added in the latest pre-release version.
-      bool latestp = (test.version_name == "GLIBCXX_3.4.29"
+      bool latestp = (test.version_name == "GLIBCXX_3.4.30"
 	  // XXX remove next 3 lines when baselines have been regenerated
 	  // to include {IEEE128,LDBL} symbols:
 		     || test.version_name == "GLIBCXX_LDBL_3.4.29"
diff --git a/libstdc++-v3/testsuite/util/testsuite_abi.h b/libstdc++-v3/testsuite/util/testsuite_abi.h
index 667c46c..4a0cf64 100644
--- a/libstdc++-v3/testsuite/util/testsuite_abi.h
+++ b/libstdc++-v3/testsuite/util/testsuite_abi.h
@@ -24,7 +24,11 @@
 #include <locale>
 #if __cplusplus >= 201103L
 # include <unordered_map>
+# ifdef _GLIBCXX_DEBUG
+namespace unord = std::_GLIBCXX_STD_C;
+# else
 namespace unord = std;
+# endif
 #else
 # include <tr1/unordered_map>
 namespace unord = std::tr1;